diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index 52f78ab5d649..9a70cb03b7c6 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -45,7 +45,7 @@ jobs: uses: actions/setup-python@v6 with: python-version: '3.13' - - run: python -m pip install "isort<6" + - run: python -m pip install isort - name: isort # Pinned to v3.0.0. uses: liskin/gh-problem-matcher-wrap@e7b7beaaafa52524748b31a381160759d68d61fb diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e1d8cec10bbd..f2a9217d6cc8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: files: 'docs/.*\.txt$' args: ["--rst-literal-block"] - repo: https://github.com/PyCQA/isort - rev: 5.13.2 + rev: 7.0.0 hooks: - id: isort - repo: https://github.com/PyCQA/flake8 diff --git a/django/contrib/contenttypes/fields.py b/django/contrib/contenttypes/fields.py index f98dda125506..62239dc715d8 100644 --- a/django/contrib/contenttypes/fields.py +++ b/django/contrib/contenttypes/fields.py @@ -16,6 +16,7 @@ ReverseManyToOneDescriptor, lazy_related_operation, ) +from django.db.models.query import prefetch_related_objects from django.db.models.query_utils import PathInfo from django.db.models.sql import AND from django.db.models.sql.where import WhereNode @@ -200,11 +201,13 @@ def get_prefetch_querysets(self, instances, querysets=None): for ct_id, fkeys in fk_dict.items(): if ct_id in custom_queryset_dict: # Return values from the custom queryset, if provided. - ret_val.extend(custom_queryset_dict[ct_id].filter(pk__in=fkeys)) + queryset = custom_queryset_dict[ct_id].filter(pk__in=fkeys) else: instance = instance_dict[ct_id] ct = self.field.get_content_type(id=ct_id, using=instance._state.db) - ret_val.extend(ct.get_all_objects_for_this_type(pk__in=fkeys)) + queryset = ct.get_all_objects_for_this_type(pk__in=fkeys) + + ret_val.extend(queryset.fetch_mode(instances[0]._state.fetch_mode)) # For doing the join in Python, we have to match both the FK val and # the content type, so we use a callable that returns a (fk, class) @@ -253,6 +256,15 @@ def __get__(self, instance, cls=None): return rel_obj else: rel_obj = None + + instance._state.fetch_mode.fetch(self, instance) + return self.field.get_cached_value(instance) + + def fetch_one(self, instance): + f = self.field.model._meta.get_field(self.field.ct_field) + ct_id = getattr(instance, f.attname, None) + pk_val = getattr(instance, self.field.fk_field) + rel_obj = None if ct_id is not None: ct = self.field.get_content_type(id=ct_id, using=instance._state.db) try: @@ -261,8 +273,14 @@ def __get__(self, instance, cls=None): ) except ObjectDoesNotExist: pass + else: + rel_obj._state.fetch_mode = instance._state.fetch_mode self.field.set_cached_value(instance, rel_obj) - return rel_obj + + def fetch_many(self, instances): + is_cached = self.field.is_cached + missing_instances = [i for i in instances if not is_cached(i)] + return prefetch_related_objects(missing_instances, self.field.name) def __set__(self, instance, value): ct = None @@ -622,7 +640,11 @@ def _apply_rel_filters(self, queryset): Filter the queryset for the instance this manager is bound to. """ db = self._db or router.db_for_read(self.model, instance=self.instance) - return queryset.using(db).filter(**self.core_filters) + return ( + queryset.using(db) + .fetch_mode(self.instance._state.fetch_mode) + .filter(**self.core_filters) + ) def _remove_prefetched_objects(self): try: diff --git a/django/core/exceptions.py b/django/core/exceptions.py index cbc80bd78fcb..0e24f6cb18a5 100644 --- a/django/core/exceptions.py +++ b/django/core/exceptions.py @@ -132,6 +132,12 @@ class FieldError(Exception): pass +class FieldFetchBlocked(FieldError): + """On-demand fetching of a model field blocked.""" + + pass + + NON_FIELD_ERRORS = "__all__" diff --git a/django/db/backends/postgresql/compiler.py b/django/db/backends/postgresql/compiler.py index 344773fb7ac1..a07ae3ea9216 100644 --- a/django/db/backends/postgresql/compiler.py +++ b/django/db/backends/postgresql/compiler.py @@ -1,10 +1,10 @@ -from django.db.models.sql.compiler import ( +from django.db.models.sql.compiler import ( # isort:skip SQLAggregateCompiler, SQLCompiler, SQLDeleteCompiler, + SQLInsertCompiler as BaseSQLInsertCompiler, + SQLUpdateCompiler, ) -from django.db.models.sql.compiler import SQLInsertCompiler as BaseSQLInsertCompiler -from django.db.models.sql.compiler import SQLUpdateCompiler __all__ = [ "SQLAggregateCompiler", diff --git a/django/db/models/__init__.py b/django/db/models/__init__.py index ec54b652409d..f15ddecfaa51 100644 --- a/django/db/models/__init__.py +++ b/django/db/models/__init__.py @@ -36,6 +36,7 @@ WindowFrame, WindowFrameExclusion, ) +from django.db.models.fetch_modes import FETCH_ONE, FETCH_PEERS, RAISE from django.db.models.fields import * # NOQA from django.db.models.fields import __all__ as fields_all from django.db.models.fields.composite import CompositePrimaryKey @@ -105,6 +106,9 @@ "GeneratedField", "JSONField", "OrderWrt", + "FETCH_ONE", + "FETCH_PEERS", + "RAISE", "Lookup", "Transform", "Manager", diff --git a/django/db/models/base.py b/django/db/models/base.py index fd51052d0199..b92a198660fe 100644 --- a/django/db/models/base.py +++ b/django/db/models/base.py @@ -32,6 +32,7 @@ from django.db.models.constants import LOOKUP_SEP from django.db.models.deletion import CASCADE, Collector from django.db.models.expressions import DatabaseDefault +from django.db.models.fetch_modes import FETCH_ONE from django.db.models.fields.composite import CompositePrimaryKey from django.db.models.fields.related import ( ForeignObjectRel, @@ -466,6 +467,14 @@ def __get__(self, instance, cls=None): return res +class ModelStateFetchModeDescriptor: + def __get__(self, instance, cls=None): + if instance is None: + return self + res = instance.fetch_mode = FETCH_ONE + return res + + class ModelState: """Store model instance state.""" @@ -476,6 +485,14 @@ class ModelState: # on the actual save. adding = True fields_cache = ModelStateFieldsCacheDescriptor() + fetch_mode = ModelStateFetchModeDescriptor() + peers = () + + def __getstate__(self): + state = self.__dict__.copy() + # Weak references can't be pickled. + state.pop("peers", None) + return state class Model(AltersData, metaclass=ModelBase): @@ -595,7 +612,7 @@ def __init__(self, *args, **kwargs): post_init.send(sender=cls, instance=self) @classmethod - def from_db(cls, db, field_names, values): + def from_db(cls, db, field_names, values, *, fetch_mode=None): if len(values) != len(cls._meta.concrete_fields): values_iter = iter(values) values = [ @@ -605,6 +622,8 @@ def from_db(cls, db, field_names, values): new = cls(*values) new._state.adding = False new._state.db = db + if fetch_mode is not None: + new._state.fetch_mode = fetch_mode return new def __repr__(self): @@ -714,8 +733,8 @@ def refresh_from_db(self, using=None, fields=None, from_queryset=None): should be an iterable of field attnames. If fields is None, then all non-deferred fields are reloaded. - When accessing deferred fields of an instance, the deferred loading - of the field will call this method. + When fetching deferred fields for a single instance (the FETCH_ONE + fetch mode), the deferred loading uses this method. """ if fields is None: self._prefetched_objects_cache = {} diff --git a/django/db/models/fetch_modes.py b/django/db/models/fetch_modes.py new file mode 100644 index 000000000000..2b5e6aa212dd --- /dev/null +++ b/django/db/models/fetch_modes.py @@ -0,0 +1,61 @@ +from django.core.exceptions import FieldFetchBlocked + + +class FetchMode: + __slots__ = () + + track_peers = False + + def fetch(self, fetcher, instance): + raise NotImplementedError("Subclasses must implement this method.") + + +class FetchOne(FetchMode): + __slots__ = () + + def fetch(self, fetcher, instance): + fetcher.fetch_one(instance) + + def __reduce__(self): + return "FETCH_ONE" + + +FETCH_ONE = FetchOne() + + +class FetchPeers(FetchMode): + __slots__ = () + + track_peers = True + + def fetch(self, fetcher, instance): + instances = [ + peer + for peer_weakref in instance._state.peers + if (peer := peer_weakref()) is not None + ] + if len(instances) > 1: + fetcher.fetch_many(instances) + else: + fetcher.fetch_one(instance) + + def __reduce__(self): + return "FETCH_PEERS" + + +FETCH_PEERS = FetchPeers() + + +class Raise(FetchMode): + __slots__ = () + + def fetch(self, fetcher, instance): + klass = instance.__class__.__qualname__ + field_name = fetcher.field.name + raise FieldFetchBlocked(f"Fetching of {klass}.{field_name} blocked.") from None + + def __reduce__(self): + return "RAISE" + + +RAISE = Raise() diff --git a/django/db/models/fields/related_descriptors.py b/django/db/models/fields/related_descriptors.py index 3e2150e0f670..4728233a6a15 100644 --- a/django/db/models/fields/related_descriptors.py +++ b/django/db/models/fields/related_descriptors.py @@ -78,7 +78,7 @@ class Child(Model): from django.db.models.fields.tuple_lookups import TupleIn from django.db.models.functions import RowNumber from django.db.models.lookups import GreaterThan, LessThanOrEqual -from django.db.models.query import QuerySet +from django.db.models.query import QuerySet, prefetch_related_objects from django.db.models.query_utils import DeferredAttribute from django.db.models.utils import AltersData, resolve_callables from django.utils.functional import cached_property @@ -166,8 +166,10 @@ def RelatedObjectDoesNotExist(self): def is_cached(self, instance): return self.field.is_cached(instance) - def get_queryset(self, **hints): - return self.field.remote_field.model._base_manager.db_manager(hints=hints).all() + def get_queryset(self, *, instance): + return self.field.remote_field.model._base_manager.db_manager( + hints={"instance": instance} + ).fetch_mode(instance._state.fetch_mode) def get_prefetch_querysets(self, instances, querysets=None): if querysets and len(querysets) != 1: @@ -175,8 +177,9 @@ def get_prefetch_querysets(self, instances, querysets=None): "querysets argument of get_prefetch_querysets() should have a length " "of 1." ) - queryset = querysets[0] if querysets else self.get_queryset() - queryset._add_hints(instance=instances[0]) + queryset = ( + querysets[0] if querysets else self.get_queryset(instance=instances[0]) + ) rel_obj_attr = self.field.get_foreign_related_value instance_attr = self.field.get_local_related_value @@ -254,13 +257,9 @@ def __get__(self, instance, cls=None): break if rel_obj is None and has_value: - rel_obj = self.get_object(instance) - remote_field = self.field.remote_field - # If this is a one-to-one relation, set the reverse accessor - # cache on the related object to the current instance to avoid - # an extra SQL query if it's accessed later on. - if not remote_field.multiple: - remote_field.set_cached_value(rel_obj, instance) + instance._state.fetch_mode.fetch(self, instance) + return self.field.get_cached_value(instance) + self.field.set_cached_value(instance, rel_obj) if rel_obj is None and not self.field.null: @@ -270,6 +269,21 @@ def __get__(self, instance, cls=None): else: return rel_obj + def fetch_one(self, instance): + rel_obj = self.get_object(instance) + self.field.set_cached_value(instance, rel_obj) + # If this is a one-to-one relation, set the reverse accessor cache on + # the related object to the current instance to avoid an extra SQL + # query if it's accessed later on. + remote_field = self.field.remote_field + if not remote_field.multiple: + remote_field.set_cached_value(rel_obj, instance) + + def fetch_many(self, instances): + is_cached = self.is_cached + missing_instances = [i for i in instances if not is_cached(i)] + prefetch_related_objects(missing_instances, self.field.name) + def __set__(self, instance, value): """ Set the related instance through the forward relation. @@ -384,6 +398,7 @@ def get_object(self, instance): obj = rel_model(**kwargs) obj._state.adding = instance._state.adding obj._state.db = instance._state.db + obj._state.fetch_mode = instance._state.fetch_mode return obj return super().get_object(instance) @@ -445,8 +460,10 @@ def RelatedObjectDoesNotExist(self): def is_cached(self, instance): return self.related.is_cached(instance) - def get_queryset(self, **hints): - return self.related.related_model._base_manager.db_manager(hints=hints).all() + def get_queryset(self, *, instance): + return self.related.related_model._base_manager.db_manager( + hints={"instance": instance} + ).fetch_mode(instance._state.fetch_mode) def get_prefetch_querysets(self, instances, querysets=None): if querysets and len(querysets) != 1: @@ -454,8 +471,9 @@ def get_prefetch_querysets(self, instances, querysets=None): "querysets argument of get_prefetch_querysets() should have a length " "of 1." ) - queryset = querysets[0] if querysets else self.get_queryset() - queryset._add_hints(instance=instances[0]) + queryset = ( + querysets[0] if querysets else self.get_queryset(instance=instances[0]) + ) rel_obj_attr = self.related.field.get_local_related_value instance_attr = self.related.field.get_foreign_related_value @@ -504,16 +522,8 @@ def __get__(self, instance, cls=None): if not instance._is_pk_set(): rel_obj = None else: - filter_args = self.related.field.get_forward_related_filter(instance) - try: - rel_obj = self.get_queryset(instance=instance).get(**filter_args) - except self.related.related_model.DoesNotExist: - rel_obj = None - else: - # Set the forward accessor cache on the related object to - # the current instance to avoid an extra SQL query if it's - # accessed later on. - self.related.field.set_cached_value(rel_obj, instance) + instance._state.fetch_mode.fetch(self, instance) + rel_obj = self.related.get_cached_value(instance) self.related.set_cached_value(instance, rel_obj) if rel_obj is None: @@ -524,6 +534,34 @@ def __get__(self, instance, cls=None): else: return rel_obj + @property + def field(self): + """ + Add compatibility with the fetcher protocol. While self.related is not + a field but a OneToOneRel, it quacks enough like a field to work. + """ + return self.related + + def fetch_one(self, instance): + # Kept for backwards compatibility with overridden + # get_forward_related_filter() + filter_args = self.related.field.get_forward_related_filter(instance) + try: + rel_obj = self.get_queryset(instance=instance).get(**filter_args) + except self.related.related_model.DoesNotExist: + rel_obj = None + else: + self.related.field.set_cached_value(rel_obj, instance) + self.related.set_cached_value(instance, rel_obj) + + def fetch_many(self, instances): + is_cached = self.is_cached + missing_instances = [i for i in instances if not is_cached(i)] + prefetch_related_objects( + missing_instances, + self.related.get_accessor_name(), + ) + def __set__(self, instance, value): """ Set the related instance through the reverse relation. @@ -703,6 +741,7 @@ def _apply_rel_filters(self, queryset): queryset._add_hints(instance=self.instance) if self._db: queryset = queryset.using(self._db) + queryset._fetch_mode = self.instance._state.fetch_mode queryset._defer_next_filter = True queryset = queryset.filter(**self.core_filters) for field in self.field.foreign_related_fields: @@ -1104,6 +1143,7 @@ def _apply_rel_filters(self, queryset): queryset._add_hints(instance=self.instance) if self._db: queryset = queryset.using(self._db) + queryset._fetch_mode = self.instance._state.fetch_mode queryset._defer_next_filter = True return queryset._next_is_sticky().filter(**self.core_filters) diff --git a/django/db/models/query.py b/django/db/models/query.py index 39cc9b6cb322..a2af672546bf 100644 --- a/django/db/models/query.py +++ b/django/db/models/query.py @@ -8,6 +8,7 @@ from contextlib import nullcontext from functools import reduce from itertools import chain, islice +from weakref import ref as weak_ref from asgiref.sync import sync_to_async @@ -26,6 +27,7 @@ from django.db.models.constants import LOOKUP_SEP, OnConflict from django.db.models.deletion import Collector from django.db.models.expressions import Case, DatabaseDefault, F, Value, When +from django.db.models.fetch_modes import FETCH_ONE from django.db.models.functions import Cast, Trunc from django.db.models.query_utils import FilteredRelation, Q from django.db.models.sql.constants import GET_ITERATOR_CHUNK_SIZE, ROW_COUNT @@ -88,6 +90,7 @@ def __iter__(self): queryset = self.queryset db = queryset.db compiler = queryset.query.get_compiler(using=db) + fetch_mode = queryset._fetch_mode # Execute the query. This will also fill compiler.select, klass_info, # and annotations. results = compiler.execute_sql( @@ -104,7 +107,7 @@ def __iter__(self): init_list = [ f[0].target.attname for f in select[model_fields_start:model_fields_end] ] - related_populators = get_related_populators(klass_info, select, db) + related_populators = get_related_populators(klass_info, select, db, fetch_mode) known_related_objects = [ ( field, @@ -122,10 +125,17 @@ def __iter__(self): ) for field, related_objs in queryset._known_related_objects.items() ] + peers = [] for row in compiler.results_iter(results): obj = model_cls.from_db( - db, init_list, row[model_fields_start:model_fields_end] + db, + init_list, + row[model_fields_start:model_fields_end], + fetch_mode=fetch_mode, ) + if fetch_mode.track_peers: + peers.append(weak_ref(obj)) + obj._state.peers = peers for rel_populator in related_populators: rel_populator.populate(row, obj) if annotation_col_map: @@ -183,10 +193,17 @@ def __iter__(self): query_iterator = compiler.composite_fields_to_tuples( query_iterator, cols ) + fetch_mode = self.queryset._fetch_mode + peers = [] for values in query_iterator: # Associate fields to values model_init_values = [values[pos] for pos in model_init_pos] - instance = model_cls.from_db(db, model_init_names, model_init_values) + instance = model_cls.from_db( + db, model_init_names, model_init_values, fetch_mode=fetch_mode + ) + if fetch_mode.track_peers: + peers.append(weak_ref(instance)) + instance._state.peers = peers if annotation_fields: for column, pos in annotation_fields: setattr(instance, column, values[pos]) @@ -293,6 +310,7 @@ def __init__(self, model=None, query=None, using=None, hints=None): self._prefetch_done = False self._known_related_objects = {} # {rel_field: {pk: rel_obj}} self._iterable_class = ModelIterable + self._fetch_mode = FETCH_ONE self._fields = None self._defer_next_filter = False self._deferred_filter = None @@ -665,6 +683,7 @@ def create(self, **kwargs): obj = self.model(**kwargs) self._for_write = True obj.save(force_insert=True, using=self.db) + obj._state.fetch_mode = self._fetch_mode return obj create.alters_data = True @@ -1442,6 +1461,7 @@ def raw(self, raw_query, params=(), translations=None, using=None): params=params, translations=translations, using=using, + fetch_mode=self._fetch_mode, ) qs._prefetch_related_lookups = self._prefetch_related_lookups[:] return qs @@ -1913,6 +1933,12 @@ def using(self, alias): clone._db = alias return clone + def fetch_mode(self, fetch_mode): + """Set the fetch mode for the QuerySet.""" + clone = self._chain() + clone._fetch_mode = fetch_mode + return clone + ################################### # PUBLIC INTROSPECTION ATTRIBUTES # ################################### @@ -2051,6 +2077,7 @@ def _clone(self): c._prefetch_related_lookups = self._prefetch_related_lookups[:] c._known_related_objects = self._known_related_objects c._iterable_class = self._iterable_class + c._fetch_mode = self._fetch_mode c._fields = self._fields return c @@ -2186,6 +2213,7 @@ def __init__( translations=None, using=None, hints=None, + fetch_mode=FETCH_ONE, ): self.raw_query = raw_query self.model = model @@ -2197,6 +2225,7 @@ def __init__( self._result_cache = None self._prefetch_related_lookups = () self._prefetch_done = False + self._fetch_mode = fetch_mode def resolve_model_init_order(self): """Resolve the init field names and value positions.""" @@ -2295,6 +2324,7 @@ def using(self, alias): params=self.params, translations=self.translations, using=alias, + fetch_mode=self._fetch_mode, ) @cached_property @@ -2758,8 +2788,9 @@ class RelatedPopulator: model instance. """ - def __init__(self, klass_info, select, db): + def __init__(self, klass_info, select, db, fetch_mode): self.db = db + self.fetch_mode = fetch_mode # Pre-compute needed attributes. The attributes are: # - model_cls: the possibly deferred model class to instantiate # - either: @@ -2812,7 +2843,9 @@ def __init__(self, klass_info, select, db): # relationship. Therefore checking for a single member of the primary # key is enough to determine if the referenced object exists or not. self.pk_idx = self.init_list.index(self.model_cls._meta.pk_fields[0].attname) - self.related_populators = get_related_populators(klass_info, select, self.db) + self.related_populators = get_related_populators( + klass_info, select, self.db, fetch_mode + ) self.local_setter = klass_info["local_setter"] self.remote_setter = klass_info["remote_setter"] @@ -2824,7 +2857,12 @@ def populate(self, row, from_obj): if obj_data[self.pk_idx] is None: obj = None else: - obj = self.model_cls.from_db(self.db, self.init_list, obj_data) + obj = self.model_cls.from_db( + self.db, + self.init_list, + obj_data, + fetch_mode=self.fetch_mode, + ) for rel_iter in self.related_populators: rel_iter.populate(row, obj) self.local_setter(from_obj, obj) @@ -2832,10 +2870,10 @@ def populate(self, row, from_obj): self.remote_setter(obj, from_obj) -def get_related_populators(klass_info, select, db): +def get_related_populators(klass_info, select, db, fetch_mode): iterators = [] related_klass_infos = klass_info.get("related_klass_infos", []) for rel_klass_info in related_klass_infos: - rel_cls = RelatedPopulator(rel_klass_info, select, db) + rel_cls = RelatedPopulator(rel_klass_info, select, db, fetch_mode) iterators.append(rel_cls) return iterators diff --git a/django/db/models/query_utils.py b/django/db/models/query_utils.py index c383b8064088..23d543211abc 100644 --- a/django/db/models/query_utils.py +++ b/django/db/models/query_utils.py @@ -264,7 +264,8 @@ def __get__(self, instance, cls=None): f"Cannot retrieve deferred field {field_name!r} " "from an unsaved model." ) - instance.refresh_from_db(fields=[field_name]) + + instance._state.fetch_mode.fetch(self, instance) else: data[field_name] = val return data[field_name] @@ -281,6 +282,20 @@ def _check_parent_chain(self, instance): return getattr(instance, link_field.attname) return None + def fetch_one(self, instance): + instance.refresh_from_db(fields=[self.field.attname]) + + def fetch_many(self, instances): + attname = self.field.attname + db = instances[0]._state.db + value_by_pk = ( + self.field.model._base_manager.using(db) + .values_list(attname) + .in_bulk({i.pk for i in instances}) + ) + for instance in instances: + setattr(instance, attname, value_by_pk[instance.pk]) + class class_or_instance_method: """ diff --git a/docs/internals/contributing/writing-code/coding-style.txt b/docs/internals/contributing/writing-code/coding-style.txt index 72429492cdb0..499866defaee 100644 --- a/docs/internals/contributing/writing-code/coding-style.txt +++ b/docs/internals/contributing/writing-code/coding-style.txt @@ -137,7 +137,7 @@ Imports .. console:: - $ python -m pip install "isort >= 5.1.0" + $ python -m pip install "isort >= 7.0.0" $ isort . This runs ``isort`` recursively from your current directory, modifying any diff --git a/docs/internals/contributing/writing-code/unit-tests.txt b/docs/internals/contributing/writing-code/unit-tests.txt index 974b4861d5e9..acbf68a7de9d 100644 --- a/docs/internals/contributing/writing-code/unit-tests.txt +++ b/docs/internals/contributing/writing-code/unit-tests.txt @@ -83,7 +83,7 @@ environments can be seen as follows: blacken-docs flake8>=3.7.0 docs - isort>=5.1.0 + isort>=7.0.0 lint-docs Testing other Python versions and database backends diff --git a/docs/ref/exceptions.txt b/docs/ref/exceptions.txt index bbd959e95d37..93c6ec420311 100644 --- a/docs/ref/exceptions.txt +++ b/docs/ref/exceptions.txt @@ -165,6 +165,16 @@ Django core exception classes are defined in ``django.core.exceptions``. - A field name is invalid - A query contains invalid order_by arguments +``FieldFetchBlocked`` +--------------------- + +.. versionadded:: 6.1 + +.. exception:: FieldFetchBlocked + + Raised when a field would be fetched on-demand and the + :attr:`~django.db.models.RAISE` fetch mode is active. + ``ValidationError`` ------------------- diff --git a/docs/ref/models/instances.txt b/docs/ref/models/instances.txt index c8cf5957ba4c..2ce8dc4a3655 100644 --- a/docs/ref/models/instances.txt +++ b/docs/ref/models/instances.txt @@ -180,10 +180,10 @@ update, you could write a test similar to this:: obj.refresh_from_db() self.assertEqual(obj.val, 2) -Note that when deferred fields are accessed, the loading of the deferred -field's value happens through this method. Thus it is possible to customize -the way deferred loading happens. The example below shows how one can reload -all of the instance's fields when a deferred field is reloaded:: +When a deferred field is loaded on-demand for a single model instance, the +loading happens through this method. Thus it is possible to customize the way +this loading happens. The example below shows how one can reload all of the +instance's fields when a deferred field is loaded on-demand:: class ExampleModel(models.Model): def refresh_from_db(self, using=None, fields=None, **kwargs): diff --git a/docs/ref/models/querysets.txt b/docs/ref/models/querysets.txt index f290970d2c3a..3840a2f97eae 100644 --- a/docs/ref/models/querysets.txt +++ b/docs/ref/models/querysets.txt @@ -1022,15 +1022,38 @@ Uses SQL's ``EXCEPT`` operator to keep only elements present in the See :meth:`union` for some restrictions. +``fetch_mode()`` +~~~~~~~~~~~~~~~~ + +.. versionadded:: 6.1 + +.. method:: fetch_mode(mode) + +Returns a ``QuerySet`` that sets the given fetch mode for all model instances +created by this ``QuerySet``. The fetch mode controls on-demand loading of +fields when they are accessed, such as for foreign keys and deferred fields. +For example, to use the :attr:`~django.db.models.FETCH_PEERS` mode to +batch-load all related objects on first access: + +.. code-block:: python + + from django.db import models + + books = Book.objects.fetch_mode(models.FETCH_PEERS) + +See more in the :doc:`fetch mode topic guide `. + ``select_related()`` ~~~~~~~~~~~~~~~~~~~~ .. method:: select_related(*fields) -Returns a ``QuerySet`` that will "follow" foreign-key relationships, selecting -additional related-object data when it executes its query. This is a -performance booster which results in a single more complex query but means -later use of foreign-key relationships won't require database queries. +Returns a ``QuerySet`` that will join in the named foreign-key relationships, +selecting additional related objects when it executes its query. This method +can be a performance booster, fetching data ahead of time rather than +triggering on-demand loading through the model instances' +:doc:`fetch mode `, at the cost of a more complex +initial query. The following examples illustrate the difference between plain lookups and ``select_related()`` lookups. Here's standard lookup:: @@ -1050,20 +1073,8 @@ And here's ``select_related`` lookup:: # in the previous query. b = e.blog -You can use ``select_related()`` with any queryset of objects:: - - from django.utils import timezone - - # Find all the blogs with entries scheduled to be published in the future. - blogs = set() - - for e in Entry.objects.filter(pub_date__gt=timezone.now()).select_related("blog"): - # Without select_related(), this would make a database query for each - # loop iteration in order to fetch the related blog for each entry. - blogs.add(e.blog) - -The order of ``filter()`` and ``select_related()`` chaining isn't important. -These querysets are equivalent:: +You can use ``select_related()`` with any queryset. The order of chaining with +other methods isn't important. For example, these querysets are equivalent:: Entry.objects.filter(pub_date__gt=timezone.now()).select_related("blog") Entry.objects.select_related("blog").filter(pub_date__gt=timezone.now()) @@ -1141,12 +1152,15 @@ that is that ``select_related('foo', 'bar')`` is equivalent to .. method:: prefetch_related(*lookups) -Returns a ``QuerySet`` that will automatically retrieve, in a single batch, -related objects for each of the specified lookups. +Returns a ``QuerySet`` that will automatically retrieve the given lookups, each +in one extra batch query. Prefetching is a way to optimize database access +when you know you'll be accessing related objects later, so you can avoid +triggering the on-demand loading behavior of the model instances' +:doc:`fetch mode `. -This has a similar purpose to ``select_related``, in that both are designed to -stop the deluge of database queries that is caused by accessing related -objects, but the strategy is quite different. +This method has a similar purpose to :meth:`select_related`, in that both are +designed to eagerly fetch related objects. However, they work in different +ways. ``select_related`` works by creating an SQL join and including the fields of the related object in the ``SELECT`` statement. For this reason, diff --git a/docs/releases/6.1.txt b/docs/releases/6.1.txt index 5e852785d9bd..80470dbcd68c 100644 --- a/docs/releases/6.1.txt +++ b/docs/releases/6.1.txt @@ -26,6 +26,51 @@ only officially support, the latest release of each series. What's new in Django 6.1 ======================== +Model field fetch modes +----------------------- + +The on-demand fetching behavior of model fields is now configurable with +:doc:`fetch modes `. These modes allow you to control +how Django fetches data from the database when an unfetched field is accessed. + +Django provides three fetch modes: + +1. ``FETCH_ONE``, the default, fetches the missing field for the current + instance only. This mode represents Django's existing behavior. + +2. ``FETCH_PEERS`` fetches a missing field for all instances that came from + the same :class:`~django.db.models.query.QuerySet`. + + This mode works like an on-demand ``prefetch_related()``. It can reduce most + cases of the "N+1 queries problem" to two queries without any work to + maintain a list of fields to prefetch. + +3. ``RAISE`` raises a :exc:`~django.core.exceptions.FieldFetchBlocked` + exception. + + This mode can prevent unintentional queries in performance-critical + sections of code. + +Use the new method :meth:`.QuerySet.fetch_mode` to set the fetch mode for model +instances fetched by the ``QuerySet``: + +.. code-block:: python + + from django.db import models + + books = Book.objects.fetch_mode(models.FETCH_PEERS) + for book in books: + print(book.author.name) + +Despite the loop accessing the ``author`` foreign key on each instance, the +``FETCH_PEERS`` fetch mode will make the above example perform only two +queries: + +1. Fetch all books. +2. Fetch associated authors. + +See :doc:`fetch modes ` for more details. + Minor features -------------- diff --git a/docs/spelling_wordlist b/docs/spelling_wordlist index 2898f85d5ba3..b35c94fc1019 100644 --- a/docs/spelling_wordlist +++ b/docs/spelling_wordlist @@ -535,6 +535,7 @@ unencrypted unescape unescaped unevaluated +unfetched unglamorous ungrouped unhandled diff --git a/docs/topics/db/fetch-modes.txt b/docs/topics/db/fetch-modes.txt new file mode 100644 index 000000000000..da7a07a0d49c --- /dev/null +++ b/docs/topics/db/fetch-modes.txt @@ -0,0 +1,143 @@ +=========== +Fetch modes +=========== + +.. versionadded:: 6.1 + +.. module:: django.db.models.fetch_modes + +.. currentmodule:: django.db.models + +When accessing model fields that were not loaded as part of the original query, +Django will fetch that field's data from the database. You can customize the +behavior of this fetching with a **fetch mode**, making it more efficient or +even blocking it. + +Use :meth:`.QuerySet.fetch_mode` to set the fetch mode for model +instances fetched by a ``QuerySet``: + +.. code-block:: python + + from django.db import models + + books = Book.objects.fetch_mode(models.FETCH_PEERS) + +Fetch modes apply to: + +* :class:`~django.db.models.ForeignKey` fields +* :class:`~django.db.models.OneToOneField` fields and their reverse accessors +* Fields deferred with :meth:`.QuerySet.defer` or :meth:`.QuerySet.only` +* :ref:`generic-relations` + +Django copies the fetch mode of an instance to any related objects it fetches, +so the mode applies to a whole tree of relationships, not just the top-level +model in the initial ``QuerySet``. This copying is also done in related +managers, even though fetch modes don't affect such managers' queries. + +Available modes +=============== + +.. admonition:: Referencing fetch modes + + Fetch modes are defined in ``django.db.models.fetch_modes``, but for + convenience they're imported into :mod:`django.db.models`. The standard + convention is to use ``from django.db import models`` and refer to the + fetch modes as ``models.``. + +Django provides three fetch modes. We'll explain them below using these models: + +.. code-block:: python + + from django.db import models + + + class Author(models.Model): ... + + + class Book(models.Model): + author = models.ForeignKey(Author, on_delete=models.CASCADE) + ... + +…and this loop: + +.. code-block:: python + + for book in books: + print(book.author.name) + +…where ``books`` is a ``QuerySet`` of ``Book`` instances using some fetch mode. + +.. attribute:: FETCH_ONE + +Fetches the missing field for the current instance only. This is the default +mode. + +Using ``FETCH_ONE`` for the above example would use: + +* 1 query to fetch ``books`` +* N queries, where N is the number of books, to fetch the missing ``author`` + field + +…for a total of 1+N queries. This query pattern is known as the "N+1 queries +problem" because it often leads to performance issues when N is large. + +.. attribute:: FETCH_PEERS + +Fetches the missing field for the current instance and its "peers"—instances +that came from the same initial ``QuerySet``. The behavior of this mode is +based on the assumption that if you need a field for one instance, you probably +need it for all instances in the same batch, since you'll likely process them +all identically. + +Using ``FETCH_PEERS`` for the above example would use: + +* 1 query to fetch ``books`` +* 1 query to fetch all missing ``author`` fields for the batch of books + +…for a total of 2 queries. The batch query makes this mode a lot more efficient +than ``FETCH_ONE`` and is similar to an on-demand call to +:meth:`.QuerySet.prefetch_related` or +:func:`~django.db.models.prefetch_related_objects`. Using ``FETCH_PEERS`` can +reduce most cases of the "N+1 queries problem" to two queries without +much effort. + +The "peer" instances are tracked in a list of weak references, to avoid +memory leaks where some peer instances are discarded. + +.. attribute:: RAISE + +Raises a :exc:`~django.core.exceptions.FieldFetchBlocked` exception. + +Using ``RAISE`` for the above example would raise an exception at the access of +``book.author`` access, like: + +.. code-block:: python + + FieldFetchBlocked("Fetching of Primary.value blocked.") + +This mode can prevent unintentional queries in performance-critical +sections of code. + +.. _fetch-modes-custom-manager: + +Make a fetch mode the default for a model class +=============================================== + +Set the default fetch mode for a model class with a +:ref:`custom manager ` that overrides ``get_queryset()``: + +.. code-block:: python + + from django.db import models + + + class BookManager(models.Manager): + def get_queryset(self): + return super().get_queryset().fetch_mode(models.FETCH_PEERS) + + + class Book(models.Model): + title = models.TextField() + author = models.ForeignKey("Author", on_delete=models.CASCADE) + + objects = BookManager() diff --git a/docs/topics/db/index.txt b/docs/topics/db/index.txt index 67a71fd820a9..6caf9f15e969 100644 --- a/docs/topics/db/index.txt +++ b/docs/topics/db/index.txt @@ -13,6 +13,7 @@ Generally, each model maps to a single database table. models queries + fetch-modes aggregation search managers diff --git a/docs/topics/db/optimization.txt b/docs/topics/db/optimization.txt index bb70efa36234..3be0bd2cb550 100644 --- a/docs/topics/db/optimization.txt +++ b/docs/topics/db/optimization.txt @@ -196,28 +196,46 @@ thousands of records are returned. The penalty will be compounded if the database lives on a separate server, where network overhead and latency also play a factor. -Retrieve everything at once if you know you will need it -======================================================== +Retrieve related objects efficiently +==================================== + +Generally, accessing the database multiple times to retrieve different parts +of a single "set" of data is less efficient than retrieving it all in one +query. This is particularly important if you have a query that is executed in a +loop, and could therefore end up doing many database queries, when only one +is needed. Below are some techniques to combine queries for efficiency. + +Use the ``FETCH_PEERS`` fetch mode +---------------------------------- -Hitting the database multiple times for different parts of a single 'set' of -data that you will need all parts of is, in general, less efficient than -retrieving it all in one query. This is particularly important if you have a -query that is executed in a loop, and could therefore end up doing many -database queries, when only one was needed. So: +Use the :attr:`~django.db.models.FETCH_PEERS` fetch mode to make on-demand +field access more efficient with bulk-fetching. Enable all it for all usage of +your models :ref:`with a custom manager `. + +Using this fetch mode is easier than declaring fields to fetch with +:meth:`~django.db.models.query.QuerySet.select_related` or +:meth:`~django.db.models.query.QuerySet.prefetch_related`, especially when it's +hard to predict which fields will be accessed. Use ``QuerySet.select_related()`` and ``prefetch_related()`` ------------------------------------------------------------ -Understand :meth:`~django.db.models.query.QuerySet.select_related` and -:meth:`~django.db.models.query.QuerySet.prefetch_related` thoroughly, and use -them: +When the :attr:`~django.db.models.FETCH_PEERS` fetch mode is not appropriate or +efficient enough, use :meth:`~django.db.models.query.QuerySet.select_related` +and :meth:`~django.db.models.query.QuerySet.prefetch_related`. Understand their +documentation thoroughly and apply them where needed. + +It may be useful to apply these methods in :doc:`managers and default managers +`. Be aware when your manager is and is not used; +sometimes this is tricky so don't make assumptions. -* in :doc:`managers and default managers ` where - appropriate. Be aware when your manager is and is not used; sometimes this is - tricky so don't make assumptions. +Use ``prefetch_related_objects()`` +---------------------------------- -* in view code or other layers, possibly making use of - :func:`~django.db.models.prefetch_related_objects` where needed. +Where :attr:`~django.db.models.query.QuerySet.prefetch_related` would be useful +after the queryset has been evaluated, use +:func:`~django.db.models.prefetch_related_objects` to execute an extra +prefetch. Don't retrieve things you don't need ==================================== diff --git a/docs/topics/db/queries.txt b/docs/topics/db/queries.txt index 3451f71fbacc..ed1d3ea9edd4 100644 --- a/docs/topics/db/queries.txt +++ b/docs/topics/db/queries.txt @@ -1683,15 +1683,15 @@ a join with an ``F()`` object, a ``FieldError`` will be raised: Related objects =============== -When you define a relationship in a model (i.e., a +When you define a relationship in a model (with :class:`~django.db.models.ForeignKey`, :class:`~django.db.models.OneToOneField`, or -:class:`~django.db.models.ManyToManyField`), instances of that model will have -a convenient API to access the related object(s). +:class:`~django.db.models.ManyToManyField`), instances of the model class gain +accessor attributes for the related object(s). Using the models at the top of this page, for example, an ``Entry`` object -``e`` can get its associated ``Blog`` object by accessing the ``blog`` -attribute: ``e.blog``. +``e`` has its associated ``Blog`` object accessible in its ``blog`` attribute: +``e.blog``. (Behind the scenes, this functionality is implemented by Python :doc:`descriptors `. This shouldn't really matter to @@ -1699,8 +1699,14 @@ you, but we point it out here for the curious.) Django also creates API accessors for the "other" side of the relationship -- the link from the related model to the model that defines the relationship. -For example, a ``Blog`` object ``b`` has access to a list of all related -``Entry`` objects via the ``entry_set`` attribute: ``b.entry_set.all()``. +For example, a ``Blog`` object ``b`` has a manager that returns all related +``Entry`` objects in the ``entry_set`` attribute: ``b.entry_set.all()``. + +These accessors may be prefetched by the ``QuerySet`` methods +:meth:`~django.db.models.query.QuerySet.select_related` or +:meth:`~django.db.models.query.QuerySet.prefetch_related`. If not prefetched, +access will trigger an on-demand fetch through the model's +:doc:`fetch mode `. All examples in this section use the sample ``Blog``, ``Author`` and ``Entry`` models defined at the top of this page. diff --git a/tests/basic/tests.py b/tests/basic/tests.py index 38d7d2a3d63c..ed655833e271 100644 --- a/tests/basic/tests.py +++ b/tests/basic/tests.py @@ -290,6 +290,13 @@ def test_create_method(self): ) self.assertEqual(Article.objects.get(headline="Article 10"), a10) + def test_create_method_propagates_fetch_mode(self): + article = Article.objects.fetch_mode(models.FETCH_PEERS).create( + headline="Article 10", + pub_date=datetime(2005, 7, 31, 12, 30, 45), + ) + self.assertEqual(article._state.fetch_mode, models.FETCH_PEERS) + def test_year_lookup_edge_case(self): # Edge-case test: A year lookup should retrieve all objects in # the given year, including Jan. 1 and Dec. 31. @@ -807,6 +814,7 @@ class ManagerTest(SimpleTestCase): "alatest", "aupdate", "aupdate_or_create", + "fetch_mode", ] def test_manager_methods(self): diff --git a/tests/contenttypes_tests/test_views.py b/tests/contenttypes_tests/test_views.py index bec79105c472..eb3ab0a92d9e 100644 --- a/tests/contenttypes_tests/test_views.py +++ b/tests/contenttypes_tests/test_views.py @@ -8,7 +8,7 @@ from django.http import Http404, HttpRequest from django.test import TestCase, override_settings -from .models import ( +from .models import ( # isort:skip Article, Author, FooWithBrokenAbsoluteUrl, @@ -17,9 +17,9 @@ ModelWithM2MToSite, ModelWithNullFKToSite, SchemeIncludedURL, + Site as MockSite, + UUIDModel, ) -from .models import Site as MockSite -from .models import UUIDModel @override_settings(ROOT_URLCONF="contenttypes_tests.urls") diff --git a/tests/defer/tests.py b/tests/defer/tests.py index c0968080b162..29c63c566acb 100644 --- a/tests/defer/tests.py +++ b/tests/defer/tests.py @@ -1,4 +1,5 @@ -from django.core.exceptions import FieldDoesNotExist, FieldError +from django.core.exceptions import FieldDoesNotExist, FieldError, FieldFetchBlocked +from django.db.models import FETCH_PEERS, RAISE from django.test import SimpleTestCase, TestCase from .models import ( @@ -29,6 +30,7 @@ class DeferTests(AssertionMixin, TestCase): def setUpTestData(cls): cls.s1 = Secondary.objects.create(first="x1", second="y1") cls.p1 = Primary.objects.create(name="p1", value="xx", related=cls.s1) + cls.p2 = Primary.objects.create(name="p2", value="yy", related=cls.s1) def test_defer(self): qs = Primary.objects.all() @@ -141,7 +143,6 @@ def test_defer_foreign_keys_are_deferred_and_not_traversed(self): def test_saving_object_with_deferred_field(self): # Saving models with deferred fields is possible (but inefficient, # since every field has to be retrieved first). - Primary.objects.create(name="p2", value="xy", related=self.s1) obj = Primary.objects.defer("value").get(name="p2") obj.name = "a new name" obj.save() @@ -181,10 +182,71 @@ def test_defer_of_overridden_scalar(self): self.assertEqual(obj.name, "adonis") def test_defer_fk_attname(self): - primary = Primary.objects.defer("related_id").get() + primary = Primary.objects.defer("related_id").get(name="p1") with self.assertNumQueries(1): self.assertEqual(primary.related_id, self.p1.related_id) + def test_only_fetch_mode_fetch_peers(self): + p1, p2 = Primary.objects.fetch_mode(FETCH_PEERS).only("name") + with self.assertNumQueries(1): + p1.value + with self.assertNumQueries(0): + p2.value + + def test_only_fetch_mode_fetch_peers_single(self): + p1 = Primary.objects.fetch_mode(FETCH_PEERS).only("name").get(name="p1") + with self.assertNumQueries(1): + p1.value + + def test_defer_fetch_mode_fetch_peers(self): + p1, p2 = Primary.objects.fetch_mode(FETCH_PEERS).defer("value") + with self.assertNumQueries(1): + p1.value + with self.assertNumQueries(0): + p2.value + + def test_defer_fetch_mode_fetch_peers_single(self): + p1 = Primary.objects.fetch_mode(FETCH_PEERS).defer("value").get(name="p1") + with self.assertNumQueries(1): + p1.value + + def test_only_fetch_mode_raise(self): + p1 = Primary.objects.fetch_mode(RAISE).only("name").get(name="p1") + msg = "Fetching of Primary.value blocked." + with self.assertRaisesMessage(FieldFetchBlocked, msg) as cm: + p1.value + self.assertIsNone(cm.exception.__cause__) + self.assertTrue(cm.exception.__suppress_context__) + + def test_defer_fetch_mode_raise(self): + p1 = Primary.objects.fetch_mode(RAISE).defer("value").get(name="p1") + msg = "Fetching of Primary.value blocked." + with self.assertRaisesMessage(FieldFetchBlocked, msg) as cm: + p1.value + self.assertIsNone(cm.exception.__cause__) + self.assertTrue(cm.exception.__suppress_context__) + + +class DeferOtherDatabaseTests(TestCase): + databases = {"other"} + + @classmethod + def setUpTestData(cls): + cls.s1 = Secondary.objects.using("other").create(first="x1", second="y1") + cls.p1 = Primary.objects.using("other").create( + name="p1", value="xx", related=cls.s1 + ) + cls.p2 = Primary.objects.using("other").create( + name="p2", value="yy", related=cls.s1 + ) + + def test_defer_fetch_mode_fetch_peers(self): + p1, p2 = Primary.objects.using("other").fetch_mode(FETCH_PEERS).defer("value") + with self.assertNumQueries(1, using="other"): + p1.value + with self.assertNumQueries(0, using="other"): + p2.value + class BigChildDeferTests(AssertionMixin, TestCase): @classmethod diff --git a/tests/foreign_object/tests.py b/tests/foreign_object/tests.py index 09fb47e771dc..233c59688590 100644 --- a/tests/foreign_object/tests.py +++ b/tests/foreign_object/tests.py @@ -5,6 +5,7 @@ from django.core.exceptions import FieldError, ValidationError from django.db import connection, models +from django.db.models import FETCH_PEERS from django.test import SimpleTestCase, TestCase, skipUnlessDBFeature from django.test.utils import CaptureQueriesContext, isolate_apps from django.utils import translation @@ -603,6 +604,42 @@ def test_isnull_lookup(self): [m4], ) + def test_fetch_mode_copied_forward_fetching_one(self): + person = Person.objects.fetch_mode(FETCH_PEERS).get(pk=self.bob.pk) + self.assertEqual(person._state.fetch_mode, FETCH_PEERS) + self.assertEqual( + person.person_country._state.fetch_mode, + FETCH_PEERS, + ) + + def test_fetch_mode_copied_forward_fetching_many(self): + people = list(Person.objects.fetch_mode(FETCH_PEERS)) + person = people[0] + self.assertEqual(person._state.fetch_mode, FETCH_PEERS) + self.assertEqual( + person.person_country._state.fetch_mode, + FETCH_PEERS, + ) + + def test_fetch_mode_copied_reverse_fetching_one(self): + country = Country.objects.fetch_mode(FETCH_PEERS).get(pk=self.usa.pk) + self.assertEqual(country._state.fetch_mode, FETCH_PEERS) + person = country.person_set.get(pk=self.bob.pk) + self.assertEqual( + person._state.fetch_mode, + FETCH_PEERS, + ) + + def test_fetch_mode_copied_reverse_fetching_many(self): + countries = list(Country.objects.fetch_mode(FETCH_PEERS)) + country = countries[0] + self.assertEqual(country._state.fetch_mode, FETCH_PEERS) + person = country.person_set.earliest("pk") + self.assertEqual( + person._state.fetch_mode, + FETCH_PEERS, + ) + class TestModelCheckTests(SimpleTestCase): @isolate_apps("foreign_object") diff --git a/tests/generic_relations/tests.py b/tests/generic_relations/tests.py index 1b53dbd8f45c..dceb8f4bae72 100644 --- a/tests/generic_relations/tests.py +++ b/tests/generic_relations/tests.py @@ -1,7 +1,8 @@ from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes.prefetch import GenericPrefetch -from django.core.exceptions import FieldError +from django.core.exceptions import FieldError, FieldFetchBlocked from django.db.models import Q, prefetch_related_objects +from django.db.models.fetch_modes import FETCH_PEERS, RAISE from django.test import SimpleTestCase, TestCase, skipUnlessDBFeature from .models import ( @@ -780,6 +781,76 @@ def test_generic_prefetch(self): self.platypus.latin_name, ) + def test_fetch_mode_fetch_peers(self): + TaggedItem.objects.bulk_create( + [ + TaggedItem(tag="lion", content_object=self.lion), + TaggedItem(tag="platypus", content_object=self.platypus), + TaggedItem(tag="quartz", content_object=self.quartz), + ] + ) + # Peers fetching should fetch all related peers GFKs at once which is + # one query per content type. + with self.assertNumQueries(1): + quartz_tag, platypus_tag, lion_tag = TaggedItem.objects.fetch_mode( + FETCH_PEERS + ).order_by("-pk")[:3] + with self.assertNumQueries(2): + self.assertEqual(lion_tag.content_object, self.lion) + with self.assertNumQueries(0): + self.assertEqual(platypus_tag.content_object, self.platypus) + self.assertEqual(quartz_tag.content_object, self.quartz) + # It should ignore already cached instances though. + with self.assertNumQueries(1): + quartz_tag, platypus_tag, lion_tag = TaggedItem.objects.fetch_mode( + FETCH_PEERS + ).order_by("-pk")[:3] + with self.assertNumQueries(2): + self.assertEqual(quartz_tag.content_object, self.quartz) + self.assertEqual(lion_tag.content_object, self.lion) + with self.assertNumQueries(0): + self.assertEqual(platypus_tag.content_object, self.platypus) + self.assertEqual(quartz_tag.content_object, self.quartz) + + def test_fetch_mode_raise(self): + tag = TaggedItem.objects.fetch_mode(RAISE).get(tag="yellow") + msg = "Fetching of TaggedItem.content_object blocked." + with self.assertRaisesMessage(FieldFetchBlocked, msg) as cm: + tag.content_object + self.assertIsNone(cm.exception.__cause__) + self.assertTrue(cm.exception.__suppress_context__) + + def test_fetch_mode_copied_forward_fetching_one(self): + tag = TaggedItem.objects.fetch_mode(FETCH_PEERS).get(tag="yellow") + self.assertEqual(tag.content_object, self.lion) + self.assertEqual( + tag.content_object._state.fetch_mode, + FETCH_PEERS, + ) + + def test_fetch_mode_copied_forward_fetching_many(self): + tags = list(TaggedItem.objects.fetch_mode(FETCH_PEERS).order_by("tag")) + tag = [t for t in tags if t.tag == "yellow"][0] + self.assertEqual(tag.content_object, self.lion) + self.assertEqual( + tag.content_object._state.fetch_mode, + FETCH_PEERS, + ) + + def test_fetch_mode_copied_reverse_fetching_one(self): + animal = Animal.objects.fetch_mode(FETCH_PEERS).get(pk=self.lion.pk) + self.assertEqual(animal._state.fetch_mode, FETCH_PEERS) + tag = animal.tags.get(tag="yellow") + self.assertEqual(tag._state.fetch_mode, FETCH_PEERS) + + def test_fetch_mode_copied_reverse_fetching_many(self): + animals = list(Animal.objects.fetch_mode(FETCH_PEERS)) + animal = animals[0] + self.assertEqual(animal._state.fetch_mode, FETCH_PEERS) + tags = list(animal.tags.all()) + tag = tags[0] + self.assertEqual(tag._state.fetch_mode, FETCH_PEERS) + class ProxyRelatedModelTest(TestCase): def test_default_behavior(self): diff --git a/tests/many_to_many/tests.py b/tests/many_to_many/tests.py index 34b7ffc67d8c..30fbde873efb 100644 --- a/tests/many_to_many/tests.py +++ b/tests/many_to_many/tests.py @@ -1,6 +1,7 @@ from unittest import mock from django.db import connection, transaction +from django.db.models import FETCH_PEERS from django.test import TestCase, skipIfDBFeature, skipUnlessDBFeature from .models import ( @@ -589,6 +590,46 @@ def test_get_prefetch_querysets_invalid_querysets_length(self): querysets=[Publication.objects.all(), Publication.objects.all()], ) + def test_fetch_mode_copied_forward_fetching_one(self): + a = Article.objects.fetch_mode(FETCH_PEERS).get(pk=self.a1.pk) + self.assertEqual(a._state.fetch_mode, FETCH_PEERS) + p = a.publications.earliest("pk") + self.assertEqual( + p._state.fetch_mode, + FETCH_PEERS, + ) + + def test_fetch_mode_copied_forward_fetching_many(self): + articles = list(Article.objects.fetch_mode(FETCH_PEERS)) + a = articles[0] + self.assertEqual(a._state.fetch_mode, FETCH_PEERS) + publications = list(a.publications.all()) + p = publications[0] + self.assertEqual( + p._state.fetch_mode, + FETCH_PEERS, + ) + + def test_fetch_mode_copied_reverse_fetching_one(self): + p1 = Publication.objects.fetch_mode(FETCH_PEERS).get(pk=self.p1.pk) + self.assertEqual(p1._state.fetch_mode, FETCH_PEERS) + a = p1.article_set.earliest("pk") + self.assertEqual( + a._state.fetch_mode, + FETCH_PEERS, + ) + + def test_fetch_mode_copied_reverse_fetching_many(self): + publications = list(Publication.objects.fetch_mode(FETCH_PEERS)) + p = publications[0] + self.assertEqual(p._state.fetch_mode, FETCH_PEERS) + articles = list(p.article_set.all()) + a = articles[0] + self.assertEqual( + a._state.fetch_mode, + FETCH_PEERS, + ) + class ManyToManyQueryTests(TestCase): """ diff --git a/tests/many_to_one/tests.py b/tests/many_to_one/tests.py index ac43c0da9574..4d2343e304fd 100644 --- a/tests/many_to_one/tests.py +++ b/tests/many_to_one/tests.py @@ -1,8 +1,13 @@ import datetime from copy import deepcopy -from django.core.exceptions import FieldError, MultipleObjectsReturned +from django.core.exceptions import ( + FieldError, + FieldFetchBlocked, + MultipleObjectsReturned, +) from django.db import IntegrityError, models, transaction +from django.db.models import FETCH_PEERS, RAISE from django.test import TestCase from django.utils.translation import gettext_lazy @@ -916,3 +921,72 @@ def test_get_prefetch_querysets_reverse_invalid_querysets_length(self): instances=countries, querysets=[City.objects.all(), City.objects.all()], ) + + def test_fetch_mode_fetch_peers_forward(self): + Article.objects.create( + headline="This is another test", + pub_date=datetime.date(2005, 7, 27), + reporter=self.r2, + ) + a1, a2 = Article.objects.fetch_mode(FETCH_PEERS) + with self.assertNumQueries(1): + a1.reporter + with self.assertNumQueries(0): + a2.reporter + + def test_fetch_mode_raise_forward(self): + a = Article.objects.fetch_mode(RAISE).get(pk=self.a.pk) + msg = "Fetching of Article.reporter blocked." + with self.assertRaisesMessage(FieldFetchBlocked, msg) as cm: + a.reporter + self.assertIsNone(cm.exception.__cause__) + self.assertTrue(cm.exception.__suppress_context__) + + def test_fetch_mode_copied_forward_fetching_one(self): + a1 = Article.objects.fetch_mode(FETCH_PEERS).get() + self.assertEqual(a1._state.fetch_mode, FETCH_PEERS) + self.assertEqual( + a1.reporter._state.fetch_mode, + FETCH_PEERS, + ) + + def test_fetch_mode_copied_forward_fetching_many(self): + Article.objects.create( + headline="This is another test", + pub_date=datetime.date(2005, 7, 27), + reporter=self.r2, + ) + a1, a2 = Article.objects.fetch_mode(FETCH_PEERS) + self.assertEqual(a1._state.fetch_mode, FETCH_PEERS) + self.assertEqual( + a1.reporter._state.fetch_mode, + FETCH_PEERS, + ) + + def test_fetch_mode_copied_reverse_fetching_one(self): + r1 = Reporter.objects.fetch_mode(FETCH_PEERS).get(pk=self.r.pk) + self.assertEqual(r1._state.fetch_mode, FETCH_PEERS) + article = r1.article_set.get() + self.assertEqual( + article._state.fetch_mode, + FETCH_PEERS, + ) + + def test_fetch_mode_copied_reverse_fetching_many(self): + Article.objects.create( + headline="This is another test", + pub_date=datetime.date(2005, 7, 27), + reporter=self.r2, + ) + r1, r2 = Reporter.objects.fetch_mode(FETCH_PEERS) + self.assertEqual(r1._state.fetch_mode, FETCH_PEERS) + a1 = r1.article_set.get() + self.assertEqual( + a1._state.fetch_mode, + FETCH_PEERS, + ) + a2 = r2.article_set.get() + self.assertEqual( + a2._state.fetch_mode, + FETCH_PEERS, + ) diff --git a/tests/model_inheritance_regress/tests.py b/tests/model_inheritance_regress/tests.py index 3310497de17c..adc2a22fc403 100644 --- a/tests/model_inheritance_regress/tests.py +++ b/tests/model_inheritance_regress/tests.py @@ -7,6 +7,7 @@ from unittest import expectedFailure from django import forms +from django.db.models import FETCH_PEERS from django.test import TestCase from .models import ( @@ -600,6 +601,22 @@ def test_queries_on_parent_access(self): self.assertEqual(restaurant.place_ptr.restaurant, restaurant) self.assertEqual(restaurant.italianrestaurant, italian_restaurant) + def test_parent_access_copies_fetch_mode(self): + italian_restaurant = ItalianRestaurant.objects.create( + name="Mom's Spaghetti", + address="2131 Woodward Ave", + serves_hot_dogs=False, + serves_pizza=False, + serves_gnocchi=True, + ) + + # No queries are made when accessing the parent objects. + italian_restaurant = ItalianRestaurant.objects.fetch_mode(FETCH_PEERS).get( + pk=italian_restaurant.pk + ) + restaurant = italian_restaurant.restaurant_ptr + self.assertEqual(restaurant._state.fetch_mode, FETCH_PEERS) + def test_id_field_update_on_ancestor_change(self): place1 = Place.objects.create(name="House of Pasta", address="944 Fullerton") place2 = Place.objects.create(name="House of Pizza", address="954 Fullerton") diff --git a/tests/one_to_one/tests.py b/tests/one_to_one/tests.py index d9bcb5d4dccf..39f24d6b1009 100644 --- a/tests/one_to_one/tests.py +++ b/tests/one_to_one/tests.py @@ -1,4 +1,6 @@ +from django.core.exceptions import FieldFetchBlocked from django.db import IntegrityError, connection, transaction +from django.db.models import FETCH_PEERS, RAISE from django.test import TestCase from .models import ( @@ -619,3 +621,77 @@ def test_get_prefetch_querysets_invalid_querysets_length(self): instances=places, querysets=[Bar.objects.all(), Bar.objects.all()], ) + + def test_fetch_mode_fetch_peers_forward(self): + Restaurant.objects.create( + place=self.p2, serves_hot_dogs=True, serves_pizza=False + ) + r1, r2 = Restaurant.objects.fetch_mode(FETCH_PEERS) + with self.assertNumQueries(1): + r1.place + with self.assertNumQueries(0): + r2.place + + def test_fetch_mode_fetch_peers_reverse(self): + Restaurant.objects.create( + place=self.p2, serves_hot_dogs=True, serves_pizza=False + ) + p1, p2 = Place.objects.fetch_mode(FETCH_PEERS) + with self.assertNumQueries(1): + p1.restaurant + with self.assertNumQueries(0): + p2.restaurant + + def test_fetch_mode_raise_forward(self): + r = Restaurant.objects.fetch_mode(RAISE).get(pk=self.r1.pk) + msg = "Fetching of Restaurant.place blocked." + with self.assertRaisesMessage(FieldFetchBlocked, msg) as cm: + r.place + self.assertIsNone(cm.exception.__cause__) + self.assertTrue(cm.exception.__suppress_context__) + + def test_fetch_mode_raise_reverse(self): + p = Place.objects.fetch_mode(RAISE).get(pk=self.p1.pk) + msg = "Fetching of Place.restaurant blocked." + with self.assertRaisesMessage(FieldFetchBlocked, msg) as cm: + p.restaurant + self.assertIsNone(cm.exception.__cause__) + self.assertTrue(cm.exception.__suppress_context__) + + def test_fetch_mode_copied_forward_fetching_one(self): + r1 = Restaurant.objects.fetch_mode(FETCH_PEERS).get(pk=self.r1.pk) + self.assertEqual(r1._state.fetch_mode, FETCH_PEERS) + self.assertEqual( + r1.place._state.fetch_mode, + FETCH_PEERS, + ) + + def test_fetch_mode_copied_forward_fetching_many(self): + Restaurant.objects.create( + place=self.p2, serves_hot_dogs=True, serves_pizza=False + ) + r1, r2 = Restaurant.objects.fetch_mode(FETCH_PEERS) + self.assertEqual(r1._state.fetch_mode, FETCH_PEERS) + self.assertEqual( + r1.place._state.fetch_mode, + FETCH_PEERS, + ) + + def test_fetch_mode_copied_reverse_fetching_one(self): + p1 = Place.objects.fetch_mode(FETCH_PEERS).get(pk=self.p1.pk) + self.assertEqual(p1._state.fetch_mode, FETCH_PEERS) + self.assertEqual( + p1.restaurant._state.fetch_mode, + FETCH_PEERS, + ) + + def test_fetch_mode_copied_reverse_fetching_many(self): + Restaurant.objects.create( + place=self.p2, serves_hot_dogs=True, serves_pizza=False + ) + p1, p2 = Place.objects.fetch_mode(FETCH_PEERS) + self.assertEqual(p1._state.fetch_mode, FETCH_PEERS) + self.assertEqual( + p1.restaurant._state.fetch_mode, + FETCH_PEERS, + ) diff --git a/tests/prefetch_related/tests.py b/tests/prefetch_related/tests.py index 54b197ad83d8..bb6417b8aecd 100644 --- a/tests/prefetch_related/tests.py +++ b/tests/prefetch_related/tests.py @@ -3,7 +3,14 @@ from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ObjectDoesNotExist from django.db import NotSupportedError, connection -from django.db.models import F, Prefetch, QuerySet, prefetch_related_objects +from django.db.models import ( + FETCH_PEERS, + F, + Prefetch, + QuerySet, + prefetch_related_objects, +) +from django.db.models.fetch_modes import RAISE from django.db.models.query import get_prefetcher from django.db.models.sql import Query from django.test import ( @@ -107,6 +114,32 @@ def test_foreignkey_forward(self): normal_books = [a.first_book for a in Author.objects.all()] self.assertEqual(books, normal_books) + def test_fetch_mode_copied_fetching_one(self): + author = ( + Author.objects.fetch_mode(FETCH_PEERS) + .prefetch_related("first_book") + .get(pk=self.author1.pk) + ) + self.assertEqual(author._state.fetch_mode, FETCH_PEERS) + self.assertEqual( + author.first_book._state.fetch_mode, + FETCH_PEERS, + ) + + def test_fetch_mode_copied_fetching_many(self): + authors = list( + Author.objects.fetch_mode(FETCH_PEERS).prefetch_related("first_book") + ) + self.assertEqual(authors[0]._state.fetch_mode, FETCH_PEERS) + self.assertEqual( + authors[0].first_book._state.fetch_mode, + FETCH_PEERS, + ) + + def test_fetch_mode_raise(self): + authors = list(Author.objects.fetch_mode(RAISE).prefetch_related("first_book")) + authors[0].first_book # No exception, already loaded + def test_foreignkey_reverse(self): with self.assertNumQueries(2): [ diff --git a/tests/queryset_pickle/tests.py b/tests/queryset_pickle/tests.py index acdb582a0a1a..074a8ed550b6 100644 --- a/tests/queryset_pickle/tests.py +++ b/tests/queryset_pickle/tests.py @@ -351,6 +351,29 @@ def test_order_by_model_with_abstract_inheritance_and_meta_ordering(self): event.edition_set.create() self.assert_pickles(event.edition_set.order_by("event")) + def test_fetch_mode_fetch_one(self): + restored = pickle.loads(pickle.dumps(self.happening)) + self.assertIs(restored._state.fetch_mode, models.FETCH_ONE) + + def test_fetch_mode_fetch_peers(self): + Happening.objects.create() + objs = list(Happening.objects.fetch_mode(models.FETCH_PEERS)) + self.assertEqual(objs[0]._state.fetch_mode, models.FETCH_PEERS) + self.assertEqual(len(objs[0]._state.peers), 2) + + restored = pickle.loads(pickle.dumps(objs)) + + self.assertIs(restored[0]._state.fetch_mode, models.FETCH_PEERS) + # Peers not restored because weak references are not picklable. + self.assertEqual(restored[0]._state.peers, ()) + + def test_fetch_mode_raise(self): + objs = list(Happening.objects.fetch_mode(models.RAISE)) + self.assertEqual(objs[0]._state.fetch_mode, models.RAISE) + + restored = pickle.loads(pickle.dumps(objs)) + self.assertIs(restored[0]._state.fetch_mode, models.RAISE) + class InLookupTests(TestCase): @classmethod diff --git a/tests/raw_query/tests.py b/tests/raw_query/tests.py index 853b7ee20e22..f66afbf28b73 100644 --- a/tests/raw_query/tests.py +++ b/tests/raw_query/tests.py @@ -1,7 +1,8 @@ from datetime import date from decimal import Decimal -from django.core.exceptions import FieldDoesNotExist +from django.core.exceptions import FieldDoesNotExist, FieldFetchBlocked +from django.db.models import FETCH_PEERS, RAISE from django.db.models.query import RawQuerySet from django.test import TestCase, skipUnlessDBFeature @@ -158,6 +159,22 @@ def test_FK_raw_query(self): books = Book.objects.all() self.assertSuccessfulRawQuery(Book, query, books) + def test_fk_fetch_mode_peers(self): + query = "SELECT * FROM raw_query_book" + books = list(Book.objects.fetch_mode(FETCH_PEERS).raw(query)) + with self.assertNumQueries(1): + books[0].author + books[1].author + + def test_fk_fetch_mode_raise(self): + query = "SELECT * FROM raw_query_book" + books = list(Book.objects.fetch_mode(RAISE).raw(query)) + msg = "Fetching of Book.author blocked." + with self.assertRaisesMessage(FieldFetchBlocked, msg) as cm: + books[0].author + self.assertIsNone(cm.exception.__cause__) + self.assertTrue(cm.exception.__suppress_context__) + def test_db_column_handler(self): """ Test of a simple raw query against a model containing a field with @@ -294,6 +311,23 @@ def test_missing_fields_without_PK(self): with self.assertRaisesMessage(FieldDoesNotExist, msg): list(Author.objects.raw(query)) + def test_missing_fields_fetch_mode_peers(self): + query = "SELECT id, first_name, dob FROM raw_query_author" + authors = list(Author.objects.fetch_mode(FETCH_PEERS).raw(query)) + with self.assertNumQueries(1): + authors[0].last_name + authors[1].last_name + + def test_missing_fields_fetch_mode_raise(self): + query = "SELECT id, first_name, dob FROM raw_query_author" + authors = list(Author.objects.fetch_mode(RAISE).raw(query)) + msg = "Fetching of Author.last_name blocked." + with self.assertRaisesMessage(FieldFetchBlocked, msg) as cm: + authors[0].last_name + self.assertIsNone(cm.exception.__cause__) + self.assertTrue(cm.exception.__suppress_context__) + self.assertTrue(cm.exception.__suppress_context__) + def test_annotations(self): query = ( "SELECT a.*, count(b.id) as book_count " diff --git a/tests/select_related/tests.py b/tests/select_related/tests.py index 68fe7a906fb1..41ed350cf3da 100644 --- a/tests/select_related/tests.py +++ b/tests/select_related/tests.py @@ -1,4 +1,5 @@ from django.core.exceptions import FieldError +from django.db.models import FETCH_PEERS from django.test import SimpleTestCase, TestCase from .models import ( @@ -210,6 +211,37 @@ def test_select_related_after_values_list(self): with self.assertRaisesMessage(TypeError, message): list(Species.objects.values_list("name").select_related("genus")) + def test_fetch_mode_copied_fetching_one(self): + fly = ( + Species.objects.fetch_mode(FETCH_PEERS) + .select_related("genus__family") + .get(name="melanogaster") + ) + self.assertEqual(fly._state.fetch_mode, FETCH_PEERS) + self.assertEqual( + fly.genus._state.fetch_mode, + FETCH_PEERS, + ) + self.assertEqual( + fly.genus.family._state.fetch_mode, + FETCH_PEERS, + ) + + def test_fetch_mode_copied_fetching_many(self): + specieses = list( + Species.objects.fetch_mode(FETCH_PEERS).select_related("genus__family") + ) + species = specieses[0] + self.assertEqual(species._state.fetch_mode, FETCH_PEERS) + self.assertEqual( + species.genus._state.fetch_mode, + FETCH_PEERS, + ) + self.assertEqual( + species.genus.family._state.fetch_mode, + FETCH_PEERS, + ) + class SelectRelatedValidationTests(SimpleTestCase): """ diff --git a/tox.ini b/tox.ini index ec90ceb1e4ff..4f1274a266b1 100644 --- a/tox.ini +++ b/tox.ini @@ -75,7 +75,7 @@ commands = [testenv:isort] basepython = python3 usedevelop = false -deps = isort >= 5.1.0 +deps = isort >= 7.0.0 changedir = {toxinidir} commands = isort --check-only --diff django tests scripts