diff --git a/CHANGELOG.md b/CHANGELOG.md index 51b4dd4b..109a935f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,23 @@ List of the most important changes for each release. +## 0.8.15 +- Improves buffer serialization performance by performing bulk counter lookup +- Corrects an initialization issue with `SessionContext` that leads to an incorrect value for `is_push` +- Save calls to Morango models during a sync are now scoped to only the changed fields + +## 0.8.14 +- Adds utility for addressing immediate FK constraints caused by Django upgrade, automatically performed for morango models in a Django migration. +- Adds retry behavior for low-level connection issues not handled by `urllib3` retries +- Allows repeat pushes and pulls of buffers during transfer +- Ignores HTTP 404 errors during sync or transfer session closure, which may occur if they're already closed + +## 0.8.13 +- Removes multiprocessing fallback for RSA key generation to avoid leaving zombie processes; key generation now stays in-process + +## 0.8.12 +- Fixes issue where dirty-bit isn't updated when calling save on a syncable model with `update_fields` + ## 0.8.11 - Adds additional `deserialization_exception` field to `Store` model to track the fully qualified exception path diff --git a/morango/api/serializers.py b/morango/api/serializers.py index 150959fc..7b872934 100644 --- a/morango/api/serializers.py +++ b/morango/api/serializers.py @@ -1,3 +1,5 @@ +from collections import defaultdict + from rest_framework import exceptions from rest_framework import serializers from rest_framework.fields import ReadOnlyField @@ -157,11 +159,40 @@ class Meta: read_only_fields = fields +class BufferListSerializer(serializers.ListSerializer): + def to_representation(self, data): + buffers = list(data) + if buffers: + rmcb_map = defaultdict(list) + transfer_session_ids = {b.transfer_session_id for b in buffers} + # Morango implementation will only ever call this for one transfer session ID at a time, + # but a loop makes the code straightforward regardless and limits the quantity of + # SQLite variables in use + for transfer_session_id in transfer_session_ids: + # bulk-fetch all RMCB records needed for this batch of buffers in a + # single query, then cache the relevant subset on each buffer, instead + # of letting each buffer's nested rmcb_list serializer issue its own query + rmcb_queryset = RecordMaxCounterBuffer.objects.filter( + transfer_session_id=transfer_session_id, + model_uuid__in={ + b.model_uuid + for b in buffers + if b.transfer_session_id == transfer_session_id + }, + ) + for rmcb in rmcb_queryset: + rmcb_map[(transfer_session_id, rmcb.model_uuid)].append(rmcb) + for buffer in buffers: + buffer._rmcb_list = rmcb_map[(buffer.transfer_session_id, buffer.model_uuid)] + return super(BufferListSerializer, self).to_representation(buffers) + + class BufferSerializer(serializers.ModelSerializer): rmcb_list = RecordMaxCounterBufferSerializer(many=True) class Meta: model = Buffer + list_serializer_class = BufferListSerializer fields = ( "serialized", "deleted", diff --git a/morango/deferrable_foreign_keys.py b/morango/deferrable_foreign_keys.py new file mode 100644 index 00000000..b680da05 --- /dev/null +++ b/morango/deferrable_foreign_keys.py @@ -0,0 +1,137 @@ +""" +Helpers for ensuring that existing SQLite foreign key constraints are +``DEFERRABLE INITIALLY DEFERRED``. + +Prior to Django 3.1, SQLite tables were created with immediate foreign key constraints, e.g.:: + + "store_model_id" char(32) NOT NULL REFERENCES "morango_store" ("id") + +Since Django 3.1 the same column is created as:: + + "store_model_id" char(32) NOT NULL REFERENCES "morango_store" ("id") DEFERRABLE INITIALLY DEFERRED + +Django relies on deferred constraint checking for correct cascade-deletion ordering. Databases +created before Morango 0.8 (with Django 1.11) therefore retain immediate constraints, which can +raise ``IntegrityError: FOREIGN KEY constraint failed`` during operations such as sync +deserialization once foreign key enforcement is enabled at the database level (the default since +Django 3.2). + +The migration framework does not regenerate these constraints on its own because the deferrable +clause is not part of the field definition that migrations track. This module rebuilds the affected +tables so that a migrated database ends up with the same schema as a freshly created one. +""" + +import logging +from typing import List +from typing import Optional + +from django.apps import apps as global_apps +from django.apps.registry import Apps +from django.db import connection +from django.db import DatabaseError +from django.db import router +from django.db.backends.base.schema import BaseDatabaseSchemaEditor +from django.db.backends.sqlite3.schema import DatabaseSchemaEditor + +logger = logging.getLogger(__name__) + + +def _get_table_sql(conn, table_name: str) -> Optional[str]: + """ + Return the ``CREATE TABLE`` statement stored by SQLite for ``table_name``, or ``None`` if the + table does not exist. + """ + with conn.cursor() as cursor: + cursor.execute( + "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = %s", + [table_name], + ) + row = cursor.fetchone() + return row[0] if row else None + + +def _table_has_immediate_foreign_key(conn, table_name: str) -> bool: + """ + Determine whether ``table_name`` has at least one foreign key constraint that is not deferrable. + Tables are created with all constraints in the same style, so the presence of a ``REFERENCES`` + clause without ``DEFERRABLE`` indicates an immediate constraint that needs to be rebuilt. + """ + sql = _get_table_sql(conn, table_name) + if not sql: + return False + references_count = sql.count("REFERENCES") + if references_count == 0: + return False + # make sure that all FK columns (identified by references) have a deferrable constraint + return references_count != sql.count("DEFERRABLE") + + +class MakeForeignKeysDeferrable: + """ + Invokable utility class that rebuilds of any table belonging to one of ``*app_labels`` whose + foreign key constraints are still immediate, so that they become + ``DEFERRABLE INITIALLY DEFERRED``. + + This can be utilized by passing this class to ``RunPython`` in a Django migration, or by + manually invoking its ``run`` method. + + This is a no-op on non-SQLite backends (PostgreSQL already creates deferrable + constraints) and on tables that already use deferrable constraints, so it is + safe to apply to any database. + """ + + def __init__( + self, + include_app_labels: Optional[List[str]] = None, + exclude_app_labels: Optional[List[str]] = None, + ): + self.include_app_labels = include_app_labels + self.exclude_app_labels = exclude_app_labels + + def _iter_apps(self, apps: Apps): + for app_config in apps.get_app_configs(): + if self.exclude_app_labels is not None and app_config.label in self.exclude_app_labels: + continue + if self.include_app_labels is None or app_config.label in self.include_app_labels: + yield app_config + + def __call__(self, apps: Apps, schema_editor: BaseDatabaseSchemaEditor): + """ + Loops through all relevant django apps and their models, checking if the model's table has + FKs, and if so, runs the schema editor's utility that synchronizes the schema to the model, + which will make FKs deferrable + """ + conn = schema_editor.connection + # these should be in sync, but since we call `_remake_table`, specific to this schema editor + # class, we ensure we have the correct instance to begin with + if conn.vendor != "sqlite" or not isinstance(schema_editor, DatabaseSchemaEditor): + return + + for app_config in self._iter_apps(apps): + for model in app_config.get_models(include_auto_created=True): + if not router.allow_migrate_model(conn.alias, model): + continue + table_name = model._meta.db_table + if not _table_has_immediate_foreign_key(conn, table_name): + continue + logger.info( + "Rebuilding table %s to make foreign key constraints deferrable", + table_name, + ) + try: + # Rebuilding the table from the current model state regenerates the column + # definitions, which SQLite always emits with the deferrable clause (see + # DatabaseSchemaEditor.sql_create_inline_fk). Only tested with Django 3.2 + schema_editor._remake_table(model) + except DatabaseError as e: + logger.error("Failed to rebuild table %s", table_name) + logger.exception(e) + + def run(self): + """ + Invoke this utility outside of Django migrations, manually passing in Django's app registry + and schema editor. + """ + # since this method is meant for manual invocation, we let the editor create a transaction + with connection.schema_editor(atomic=True) as schema_editor: + self.__call__(global_apps, schema_editor) diff --git a/morango/migrations/0004_deferrable_foreign_keys.py b/morango/migrations/0004_deferrable_foreign_keys.py new file mode 100644 index 00000000..421cf178 --- /dev/null +++ b/morango/migrations/0004_deferrable_foreign_keys.py @@ -0,0 +1,24 @@ +""" +Make existing foreign key constraints ``DEFERRABLE INITIALLY DEFERRED``. + +Databases created before Morango 0.8 (with Django 1.11) have immediate SQLite foreign key +constraints, which can break cascade deletions (e.g. during sync) now that foreign keys are enforced +at the database level. See https://github.com/learningequality/kolibri/issues/14884. +""" + +from django.db import migrations + +from morango.deferrable_foreign_keys import MakeForeignKeysDeferrable + + +class Migration(migrations.Migration): + dependencies = [ + ("morango", "0003_store_deserialization_errors"), + ] + + operations = [ + migrations.RunPython( + MakeForeignKeysDeferrable(include_app_labels=["morango"]), + migrations.RunPython.noop, + ), + ] diff --git a/morango/migrations/0004_self_ref_order.py b/morango/migrations/0005_self_ref_order.py similarity index 92% rename from morango/migrations/0004_self_ref_order.py rename to morango/migrations/0005_self_ref_order.py index 960f000a..4bd6b15d 100644 --- a/morango/migrations/0004_self_ref_order.py +++ b/morango/migrations/0005_self_ref_order.py @@ -7,7 +7,7 @@ class Migration(migrations.Migration): dependencies = [ - ("morango", "0003_store_deserialization_errors"), + ("morango", "0004_deferrable_foreign_keys"), ] operations = [ diff --git a/morango/models/core.py b/morango/models/core.py index bb27bd46..b32f35b3 100644 --- a/morango/models/core.py +++ b/morango/models/core.py @@ -322,6 +322,7 @@ def update_state(self, stage=None, stage_status=None): :type stage: morango.constants.transfer_stages.*|None :type stage_status: morango.constants.transfer_statuses.*|None """ + update_fields = [] if stage is not None: if self.transfer_stage and transfer_stages.stage( self.transfer_stage @@ -332,13 +333,18 @@ def update_state(self, stage=None, stage_status=None): ) ) self.transfer_stage = stage + update_fields.append("transfer_stage") + if stage_status is not None: self.transfer_stage_status = stage_status - if stage is not None or stage_status is not None: + update_fields.append("transfer_stage_status") + + if update_fields: self.last_activity_timestamp = timezone.now() - self.save() + update_fields.append("last_activity_timestamp") + self.save(update_fields=update_fields) self.sync_session.last_activity_timestamp = timezone.now() - self.sync_session.save() + self.sync_session.save(update_fields=["last_activity_timestamp"]) def delete_buffers(self): """ @@ -565,6 +571,11 @@ class Meta: unique_together = ("transfer_session", "model_uuid") def rmcb_list(self): + # allow callers (e.g. BufferListSerializer) to batch-fetch RMCB records + # for many buffers at once and cache them here, to avoid an N+1 query + # pattern when serializing a large number of buffers + if hasattr(self, "_rmcb_list"): + return self._rmcb_list return RecordMaxCounterBuffer.objects.filter( model_uuid=self.model_uuid, transfer_session_id=self.transfer_session_id ) @@ -844,6 +855,11 @@ def save(self, update_dirty_bit_to=True, *args, **kwargs): self._morango_dirty_bit = True elif not update_dirty_bit_to: self._morango_dirty_bit = False + + # ensure the dirty bit field is in the fields to update if present, to keep it in sync + if update_dirty_bit_to is not None and kwargs.get("update_fields") is not None: + kwargs["update_fields"] = set(kwargs["update_fields"]) | {"_morango_dirty_bit"} + super(SyncableModel, self).save(*args, **kwargs) def delete(self, using=None, keep_parents=False, hard_delete=False, *args, **kwargs): diff --git a/morango/models/fields/crypto.py b/morango/models/fields/crypto.py index 4db78843..5a059e1b 100644 --- a/morango/models/fields/crypto.py +++ b/morango/models/fields/crypto.py @@ -178,10 +178,7 @@ class PythonRSAKey(BaseKey): _private_key = None def generate_new_key(self, keysize=2048): - try: - self._public_key, self._private_key = PYRSA.newkeys(keysize, poolsize=4) - except: # noqa: E722 - self._public_key, self._private_key = PYRSA.newkeys(keysize) + self._public_key, self._private_key = PYRSA.newkeys(keysize) def _sign(self, message): diff --git a/morango/sync/context.py b/morango/sync/context.py index 1c040966..3c4d72d3 100644 --- a/morango/sync/context.py +++ b/morango/sync/context.py @@ -52,7 +52,7 @@ def __init__( if self.transfer_session: self.sync_session = transfer_session.sync_session or self.sync_session - self.is_push = transfer_session.push or self.is_push + self.is_push = transfer_session.push if transfer_session.filter: self.filter = transfer_session.get_filter() diff --git a/morango/sync/controller.py b/morango/sync/controller.py index e29d944c..5fe180a8 100644 --- a/morango/sync/controller.py +++ b/morango/sync/controller.py @@ -254,7 +254,7 @@ def _invoke_middleware(self, context, middleware): return context.stage_status except Exception as e: # always log the error itself - logger.error(e) + logger.exception(e) context.update(stage_status=transfer_statuses.ERRORED, error=e) # fire completed signal, after context update. handlers can use context to detect error signal.completed.fire(context=prepared_context or context) diff --git a/morango/sync/operations.py b/morango/sync/operations.py index a56bdf11..f8452443 100644 --- a/morango/sync/operations.py +++ b/morango/sync/operations.py @@ -952,9 +952,11 @@ def handle(self, context): context.transfer_session.client_fsic = context.request.data.get( "client_fsic", "{}" ) + update_fields = ["server_fsic", "client_fsic"] else: context.transfer_session.client_fsic = fsic - context.transfer_session.save() + update_fields = ["client_fsic"] + context.transfer_session.save(update_fields=update_fields) return transfer_statuses.COMPLETED @@ -983,7 +985,7 @@ def handle(self, context): logger.debug("[morango] Queued {} records".format(records_total)) context.transfer_session.records_total = records_total - context.transfer_session.save() + context.transfer_session.save(update_fields=["records_total"]) return transfer_statuses.COMPLETED @@ -1192,7 +1194,7 @@ def handle(self, context): context.transfer_session.delete_buffers() context.transfer_session.active = False - context.transfer_session.save() + context.transfer_session.save(update_fields=["active"]) return transfer_statuses.COMPLETED @@ -1240,9 +1242,8 @@ def close_transfer_session(self, context): Closes remote transfer session :type context: NetworkSessionContext - :return: The Response """ - return context.connection._close_transfer_session(context.transfer_session) + context.connection._close_transfer_session(context.transfer_session) def put_buffers(self, context, buffers): """ @@ -1344,6 +1345,7 @@ def handle(self, context): data = self.create_transfer_session(context) context.transfer_session.server_fsic = data.get("server_fsic") or "{}" + update_fields = ["server_fsic"] # A legacy instance performs queuing during the creation of the transfer session, so since we use a new # workflow we need to update the network server when pushing to say how many records we've queued. For pull, @@ -1351,8 +1353,9 @@ def handle(self, context): # since that's when it's first available. if context.transfer_session.pull: context.transfer_session.records_total = data.get("records_total", 0) + update_fields.append("records_total") - context.transfer_session.save() + context.transfer_session.save(update_fields=update_fields) return transfer_statuses.COMPLETED @@ -1420,7 +1423,7 @@ def handle(self, context): if remote_status == transfer_statuses.COMPLETED: context.transfer_session.server_fsic = data.get("server_fsic") - context.transfer_session.save() + context.transfer_session.save(update_fields=["server_fsic"]) return remote_status @@ -1469,7 +1472,7 @@ def handle(self, context): if context.is_pull and remote_status == transfer_statuses.COMPLETED: context.transfer_session.records_total = data.get("records_total", 0) - context.transfer_session.save() + context.transfer_session.save(update_fields=["records_total"]) return remote_status @@ -1505,7 +1508,7 @@ def handle(self, context): ) context.transfer_session.bytes_sent = context.connection.bytes_sent context.transfer_session.bytes_received = context.connection.bytes_received - context.transfer_session.save() + context.transfer_session.save(update_fields=["records_transferred", "bytes_sent", "bytes_received"]) # if we've transferred all records, return a completed status op_status = transfer_statuses.PENDING @@ -1613,8 +1616,5 @@ def handle(self, context): """ :type context: NetworkSessionContext """ - response = self.close_transfer_session(context) - remote_status = transfer_statuses.COMPLETED - if response.status_code < 200 or response.status_code >= 300: - remote_status = transfer_statuses.ERRORED - return remote_status + self.close_transfer_session(context) + return transfer_statuses.COMPLETED diff --git a/morango/sync/session.py b/morango/sync/session.py index c8ee2b18..6961acc9 100644 --- a/morango/sync/session.py +++ b/morango/sync/session.py @@ -1,18 +1,37 @@ import logging +from contextlib import contextmanager from requests import exceptions -from requests.packages.urllib3.util.url import parse_url +from requests.adapters import HTTPAdapter from requests.sessions import Session from requests.utils import super_len +from urllib3.exceptions import MaxRetryError +from urllib3.util.url import parse_url from morango import __version__ +from morango.utils import nullcontext from morango.utils import serialize_capabilities_to_client_request from morango.utils import SETTINGS logger = logging.getLogger(__name__) +_RETRY_REQUEST_EXCEPTIONS = ( + exceptions.ConnectionError, + exceptions.ChunkedEncodingError, + exceptions.ContentDecodingError, +) + + def _headers_content_length(headers): + """ + Obtains the value of 'Content-Length' from the provided headers. + + :param headers: Dictionary of headers + :type headers: dict + :return: The integer value of 'Content-Length' if found and valid, otherwise 0. + :rtype: int + """ try: content_length = int(headers.get("Content-Length", 0)) if content_length > 0: @@ -23,19 +42,78 @@ def _headers_content_length(headers): def _length_of_headers(headers): + """ + Calculates the total length of all headers. + + :param headers: Dictionary of headers + :type headers: dict + :return: The total length of the string representation of all headers. + :rtype: int + """ return super_len("\n".join(["{}: {}".format(key, value) for key, value in headers.items()])) +def _is_retryable_method(retries, method): + """ + Checks if the request method is configured as retryable. + + :type retries: urllib3.util.retry.Retry + :type method: str|None + :rtype: bool + """ + allowed_methods = getattr(retries, "allowed_methods", None) + if allowed_methods is False or allowed_methods is None: + return True + return method.upper() in allowed_methods if method is not None else False + + +def _log_response_error(err, response): + """ + Logs an error and its associated response content. + + :param err: The exception instance that represents the error encountered. + :type err: Exception + :param response: The HTTP response object associated with the error. + If None, it is interpreted as no response being available. + :type response: Optional[Response] + """ + try: + response_content = response.content if response else "(no response)" + except Exception: + response_content = "(unable to read response)" + logger.error("{} Reason: {}".format(err.__class__.__name__, response_content)) + + +class ContextualRetryHTTPAdapter(HTTPAdapter): + @contextmanager + def use_retries(self, max_retries): + """ + Context manager for temporarily changing the retry configuration. + + :param max_retries: The temporary Retry object + :type max_retries: urllib3.util.retry.Retry + """ + original_retries = self.max_retries + try: + self.max_retries = max_retries + yield + finally: + self.max_retries = original_retries + + class SessionWrapper(Session): """ Wrapper around `requests.sessions.Session` in order to implement logging around all request errors. """ - bytes_sent = 0 - bytes_received = 0 - - def __init__(self): + def __init__(self, max_retries): + """ + :param max_retries: The urllib3 Retry object + :type max_retries: urllib3.util.retry.Retry + """ super(SessionWrapper, self).__init__() + self.max_retries = max_retries + user_agent_header = "morango/{}".format(__version__) if SETTINGS.CUSTOM_INSTANCE_INFO is not None: instances = list(SETTINGS.CUSTOM_INSTANCE_INFO) @@ -44,56 +122,195 @@ def __init__(self): instances[0], SETTINGS.CUSTOM_INSTANCE_INFO.get(instances[0]) ) self.headers["User-Agent"] = "{} {}".format(user_agent_header, self.headers["User-Agent"]) + self.hooks["response"].append(self._track_bytes_received) + self.bytes_sent = 0 + self.bytes_received = 0 - def request(self, method, url, **kwargs): - response = None + # use custom adapter + adapter = ContextualRetryHTTPAdapter() + self.mount("http://", adapter) + self.mount("https://", adapter) + + def _track_bytes_sent(self, request): + """ + Request hook that tracks the size of the request, by capturing the size of headers and the + request body. Note: python requests only supports the `response` hook, so this is invoked + manually + + :type request: requests.Request|requests.PreparedRequest + """ try: - response = super(SessionWrapper, self).request(method, url, **kwargs) + parsed_url = parse_url(request.url) + # we don't bother checking if the content length header exists here because we've probably + # been given the request body as Morango sends bodies that aren't streamed, so the + # underlying requests code will set it appropriately + self.bytes_sent += len("{} {} HTTP/1.1".format(request.method, parsed_url.path)) + self.bytes_sent += _length_of_headers(request.headers) + self.bytes_sent += _headers_content_length(request.headers) + except Exception as e: + # tracking bandwidth usage is useful but not critical + logger.exception(e) - # capture bytes received from the response, the length header could be missing if it's - # a chunked response though - content_length = _headers_content_length(response.headers) - if not content_length: - content_length = super_len(response.content) + def _track_bytes_received(self, response, *args, **kwargs): + """ + Response hook that tracks the size of the response, by capturing the size of headers and + the response body + :type response: requests.Response + """ + try: + # headers: self.bytes_received += len( "HTTP/1.1 {} {}".format(response.status_code, response.reason) ) self.bytes_received += _length_of_headers(response.headers) - self.bytes_received += content_length - response.raise_for_status() - return response - except exceptions.RequestException as req_err: - # we want to log all request errors for debugging purposes - if response is None: - response = req_err.response + # body: + # capture bytes received from the response, the length header could be missing if it's + # a chunked response though + content_length = _headers_content_length(response.headers) + if not content_length: + content_length = super_len(response.content) + self.bytes_received += content_length + except Exception as e: + # tracking bandwidth usage is useful but not critical + logger.exception(e) - response_content = response.content if response else "(no response)" - logger.error("{} Reason: {}".format(req_err.__class__.__name__, response_content)) - raise req_err + def _get_adapter(self, url): + """ + :param url: the request URL + :type url: bytes|str|None + :rtype: Optional[HTTPAdapter] + """ + if url is None: + return None + # requests allows bytes + if isinstance(url, bytes): + url = url.decode("utf-8") + try: + return self.get_adapter(url) + except exceptions.InvalidSchema: + return None def prepare_request(self, request): """ - Override request preparer so we can get the prepared content length, for tracking - transfer sizes + Override request preparer so we can add morango capabilities to the request, and invoke + the sent bytes hook. :type request: requests.Request :rtype: requests.PreparedRequest """ # add header with client's morango capabilities so server has that information serialize_capabilities_to_client_request(request) - prepped = super(SessionWrapper, self).prepare_request(request) - parsed_url = parse_url(request.url) + return super(SessionWrapper, self).prepare_request(request) - # we don't bother checking if the content length header exists here because we've probably - # been given the request body as Morango sends bodies that aren't streamed, so the - # underlying requests code will set it appropriately - self.bytes_sent += len("{} {} HTTP/1.1".format(request.method, parsed_url.path)) - self.bytes_sent += _length_of_headers(prepped.headers) - self.bytes_sent += _headers_content_length(prepped.headers) + def request(self, method, url, **kwargs): + """ + Issues an HTTP request, with conditional retry behavior if passed `is_retryable` kwarg, and + logs any errors from the request flow. - return prepped + :param method: The HTTP request method (e.g., 'GET', 'POST', 'PUT', etc.). + :type method: str + :param url: The URL to send the request to. + :type url: str + :param kwargs: Additional arguments to pass to the underlying request method. + :return: The HTTP response object obtained from the request. + :rtype: requests.Response + :raises Exception: Logs then re-raises any exception that occurs during the request. + """ + adapter = self._get_adapter(url) + + # super's request has strict kwarg list, so we have to pop `is_retryable` and modify + # the adapter state based on the value + is_retryable = kwargs.pop("is_retryable", False) + if is_retryable and isinstance(adapter, ContextualRetryHTTPAdapter): + ctx = adapter.use_retries(self.max_retries) + else: + ctx = nullcontext() + + with ctx: + response = None + try: + response = super(SessionWrapper, self).request(method, url, **kwargs) + response.raise_for_status() + return response + except Exception as e: + if response is None: + response = getattr(e, "response", None) + + _log_response_error(e, response) + raise e + + def send(self, request, **kwargs): + """ + Issues an HTTP request with automatic retry handling for transport-level failures and + logging of request-related errors. + + :param request: The prepared request + :type request: requests.PreparedRequest + :param kwargs: Additional arguments to pass to the underlying `send` method. + :return: The HTTP response object obtained from the request. + :rtype: requests.Response + """ + adapter = self._get_adapter(request.url) + retries = adapter.max_retries if adapter is not None else None + + while True: + # sent bytes from low-level retries in urllib3 are not captured, but this is good enough + self._track_bytes_sent(request) + response = None + try: + response = super(SessionWrapper, self).send(request, **kwargs) + return response + except exceptions.RequestException as e: + retries = self._should_retry(retries, request, e, response=response) + if retries is None: + raise e + + def _should_retry(self, retries, request, e, response=None): + """ + Determines whether a request should be retried based on the provided criteria, including + the retry settings, the type of exception occurred, and the HTTP method used. This method + also handles sleeping between retries and increments retry counters appropriately. If + retries are exhausted, it raises a `RetryError`. + + :param retries: The retry configuration instance that tracks and manages retry attempts. + :type retries: Optional[urllib3.util.retry.Retry] + :param request: The HTTP request object that specifies the details of the request being made. + :type request: requests.PreparedRequest + :param e: The exception raised during the request execution that triggered this check. + :type e: requests.exceptions.RequestException + :param response: Optional. The HTTP response object corresponding to the request, if available. + :type response: Optional[requests.Response] + :return: The new Retry object if the retry is allowed, otherwise None. + :rtype: Optional[urllib3.util.retry.Retry] + """ + if response is None: + response = e.response + + if ( + retries is None + or retries.total == 0 + or not isinstance(e, _RETRY_REQUEST_EXCEPTIONS) + or not _is_retryable_method(retries, request.method) + ): + return None + + # requests might wrap `MaxRetryError` in a `ConnectionError` + if any(isinstance(arg, MaxRetryError) for arg in e.args): + return None + + try: + # may raise if retries have been exhausted + retries = retries.increment( + method=request.method, url=request.url, response=response, error=e + ) + logger.debug(f"Sleeping before retrying {request.method} request to {request.url}") + retries.sleep(response) + return retries + except MaxRetryError as _e: + # re-raise wrapped exception, like requests would. see `HTTPAdapter.send` + raise exceptions.RetryError(_e.reason, request=request, response=response) def reset_transfer_bytes(self): """ diff --git a/morango/sync/syncsession.py b/morango/sync/syncsession.py index 9c387e98..777004ec 100644 --- a/morango/sync/syncsession.py +++ b/morango/sync/syncsession.py @@ -7,6 +7,7 @@ import os import socket import uuid +from functools import wraps from io import BytesIO from urllib.parse import urljoin from urllib.parse import urlparse @@ -14,9 +15,8 @@ from django.db import connection from django.db import transaction from django.utils import timezone -from requests.adapters import HTTPAdapter from requests.exceptions import HTTPError -from requests.packages.urllib3.util.retry import Retry +from urllib3.util.retry import Retry from morango.api.serializers import CertificateSerializer from morango.api.serializers import InstanceIDSerializer @@ -80,6 +80,26 @@ def _get_client_ip_for_server(server_host, server_port): return IP +def ignore_404(target): + """ + Decorator that wraps callables to ignore 404s caused by its use of requests + :param target: A callable + :return: A callable + """ + + @wraps(target) + def wrapper(*args, **kwargs): + try: + return target(*args, **kwargs) + except HTTPError as e: + if e.response is None or e.response.status_code != 404: + raise e + else: + logger.debug(f"Ignoring 404 raised by {target.__name__}") + + return wrapper + + # borrowed from https://github.com/django/django/blob/1.11.20/django/utils/text.py#L295 def compress_string(s, compresslevel=9): zbuf = BytesIO() @@ -132,15 +152,19 @@ def __init__( self.base_url = base_url self.compresslevel = compresslevel # set up requests session with retry logic - self.session = SessionWrapper() # sleep for {backoff factor} * (2 ^ ({number of total retries} - 1)) between requests # with 7 retry attempts, sleep escalation becomes (0.6s, 1.2s, ..., 38.4s) - retry = Retry(total=retries, backoff_factor=backoff_factor) - adapter = HTTPAdapter(max_retries=retry) - self.session.mount("http://", adapter) - self.session.mount("https://", adapter) + self.session = SessionWrapper( + Retry( + total=retries, + backoff_factor=backoff_factor, + allowed_methods=None, # allow any method + ) + ) # get morango information about server - self.server_info = self.session.get(urljoin(self.base_url, api_urls.INFO)).json() + self.server_info = self.session.get( + urljoin(self.base_url, api_urls.INFO), is_retryable=True + ).json() self.capabilities = self.server_info.get("capabilities", []) self.chunk_size = chunk_size @@ -409,13 +433,15 @@ def push_signed_client_certificate_chain( return certificate def _get_public_key(self): - return self.session.get(self.urlresolve(api_urls.PUBLIC_KEY)) + return self.session.get(self.urlresolve(api_urls.PUBLIC_KEY), is_retryable=True) def _get_nonce(self): - return self.session.post(self.urlresolve(api_urls.NONCE)) + return self.session.post(self.urlresolve(api_urls.NONCE), is_retryable=True) def _get_certificate_chain(self, params): - return self.session.get(self.urlresolve(api_urls.CERTIFICATE), params=params) + return self.session.get( + self.urlresolve(api_urls.CERTIFICATE), params=params, is_retryable=True + ) def _certificate_signing(self, data, userargs, password): # convert user arguments into query str for passing to auth layer @@ -432,29 +458,38 @@ def _create_sync_session(self, data): return self.session.post(self.urlresolve(api_urls.SYNCSESSION), json=data) def _get_sync_session(self, sync_session): - return self.session.get(self.urlresolve(api_urls.SYNCSESSION, lookup=sync_session.id)) + return self.session.get( + self.urlresolve(api_urls.SYNCSESSION, lookup=sync_session.id), is_retryable=True + ) def _create_transfer_session(self, data): - return self.session.post(self.urlresolve(api_urls.TRANSFERSESSION), json=data) + return self.session.post( + self.urlresolve(api_urls.TRANSFERSESSION), json=data, is_retryable=True + ) def _get_transfer_session(self, transfer_session): return self.session.get( - self.urlresolve(api_urls.TRANSFERSESSION, lookup=transfer_session.id) + self.urlresolve(api_urls.TRANSFERSESSION, lookup=transfer_session.id), is_retryable=True ) def _update_transfer_session(self, data, transfer_session): return self.session.patch( self.urlresolve(api_urls.TRANSFERSESSION, lookup=transfer_session.id), json=data, + is_retryable=True, ) + @ignore_404 def _close_transfer_session(self, transfer_session): - return self.session.delete( - self.urlresolve(api_urls.TRANSFERSESSION, lookup=transfer_session.id) + self.session.delete( + self.urlresolve(api_urls.TRANSFERSESSION, lookup=transfer_session.id), is_retryable=True ) + @ignore_404 def _close_sync_session(self, sync_session): - return self.session.delete(self.urlresolve(api_urls.SYNCSESSION, lookup=sync_session.id)) + self.session.delete( + self.urlresolve(api_urls.SYNCSESSION, lookup=sync_session.id), is_retryable=True + ) def _push_record_chunk(self, data): # gzip the data if both client and server have gzipping capabilities @@ -467,9 +502,10 @@ def _push_record_chunk(self, data): self.urlresolve(api_urls.BUFFER), data=gzipped_data, headers={"content-type": "application/gzip"}, + is_retryable=True, ) else: - return self.session.post(self.urlresolve(api_urls.BUFFER), json=data) + return self.session.post(self.urlresolve(api_urls.BUFFER), json=data, is_retryable=True) def _pull_record_chunk(self, transfer_session): # pull records from server for given transfer session @@ -478,7 +514,7 @@ def _pull_record_chunk(self, transfer_session): "offset": transfer_session.records_transferred, "transfer_session_id": transfer_session.id, } - return self.session.get(self.urlresolve(api_urls.BUFFER), params=params) + return self.session.get(self.urlresolve(api_urls.BUFFER), params=params, is_retryable=True) class SyncClientSignals(SyncSignal): diff --git a/morango/sync/utils.py b/morango/sync/utils.py index fa4f6919..cade7039 100644 --- a/morango/sync/utils.py +++ b/morango/sync/utils.py @@ -125,18 +125,27 @@ def validate_and_create_buffer_data( # noqa: C901 buffer_list += [Buffer(**record)] with transaction.atomic(): - transfer_session.records_transferred += len(data) - + deleted_buffers, _ = Buffer.objects.filter( + transfer_session=transfer_session, + model_uuid__in=[record["model_uuid"] for record in data], + ).delete() + RecordMaxCounterBuffer.objects.filter( + transfer_session=transfer_session, + model_uuid__in=[record["model_uuid"] for record in data], + ).delete() + + update_fields = ["records_transferred"] if connection is not None: transfer_session.bytes_sent = connection.bytes_sent - if connection is not None: transfer_session.bytes_received = connection.bytes_received - - transfer_session.save() + update_fields.extend(["bytes_sent", "bytes_received"]) Buffer.objects.bulk_create(buffer_list) RecordMaxCounterBuffer.objects.bulk_create(rmcb_list) + transfer_session.records_transferred += len(buffer_list) - deleted_buffers + transfer_session.save(update_fields=update_fields) + class SyncSignal(object): """ diff --git a/morango/utils.py b/morango/utils.py index f47205a7..7e7f3a54 100644 --- a/morango/utils.py +++ b/morango/utils.py @@ -156,3 +156,16 @@ def exception_path(exc): if isinstance(exc, Exception): exc_cls = exc.__class__ return f"{exc_cls.__module__}.{exc_cls.__name__}" + + +class nullcontext: + """Replace this with contextlib.nullcontext when python3.6 support is dropped""" + + def __init__(self, value=None): + self.value = value + + def __enter__(self, *args, **kwargs): + return self.value + + def __exit__(self, exc_type, exc_val, exc_tb): + pass diff --git a/pyproject.toml b/pyproject.toml index f421e499..ee1dd8a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,8 @@ dependencies = [ "rsa<4.10", "djangorestframework>3.10", "django-ipware==4.0.2", - "requests", + "requests>=2.27.1", + "urllib3>=1.26.0", "ifcfg", ] diff --git a/tests/testapp/tests/integration/test_syncing_models.py b/tests/testapp/tests/integration/test_syncing_models.py index 904124f3..1de0985d 100644 --- a/tests/testapp/tests/integration/test_syncing_models.py +++ b/tests/testapp/tests/integration/test_syncing_models.py @@ -68,6 +68,24 @@ def test_syncable_save(self): user.save(update_dirty_bit_to=None) self.assertTrue(MyUser.objects.first()._morango_dirty_bit) + def test_syncable_save_with_update_fields_persists_dirty_bit(self): + user = MyUser.objects.first() + user.save(update_dirty_bit_to=False) + self.assertFalse(MyUser.objects.first()._morango_dirty_bit) + + user.username = "updated-name" + user.save(update_fields=["username"]) + self.assertTrue(MyUser.objects.first()._morango_dirty_bit) + + def test_syncable_save_with_update_fields_when_to_ignore_dirty_bit(self): + user = MyUser.objects.first() + user.save(update_dirty_bit_to=False) + self.assertFalse(MyUser.objects.first()._morango_dirty_bit) + + user.username = "updated-name" + user.save(update_dirty_bit_to=None, update_fields=["username"]) + self.assertFalse(MyUser.objects.first()._morango_dirty_bit) + def test_syncing_objects_manager_with_custom_default_manager(self): """Test that syncing_objects manager includes all objects even when default manager filters them out""" # Create some test objects diff --git a/tests/testapp/tests/sync/test_context.py b/tests/testapp/tests/sync/test_context.py index 00d6d904..f9941a19 100644 --- a/tests/testapp/tests/sync/test_context.py +++ b/tests/testapp/tests/sync/test_context.py @@ -70,6 +70,26 @@ def test_init__with_transfer_session(self): self.assertFalse(context.is_push) self.assertTrue(context.is_pull) + def test_init__with_transfer_session__direction_handling(self): + sync_session = mock.Mock(spec=SyncSession) + sync_filter = Filter("before_filter") + transfer_session = mock.Mock( + spec=TransferSession, + sync_session=sync_session, + push=False, + filter="after_filter", + transfer_stage=transfer_stages.TRANSFERRING, + transfer_stage_status=transfer_statuses.STARTED, + ) + transfer_session.get_filter.return_value = Filter(transfer_session.filter) + + # the transfer session's `push=False` should take precedence over the constructor arg + context = TestSessionContext( + transfer_session=transfer_session, sync_filter=sync_filter, is_push=True + ) + self.assertFalse(context.is_push) + self.assertTrue(context.is_pull) + def test_init__with_transfer_session__no_filter(self): sync_session = mock.Mock(spec=SyncSession) sync_filter = Filter("before_filter") diff --git a/tests/testapp/tests/sync/test_session.py b/tests/testapp/tests/sync/test_session.py index 439bbb7a..118ae05d 100644 --- a/tests/testapp/tests/sync/test_session.py +++ b/tests/testapp/tests/sync/test_session.py @@ -1,7 +1,11 @@ import mock from django.test import TestCase +from requests.exceptions import ConnectionError from requests.exceptions import HTTPError from requests.exceptions import RequestException +from requests.exceptions import RetryError +from urllib3.exceptions import MaxRetryError +from urllib3.util.retry import Retry from morango.sync.session import _length_of_headers from morango.sync.session import SessionWrapper @@ -11,14 +15,22 @@ class SessionWrapperTestCase(TestCase): @mock.patch("morango.sync.session.Session.request") def test_request(self, mocked_super_request): headers = {"Content-Length": 1024} - expected = mocked_super_request.return_value = mock.Mock( + mock_response = mock.Mock( headers=headers, raise_for_status=mock.Mock(), status_code=200, reason="OK" ) - wrapper = SessionWrapper() + wrapper = SessionWrapper(Retry.DEFAULT) + + def dispatch_hooks(method, url, **kwargs): + for hook in wrapper.hooks.get("response", []): + hook(mock_response) + return mock_response + + mocked_super_request.side_effect = dispatch_hooks + actual = wrapper.request("GET", "test_url", is_test=True) mocked_super_request.assert_called_once_with("GET", "test_url", is_test=True) - self.assertEqual(expected, actual) + self.assertEqual(mock_response, actual) head_length = len("HTTP/1.1 200 OK") + _length_of_headers(headers) self.assertEqual(wrapper.bytes_received, 1024 + head_length) @@ -28,14 +40,14 @@ def test_request_user_agent(self): from morango import __version__ as morango_version - wrapper = SessionWrapper() + wrapper = SessionWrapper(Retry.DEFAULT) expected_user_agent = "morango/{} python-requests/{}".format( morango_version, requests_version ) self.assertEqual(wrapper.headers["User-Agent"], expected_user_agent) with self.settings(CUSTOM_INSTANCE_INFO={"kolibri": "0.16.0"}): - wrapper = SessionWrapper() + wrapper = SessionWrapper(Retry.DEFAULT) expected_user_agent = "morango/{} kolibri/0.16.0 python-requests/{}".format( morango_version, requests_version ) @@ -53,7 +65,7 @@ def test_request__not_ok(self, mocked_super_request, mocked_logger): raise_for_status.side_effect = HTTPError(response=expected) - wrapper = SessionWrapper() + wrapper = SessionWrapper(Retry.DEFAULT) with self.assertRaises(HTTPError): wrapper.request("GET", "test_url", is_test=True) @@ -66,7 +78,7 @@ def test_request__not_ok(self, mocked_super_request, mocked_logger): def test_request__really_not_ok(self, mocked_super_request, mocked_logger): mocked_super_request.side_effect = RequestException() - wrapper = SessionWrapper() + wrapper = SessionWrapper(Retry.DEFAULT) with self.assertRaises(RequestException): wrapper.request("GET", "test_url", is_test=True) @@ -76,16 +88,45 @@ def test_request__really_not_ok(self, mocked_super_request, mocked_logger): @mock.patch("morango.sync.session.Session.prepare_request") def test_prepare_request(self, mocked_super_prepare_request): - headers = {"Content-Length": 256} - expected = mocked_super_prepare_request.return_value = mock.Mock( - headers=headers, - ) + expected = mocked_super_prepare_request.return_value = mock.Mock() request = mock.Mock(url="http://test_app/path/to/resource", method="GET", headers={}) - wrapper = SessionWrapper() + wrapper = SessionWrapper(Retry.DEFAULT) actual = wrapper.prepare_request(request) mocked_super_prepare_request.assert_called_once_with(request) self.assertEqual(expected, actual) + self.assertEqual(wrapper.bytes_sent, 0) + + @mock.patch("morango.sync.session.Session.send") + def test_send(self, mocked_super_send): + headers = {"Content-Length": 256} + mocked_super_send.return_value = mock.Mock() + + prepared_request = mock.Mock( + url="http://test_app/path/to/resource", + method="GET", + headers=headers, + ) + wrapper = SessionWrapper(Retry.DEFAULT) + wrapper.send(prepared_request) + + mocked_super_send.assert_called_once_with(prepared_request) head_length = len("GET /path/to/resource HTTP/1.1") + _length_of_headers(headers) self.assertEqual(wrapper.bytes_sent, 256 + head_length) + + def test_should_retry__raises_retry_error_on_max_retries(self): + retries = mock.Mock() + retries.allowed_methods = None + retries.increment.side_effect = MaxRetryError(None, "http://test_app/path/to/resource") + + prepared_request = mock.Mock( + url="http://test_app/path/to/resource", + method="GET", + ) + + wrapper = SessionWrapper(Retry.DEFAULT) + with self.assertRaises(RetryError) as cm: + wrapper._should_retry(retries, prepared_request, ConnectionError()) + + self.assertIs(cm.exception.request, prepared_request) diff --git a/tests/testapp/tests/sync/test_syncsession.py b/tests/testapp/tests/sync/test_syncsession.py index abd9fa68..e715329b 100644 --- a/tests/testapp/tests/sync/test_syncsession.py +++ b/tests/testapp/tests/sync/test_syncsession.py @@ -4,7 +4,9 @@ import mock from django.test.testcases import LiveServerTestCase from django.test.utils import override_settings +from requests.exceptions import ChunkedEncodingError from requests.exceptions import HTTPError +from requests.sessions import Session from morango.api.serializers import CertificateSerializer from morango.constants import transfer_stages @@ -19,6 +21,7 @@ from morango.models.certificates import Key from morango.models.certificates import ScopeDefinition from morango.models.core import SyncSession +from morango.models.core import TransferSession from morango.models.fields.crypto import SharedKey from morango.sync.context import LocalSessionContext from morango.sync.context import NetworkSessionContext @@ -127,6 +130,27 @@ def test_get_remote_certs(self): remote_certs = self.network_connection.get_remote_certificates(self.root_cert.id) self.assertSetEqual(set(certs), set(remote_certs)) + def test_get_remote_certs__retries_chunked_encoding_error(self): + certs = self.subset_cert.get_ancestors(include_self=True) + original_send = Session.send + attempts = {"chunked_encoding_errors": 0} + + def flaky_request(session, request, **kwargs): + if request.method == "GET" and attempts["chunked_encoding_errors"] == 0: + attempts["chunked_encoding_errors"] += 1 + raise ChunkedEncodingError("Connection broken") + return original_send(session, request, **kwargs) + + with mock.patch( + "morango.sync.session.Session.send", + autospec=True, + side_effect=flaky_request, + ): + remote_certs = self.network_connection.get_remote_certificates(self.root_cert.id) + + self.assertEqual(1, attempts["chunked_encoding_errors"]) + self.assertSetEqual(set(certs), set(remote_certs)) + @mock.patch.object(SessionWrapper, "request") def test_csr(self, mock_request): # mock a "signed" cert being returned by server @@ -225,6 +249,66 @@ def create(**data): self.network_connection.close_sync_session(client.sync_session) self.assertEqual(SyncSession.objects.filter(active=True).count(), 0) + def test_close_transfer_session_ignores_404(self): + mock_response = mock.Mock() + mock_response.status_code = 404 + transfer_session = mock.Mock(spec=TransferSession) + transfer_session.id = uuid.uuid4().hex + + with mock.patch.object( + self.network_connection.session, + "delete", + side_effect=HTTPError(response=mock_response), + ): + # should not raise even though the server returned 404 + self.network_connection._close_transfer_session(transfer_session) + + def test_close_sync_session_ignores_404(self): + mock_response = mock.Mock() + mock_response.status_code = 404 + sync_session = mock.Mock(spec=SyncSession) + sync_session.id = uuid.uuid4().hex + + with mock.patch.object( + self.network_connection.session, + "delete", + side_effect=HTTPError(response=mock_response), + ): + # should not raise even though the server returned 404 + self.network_connection._close_sync_session(sync_session) + + def test_close_sync_session_raises_500(self): + mock_response = mock.Mock() + mock_response.status_code = 500 + sync_session = mock.Mock(spec=SyncSession) + sync_session.id = uuid.uuid4().hex + + e = HTTPError(response=mock_response) + + with mock.patch.object( + self.network_connection.session, + "delete", + side_effect=e, + ): + with self.assertRaises(type(e)) as raised: + self.network_connection._close_sync_session(sync_session) + self.assertEqual(e, raised.exception) + + def test_close_sync_session_raises_no_response(self): + sync_session = mock.Mock(spec=SyncSession) + sync_session.id = uuid.uuid4().hex + + e = HTTPError() + + with mock.patch.object( + self.network_connection.session, + "delete", + side_effect=e, + ): + with self.assertRaises(type(e)) as raised: + self.network_connection._close_sync_session(sync_session) + self.assertEqual(e, raised.exception) + @mock.patch.object(SyncSession.objects, "create") def test_resume_sync_session(self, mock_create): def create(**data): diff --git a/tests/testapp/tests/test_api.py b/tests/testapp/tests/test_api.py index 3bd64d2b..594b1be0 100644 --- a/tests/testapp/tests/test_api.py +++ b/tests/testapp/tests/test_api.py @@ -848,7 +848,7 @@ def build_buffer_item(self, **kwargs): return buffermodel - def make_buffer_post_request(self, buffers, expected_status=201, gzip=False): + def make_buffer_post_request(self, buffers, expected_status=201, gzip=False, pre_count=0): serialized_recs = BufferSerializer(buffers, many=True) # extract that data that is to be posted @@ -862,9 +862,13 @@ def make_buffer_post_request(self, buffers, expected_status=201, gzip=False): headers["content_type"] = "application/gzip" headers["format"] = None - # delete the records from the DB so we don't conflict when we POST - Buffer.objects.all().delete() - RecordMaxCounterBuffer.objects.all().delete() + # delete the records to match pre_count if zero + if pre_count == 0: + Buffer.objects.all().delete() + RecordMaxCounterBuffer.objects.all().delete() + else: + self.assertEqual(Buffer.objects.count(), pre_count) + self.assertEqual(RecordMaxCounterBuffer.objects.count(), pre_count * 3) response = self.client.post(reverse("buffers-list"), data, **headers) self.assertEqual(response.status_code, expected_status) @@ -913,6 +917,22 @@ def test_push_fails_for_pull_transfersession(self): rec_3 = self.build_buffer_item(transfer_session=rec_1.transfer_session) self.make_buffer_post_request([rec_1, rec_2, rec_3], expected_status=403) + def test_push_repeat_chunk(self): + rec_1 = self.build_buffer_item(push=True, filter=self.default_push_filter) + transfer_session = rec_1.transfer_session + rec_2 = self.build_buffer_item(transfer_session=transfer_session) + # need to create a third buffer so it doesn't mark the transfer complete after pushing 2 + self.build_buffer_item(transfer_session=transfer_session) + self.assertEqual(transfer_session.records_transferred, 0) + + self.make_buffer_post_request([rec_1, rec_2], expected_status=201) + transfer_session.refresh_from_db() + self.assertEqual(transfer_session.records_transferred, 2) + + self.make_buffer_post_request([rec_1, rec_2], expected_status=201, pre_count=2) + transfer_session.refresh_from_db() + self.assertEqual(transfer_session.records_transferred, 2) + def create_records_for_pulling(self, count=3, **kwargs): assert count >= 1 @@ -965,9 +985,7 @@ def make_buffer_get_request(self, expected_status=200, expected_count=None, **ge # delete "local" buffer records to avoid uniqueness constraint failures in validation Buffer.objects.filter(transfer_session_id=t_id, model_uuid__in=model_uuids).delete() - # run the validation logic to ensure no errors were returned - errors = validate_and_create_buffer_data(data, TransferSession.objects.get(id=t_id)) - self.assertFalse(errors) + validate_and_create_buffer_data(data, TransferSession.objects.get(id=t_id)) # check that the correct number of buffer items were created self.assertEqual( @@ -997,12 +1015,31 @@ def test_buffer_serializer_includes_self_ref_order(self): buffer = Buffer.objects.filter(transfer_session_id=transfer_session_id).first() self.assertEqual(BufferSerializer(instance=buffer).data["_self_ref_order"], 4) + def test_buffer_serializer_batches_rmcb_queries_for_many(self): + transfer_session_id = self.create_records_for_pulling(count=10) + buffers = Buffer.objects.filter(transfer_session_id=transfer_session_id).order_by("pk") + + with CaptureQueriesContext(connection) as ctx: + BufferSerializer(buffers, many=True).data + + rmcb_queries = [ + q for q in ctx.captured_queries if "recordmaxcounterbuffer" in q["sql"].lower() + ] + self.assertEqual(len(rmcb_queries), 1) + def test_pull_valid_buffer_list(self): transfer_session_id = self.create_records_for_pulling() self.make_buffer_get_request(transfer_session_id=transfer_session_id) + def test_pull_repeat(self): + + transfer_session_id = self.create_records_for_pulling(count=3) + + self.make_buffer_get_request(transfer_session_id=transfer_session_id, expected_count=3) + self.make_buffer_get_request(transfer_session_id=transfer_session_id, expected_count=3) + def test_pull_fails_when_transfer_session_id_not_specified(self): self.create_records_for_pulling() diff --git a/tests/testapp/tests/test_deferrable_foreign_keys.py b/tests/testapp/tests/test_deferrable_foreign_keys.py new file mode 100644 index 00000000..f90c2fc4 --- /dev/null +++ b/tests/testapp/tests/test_deferrable_foreign_keys.py @@ -0,0 +1,89 @@ +import pytest +from django.apps import apps as global_apps +from django.conf import settings +from django.db import connection +from django.test import TransactionTestCase +from facility_profile.models import Facility + +from morango.deferrable_foreign_keys import _get_table_sql +from morango.deferrable_foreign_keys import MakeForeignKeysDeferrable + +BASE_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "rest_framework", +] + + +class MakeDeferrableForeignKeysTestCase(TransactionTestCase): + def _rewrite_model_fk_immediate(self): + table = Facility._meta.db_table + sql = _get_table_sql(connection, table) + immediate = sql.replace(" DEFERRABLE INITIALLY DEFERRED", "") + assert "DEFERRABLE" not in immediate + with connection.cursor() as cursor: + cursor.execute( + "SELECT sql FROM sqlite_master WHERE type='index' AND tbl_name=%s AND sql IS NOT NULL", + [table], + ) + indexes = [r[0] for r in cursor.fetchall()] + cursor.execute("PRAGMA foreign_keys = OFF") + cursor.execute("DROP TABLE {}".format(connection.ops.quote_name(table))) + cursor.execute(immediate) + for idx in indexes: + cursor.execute(idx) + cursor.execute("PRAGMA foreign_keys = ON") + + @pytest.mark.skipif(getattr(settings, "MORANGO_TEST_POSTGRESQL", False), reason="SQLite only") + def test_remake_makes_deferrable_and_preserves_data(self): + table = Facility._meta.db_table + # fresh schema should be deferrable + self.assertIn("DEFERRABLE", _get_table_sql(connection, table)) + + # downgrade schema to immediate FK (table is rebuilt empty) + self._rewrite_model_fk_immediate() + self.assertNotIn("DEFERRABLE", _get_table_sql(connection, table)) + + # create some data on the immediate-FK schema + f = Facility.objects.create(name="testfac") + f_id = f.id + self.assertTrue(Facility.objects.filter(id=f_id).exists()) + + # run the helper with the real app registry + a schema editor + op = MakeForeignKeysDeferrable(include_app_labels=["facility_profile"]) + with connection.schema_editor(atomic=False) as schema_editor: + op(global_apps, schema_editor) + + # now deferrable again, and data preserved + self.assertIn("DEFERRABLE", _get_table_sql(connection, table)) + self.assertTrue(Facility.objects.filter(id=f_id).exists()) + + def test_iter_apps__all(self): + op = MakeForeignKeysDeferrable() + self.assertEqual( + [app.name for app in op._iter_apps(global_apps)], + [*BASE_APPS, "morango", "facility_profile"], + ) + + def test_iter_apps__exclude(self): + op = MakeForeignKeysDeferrable(exclude_app_labels=["facility_profile"]) + self.assertEqual( + [app.name for app in op._iter_apps(global_apps)], + [ + *BASE_APPS, + "morango", + ], + ) + + def test_iter_apps__include(self): + op = MakeForeignKeysDeferrable(include_app_labels=["morango"]) + self.assertEqual( + [app.name for app in op._iter_apps(global_apps)], + [ + "morango", + ], + ) diff --git a/uv.lock b/uv.lock index deeac6a7..2d567899 100644 --- a/uv.lock +++ b/uv.lock @@ -63,7 +63,7 @@ version = "4.12.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "exceptiongroup", marker = "python_full_version >= '3.9' and python_full_version < '3.11'" }, - { name = "idna", marker = "python_full_version >= '3.9'" }, + { name = "idna", version = "3.15", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9' and python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/96/f0/5eb65b2bb0d09ac6776f2eb54adee6abe8228ea05b20a5ad0e4945de8aac/anyio-4.12.1.tar.gz", hash = "sha256:41cfcc3a4c85d3f05c932da7c26d0201ac36f72abd4435ba90d0464a3ffed703", size = 228685, upload-time = "2026-01-06T11:45:21.246Z" } @@ -226,10 +226,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/06/f3/39cf3367b8107baa44f861dc802cbf16263c945b62d8265d36034fc07bea/cachetools-7.0.5-py3-none-any.whl", hash = "sha256:46bc8ebefbe485407621d0a4264b23c080cedd913921bad7ac3ed2f26c183114", size = 13918, upload-time = "2026-03-09T20:51:27.33Z" }, ] +[[package]] +name = "certifi" +version = "2025.4.26" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.6.8' and python_full_version < '3.7'", + "python_full_version < '3.6.8'", +] +sdist = { url = "https://files.pythonhosted.org/packages/e8/9e/c05b3920a3b7d20d3d3310465f50348e5b3694f4f88c6daf736eef3024c4/certifi-2025.4.26.tar.gz", hash = "sha256:0a816057ea3cdefcef70270d2c515e4506bbc954f417fa5ade2021213bb8f0c6", size = 160705, upload-time = "2025-04-26T02:12:29.51Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/7e/3db2bd1b1f9e95f7cddca6d6e75e2f2bd9f51b1246e546d88addca0106bd/certifi-2025.4.26-py3-none-any.whl", hash = "sha256:30350364dfe371162649852c63336a15c70c6510c2ad5015b21c2345311805f3", size = 159618, upload-time = "2025-04-26T02:12:27.662Z" }, +] + [[package]] name = "certifi" version = "2026.2.25" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", + "python_full_version == '3.8.*'", + "python_full_version == '3.7.*'", +] sdist = { url = "https://files.pythonhosted.org/packages/af/2d/7bf41579a8986e348fa033a31cdd0e4121114f6bce2457e8876010b092dd/certifi-2026.2.25.tar.gz", hash = "sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7", size = 155029, upload-time = "2026-02-25T02:54:17.342Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/9a/3c/c17fb3ca2d9c3acff52e30b309f538586f9f5b9c9cf454f3845fc9af4881/certifi-2026.2.25-py3-none-any.whl", hash = "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", size = 153684, upload-time = "2026-02-25T02:54:15.766Z" }, @@ -497,10 +518,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/38/6f/f5fbc992a329ee4e0f288c1fe0e2ad9485ed064cac731ed2fe47dcc38cbf/chardet-5.2.0-py3-none-any.whl", hash = "sha256:e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970", size = 199385, upload-time = "2023-08-01T19:23:00.661Z" }, ] +[[package]] +name = "charset-normalizer" +version = "2.0.12" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.6.8' and python_full_version < '3.7'", + "python_full_version < '3.6.8'", +] +sdist = { url = "https://files.pythonhosted.org/packages/56/31/7bcaf657fafb3c6db8c787a865434290b726653c912085fbd371e9b92e1c/charset-normalizer-2.0.12.tar.gz", hash = "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597", size = 79105, upload-time = "2022-02-12T14:33:13.788Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/b3/24afc8868eba069a7f03650ac750a778862dc34941a4bebeb58706715726/charset_normalizer-2.0.12-py3-none-any.whl", hash = "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df", size = 39623, upload-time = "2022-02-12T14:33:12.294Z" }, +] + [[package]] name = "charset-normalizer" version = "3.4.6" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", + "python_full_version == '3.8.*'", + "python_full_version == '3.7.*'", +] sdist = { url = "https://files.pythonhosted.org/packages/7b/60/e3bec1881450851b087e301bedc3daa9377a4d45f1c26aa90b0b235e38aa/charset_normalizer-3.4.6.tar.gz", hash = "sha256:1ae6b62897110aa7c79ea2f5dd38d1abca6db663687c0b1ad9aed6f6bae3d9d6", size = 143363, upload-time = "2026-03-15T18:53:25.478Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/e6/8c/2c56124c6dc53a774d435f985b5973bc592f42d437be58c0c92d65ae7296/charset_normalizer-3.4.6-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2e1d8ca8611099001949d1cdfaefc510cf0f212484fe7c565f735b68c78c3c95", size = 298751, upload-time = "2026-03-15T18:50:00.003Z" }, @@ -1108,10 +1150,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] +[[package]] +name = "idna" +version = "3.10" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.7.*'", + "python_full_version >= '3.6.8' and python_full_version < '3.7'", + "python_full_version < '3.6.8'", +] +sdist = { url = "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", size = 190490, upload-time = "2024-09-15T18:07:39.745Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" }, +] + [[package]] name = "idna" version = "3.15" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", + "python_full_version == '3.8.*'", +] sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", size = 199245, upload-time = "2026-05-12T22:45:57.011Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" }, @@ -1585,10 +1648,15 @@ dependencies = [ { name = "django-mptt", version = "0.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, { name = "djangorestframework" }, { name = "ifcfg" }, - { name = "requests", version = "2.15.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8'" }, + { name = "requests", version = "2.27.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.7'" }, + { name = "requests", version = "2.31.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.7.*'" }, { name = "requests", version = "2.32.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.8.*'" }, { name = "requests", version = "2.32.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, { name = "rsa" }, + { name = "urllib3", version = "1.26.20", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.7'" }, + { name = "urllib3", version = "2.0.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.7.*'" }, + { name = "urllib3", version = "2.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.8.*'" }, + { name = "urllib3", version = "2.6.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, ] [package.optional-dependencies] @@ -1673,8 +1741,9 @@ requires-dist = [ { name = "m2crypto", marker = "extra == 'accelerated'", specifier = "==0.41.0" }, { name = "m2crypto", marker = "extra == 'postgres'", specifier = "==0.41.0" }, { name = "psycopg2-binary", marker = "extra == 'postgres'", specifier = ">=2.9.8,<=2.9.9" }, - { name = "requests" }, + { name = "requests", specifier = ">=2.27.1" }, { name = "rsa", specifier = "<4.10" }, + { name = "urllib3", specifier = ">=1.26.0" }, ] provides-extras = ["postgres", "accelerated"] @@ -2365,16 +2434,39 @@ wheels = [ [[package]] name = "requests" -version = "2.15.1" +version = "2.27.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.7.*'", "python_full_version >= '3.6.8' and python_full_version < '3.7'", "python_full_version < '3.6.8'", ] -sdist = { url = "https://files.pythonhosted.org/packages/6d/ed/3adebdc29ca33f11bca00c38c72125cd4a51091e13685375ba4426fb59dc/requests-2.15.1.tar.gz", hash = "sha256:e5659b9315a0610505e050bb7190bf6fa2ccee1ac295f2b760ef9d8a03ebbb2e", size = 548172, upload-time = "2017-05-27T02:14:22.414Z" } +dependencies = [ + { name = "certifi", version = "2025.4.26", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.7'" }, + { name = "charset-normalizer", version = "2.0.12", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.7'" }, + { name = "idna", version = "3.10", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.7'" }, + { name = "urllib3", version = "1.26.20", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.7'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/60/f3/26ff3767f099b73e0efa138a9998da67890793bfa475d8278f84a30fec77/requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61", size = 106758, upload-time = "2022-01-05T15:40:51.698Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2d/61/08076519c80041bc0ffa1a8af0cbd3bf3e2b62af10435d269a9d0f40564d/requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d", size = 63133, upload-time = "2022-01-05T15:40:49.334Z" }, +] + +[[package]] +name = "requests" +version = "2.31.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.7.*'", +] +dependencies = [ + { name = "certifi", version = "2026.2.25", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.7.*'" }, + { name = "charset-normalizer", version = "3.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.7.*'" }, + { name = "idna", version = "3.10", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.7.*'" }, + { name = "urllib3", version = "2.0.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.7.*'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/be/10918a2eac4ae9f02f6cfe6414b7a155ccd8f7f9d4380d62fd5b955065c3/requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1", size = 110794, upload-time = "2023-05-22T15:12:44.175Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/a5/e04c4607dc96e3e6b22dfa13ba8776c64bb65cb97ab90f05a3ee14096a0a/requests-2.15.1-py2.py3-none-any.whl", hash = "sha256:ff753b2196cd18b1bbeddc9dcd5c864056599f7a7d9a4fb5677e723efa2b7fb9", size = 558730, upload-time = "2017-05-27T02:14:19.048Z" }, + { url = "https://files.pythonhosted.org/packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f", size = 62574, upload-time = "2023-05-22T15:12:42.313Z" }, ] [[package]] @@ -2385,9 +2477,9 @@ resolution-markers = [ "python_full_version == '3.8.*'", ] dependencies = [ - { name = "certifi", marker = "python_full_version == '3.8.*'" }, - { name = "charset-normalizer", marker = "python_full_version == '3.8.*'" }, - { name = "idna", marker = "python_full_version == '3.8.*'" }, + { name = "certifi", version = "2026.2.25", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.8.*'" }, + { name = "charset-normalizer", version = "3.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.8.*'" }, + { name = "idna", version = "3.15", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.8.*'" }, { name = "urllib3", version = "2.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.8.*'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e1/0a/929373653770d8a0d7ea76c37de6e41f11eb07559b103b1c02cafb3f7cf8/requests-2.32.4.tar.gz", hash = "sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422", size = 135258, upload-time = "2025-06-09T16:43:07.34Z" } @@ -2406,9 +2498,9 @@ resolution-markers = [ "python_full_version == '3.9.*'", ] dependencies = [ - { name = "certifi", marker = "python_full_version >= '3.9'" }, - { name = "charset-normalizer", marker = "python_full_version >= '3.9'" }, - { name = "idna", marker = "python_full_version >= '3.9'" }, + { name = "certifi", version = "2026.2.25", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, + { name = "charset-normalizer", version = "3.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, + { name = "idna", version = "3.15", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, { name = "urllib3", version = "2.6.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload-time = "2025-08-18T20:46:02.573Z" } @@ -2533,7 +2625,8 @@ dependencies = [ { name = "pygments", version = "2.14.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.7'" }, { name = "pygments", version = "2.17.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.7.*'" }, { name = "pygments", version = "2.19.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.8' and python_full_version < '3.10'" }, - { name = "requests", version = "2.15.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8'" }, + { name = "requests", version = "2.27.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.7'" }, + { name = "requests", version = "2.31.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.7.*'" }, { name = "requests", version = "2.32.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.8.*'" }, { name = "requests", version = "2.32.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, { name = "setuptools", version = "59.6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.7'" }, @@ -3505,6 +3598,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl", hash = "sha256:06a47e5700f3081aab02b2e513160914ff0694bce9947d6b76ebd6bf57cfc5d1", size = 348521, upload-time = "2025-12-13T17:45:33.889Z" }, ] +[[package]] +name = "urllib3" +version = "1.26.20" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.6.8' and python_full_version < '3.7'", + "python_full_version < '3.6.8'", +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/e8/6ff5e6bc22095cfc59b6ea711b687e2b7ed4bdb373f7eeec370a97d7392f/urllib3-1.26.20.tar.gz", hash = "sha256:40c2dc0c681e47eb8f90e7e27bf6ff7df2e677421fd46756da1161c39ca70d32", size = 307380, upload-time = "2024-08-29T15:43:11.37Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/cf/8435d5a7159e2a9c83a95896ed596f68cf798005fe107cc655b5c5c14704/urllib3-1.26.20-py2.py3-none-any.whl", hash = "sha256:0ed14ccfbf1c30a9072c7ca157e4319b70d65f623e91e7b32fadb2853431016e", size = 144225, upload-time = "2024-08-29T15:43:08.921Z" }, +] + +[[package]] +name = "urllib3" +version = "2.0.7" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.7.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/af/47/b215df9f71b4fdba1025fc05a77db2ad243fa0926755a52c5e71659f4e3c/urllib3-2.0.7.tar.gz", hash = "sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84", size = 282546, upload-time = "2023-10-17T17:46:50.542Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/b2/b157855192a68541a91ba7b2bbcb91f1b4faa51f8bae38d8005c034be524/urllib3-2.0.7-py3-none-any.whl", hash = "sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e", size = 124213, upload-time = "2023-10-17T17:46:48.538Z" }, +] + [[package]] name = "urllib3" version = "2.2.3"