Use DEFAULT_CHUNK_SIZE global (aio-libs#12356) (aio-libs#12365)

(cherry picked from commit bec74bb)

Author: Dreamsorcerer

aiohttp/_websocket/writer.py

@@ -9,6 +9,7 @@
 from ..base_protocol import BaseProtocol
 from ..client_exceptions import ClientConnectionResetError
 from ..compression_utils import ZLibBackend, ZLibCompressor
+from ..helpers import DEFAULT_CHUNK_SIZE
 from .helpers import (
     MASK_LEN,
     MSG_SIZE,
@@ -21,8 +22,6 @@
 )
 from .models import WS_DEFLATE_TRAILING, WSMsgType
 
-DEFAULT_LIMIT: Final[int] = 2**18
-
 # WebSocket opcode boundary: opcodes 0-7 are data frames, 8-15 are control frames
 # Control frames (ping, pong, close) are never compressed
 WS_CONTROL_FRAME_OPCODE: Final[int] = 8
@@ -52,7 +51,7 @@ def __init__(
         transport: asyncio.Transport,
         *,
         use_mask: bool = False,
-        limit: int = DEFAULT_LIMIT,
+        limit: int = DEFAULT_CHUNK_SIZE,
         random: random.Random = random.Random(),
         compress: int = 0,
         notakeover: bool = False,

aiohttp/client.py

@@ -91,6 +91,7 @@
 from .helpers import (
     _SENTINEL,
     DEBUG,
+    DEFAULT_CHUNK_SIZE,
     EMPTY_BODY_METHODS,
     BasicAuth,
     TimeoutHandle,
@@ -333,7 +334,7 @@ def __init__(
         trust_env: bool = False,
         requote_redirect_url: bool = True,
         trace_configs: list[TraceConfig] | None = None,
-        read_bufsize: int = 2**18,
+        read_bufsize: int = DEFAULT_CHUNK_SIZE,
         max_line_size: int = 8190,
         max_field_size: int = 8190,
         max_headers: int = 128,
@@ -1294,7 +1295,7 @@ async def _ws_connect(
 
             transport = conn.transport
             assert transport is not None
-            reader = WebSocketDataQueue(conn_proto, 2**18, loop=self._loop)
+            reader = WebSocketDataQueue(conn_proto, DEFAULT_CHUNK_SIZE, loop=self._loop)
             writer = WebSocketWriter(
                 conn_proto,
                 transport,

aiohttp/client_proto.py

@@ -12,6 +12,7 @@
 )
 from .helpers import (
     _EXC_SENTINEL,
+    DEFAULT_CHUNK_SIZE,
     EMPTY_BODY_STATUS_CODES,
     BaseTimerContext,
     set_exception,
@@ -230,7 +231,7 @@ def set_response_params(
         read_until_eof: bool = False,
         auto_decompress: bool = True,
         read_timeout: float | None = None,
-        read_bufsize: int = 2**18,
+        read_bufsize: int = DEFAULT_CHUNK_SIZE,
         timeout_ceil_threshold: float = 5,
         max_line_size: int = 8190,
         max_field_size: int = 8190,

aiohttp/compression_utils.py

@@ -34,9 +34,6 @@
 
 
 MAX_SYNC_CHUNK_SIZE = 4096
-# Matches the max size we receive from sockets:
-# https://github.com/python/cpython/blob/1857a40807daeae3a1bf5efb682de9c9ae6df845/Lib/asyncio/selector_events.py#L766
-DEFAULT_MAX_DECOMPRESS_SIZE = 256 * 1024
 
 # Unlimited decompression constants - different libraries use different conventions
 ZLIB_MAX_LENGTH_UNLIMITED = 0  # zlib uses 0 to mean unlimited

aiohttp/helpers.py

@@ -58,6 +58,10 @@
 
 PY_311 = sys.version_info >= (3, 11)
 
+# This is the default size/limit for several operations.
+# Matches the max size we receive from sockets:
+# https://github.com/python/cpython/blob/1857a40807daeae3a1bf5efb682de9c9ae6df845/Lib/asyncio/selector_events.py#L766
+DEFAULT_CHUNK_SIZE = 2**18  # 256 KiB
 
 _T = TypeVar("_T")
 _S = TypeVar("_S")

aiohttp/http_parser.py

@@ -22,7 +22,6 @@
 from . import hdrs
 from .base_protocol import BaseProtocol
 from .compression_utils import (
-    DEFAULT_MAX_DECOMPRESS_SIZE,
     HAS_BROTLI,
     HAS_ZSTD,
     BrotliDecompressor,
@@ -32,6 +31,7 @@
 from .helpers import (
     _EXC_SENTINEL,
     DEBUG,
+    DEFAULT_CHUNK_SIZE,
     EMPTY_BODY_METHODS,
     EMPTY_BODY_STATUS_CODES,
     NO_EXTENSIONS,
@@ -817,7 +817,7 @@ def __init__(
         max_line_size: int = 8190,
         max_field_size: int = 8190,
         max_trailers: int = 128,
-        limit: int = DEFAULT_MAX_DECOMPRESS_SIZE,
+        limit: int = DEFAULT_CHUNK_SIZE,
     ) -> None:
         self._length = 0
         self._paused = False
@@ -1074,7 +1074,7 @@ def __init__(
         self,
         out: StreamReader,
         encoding: str | None,
-        max_decompress_size: int = DEFAULT_MAX_DECOMPRESS_SIZE,
+        max_decompress_size: int = DEFAULT_CHUNK_SIZE,
     ) -> None:
         self.out = out
         self.size = 0

aiohttp/multipart.py

@@ -14,19 +14,15 @@
 from multidict import CIMultiDict, CIMultiDictProxy
 
 from .abc import AbstractStreamWriter
-from .compression_utils import (
-    DEFAULT_MAX_DECOMPRESS_SIZE,
-    ZLibCompressor,
-    ZLibDecompressor,
-)
+from .compression_utils import ZLibCompressor, ZLibDecompressor
 from .hdrs import (
     CONTENT_DISPOSITION,
     CONTENT_ENCODING,
     CONTENT_LENGTH,
     CONTENT_TRANSFER_ENCODING,
     CONTENT_TYPE,
 )
-from .helpers import CHAR, TOKEN, parse_mimetype, reify
+from .helpers import CHAR, DEFAULT_CHUNK_SIZE, TOKEN, parse_mimetype, reify
 from .http import HeadersParser
 from .http_exceptions import BadHttpMessage
 from .log import internal_logger
@@ -267,7 +263,7 @@ def __init__(
         *,
         subtype: str = "mixed",
         default_charset: str | None = None,
-        max_decompress_size: int = DEFAULT_MAX_DECOMPRESS_SIZE,
+        max_decompress_size: int = DEFAULT_CHUNK_SIZE,
         client_max_size: int = sys.maxsize,
         max_size_error_cls: type[Exception] = ValueError,
     ) -> None:
@@ -640,7 +636,7 @@ async def as_bytes(self, encoding: str = "utf-8", errors: str = "strict") -> byt
 
     async def write(self, writer: AbstractStreamWriter) -> None:
         field = self._value
-        while chunk := await field.read_chunk(size=2**18):
+        while chunk := await field.read_chunk(size=DEFAULT_CHUNK_SIZE):
             async for d in field.decode_iter(chunk):
                 await writer.write(d)
 

aiohttp/payload.py

@@ -17,6 +17,7 @@
 from .abc import AbstractStreamWriter
 from .helpers import (
     _SENTINEL,
+    DEFAULT_CHUNK_SIZE,
     content_disposition_header,
     guess_filename,
     parse_mimetype,
@@ -43,7 +44,6 @@
 )
 
 TOO_LARGE_BYTES_BODY: Final[int] = 2**20  # 1 MB
-READ_SIZE: Final[int] = 2**18  # 256 KiB
 _CLOSE_FUTURES: set[asyncio.Future[None]] = set()
 
 
@@ -491,7 +491,7 @@ def _read_and_available_len(
 
         Args:
             remaining_content_len: Optional limit on how many bytes to read in this operation.
-                If None, READ_SIZE will be used as the default chunk size.
+                If None, DEFAULT_CHUNK_SIZE will be used as the default chunk size.
 
         Returns:
             A tuple containing:
@@ -506,7 +506,11 @@ def _read_and_available_len(
         self._set_or_restore_start_position()
         size = self.size  # Call size only once since it does I/O
         return size, self._value.read(
-            min(READ_SIZE, size or READ_SIZE, remaining_content_len or READ_SIZE)
+            min(
+                DEFAULT_CHUNK_SIZE,
+                size or DEFAULT_CHUNK_SIZE,
+                remaining_content_len or DEFAULT_CHUNK_SIZE,
+            )
         )
 
     def _read(self, remaining_content_len: int | None) -> bytes:
@@ -515,7 +519,7 @@ def _read(self, remaining_content_len: int | None) -> bytes:
 
         Args:
             remaining_content_len: Optional maximum number of bytes to read.
-                If None, READ_SIZE will be used as the default chunk size.
+                If None, DEFAULT_CHUNK_SIZE will be used as the default chunk size.
 
         Returns:
             A chunk of bytes read from the file object, respecting the
@@ -525,7 +529,7 @@ def _read(self, remaining_content_len: int | None) -> bytes:
         the initial _read_and_available_len call has been made.
 
         """
-        return self._value.read(remaining_content_len or READ_SIZE)  # type: ignore[no-any-return]
+        return self._value.read(remaining_content_len or DEFAULT_CHUNK_SIZE)  # type: ignore[no-any-return]
 
     @property
     def size(self) -> int | None:
@@ -628,9 +632,9 @@ async def write_with_length(
                 None,
                 self._read,
                 (
-                    min(READ_SIZE, remaining_content_len)
+                    min(DEFAULT_CHUNK_SIZE, remaining_content_len)
                     if remaining_content_len is not None
-                    else READ_SIZE
+                    else DEFAULT_CHUNK_SIZE
                 ),
             )
 
@@ -756,7 +760,7 @@ def _read_and_available_len(
 
         Args:
             remaining_content_len: Optional limit on how many bytes to read in this operation.
-                If None, READ_SIZE will be used as the default chunk size.
+                If None, DEFAULT_CHUNK_SIZE will be used as the default chunk size.
 
         Returns:
             A tuple containing:
@@ -775,7 +779,11 @@ def _read_and_available_len(
         self._set_or_restore_start_position()
         size = self.size
         chunk = self._value.read(
-            min(READ_SIZE, size or READ_SIZE, remaining_content_len or READ_SIZE)
+            min(
+                DEFAULT_CHUNK_SIZE,
+                size or DEFAULT_CHUNK_SIZE,
+                remaining_content_len or DEFAULT_CHUNK_SIZE,
+            )
         )
         return size, chunk.encode(self._encoding) if self._encoding else chunk.encode()
 
@@ -785,7 +793,7 @@ def _read(self, remaining_content_len: int | None) -> bytes:
 
         Args:
             remaining_content_len: Optional maximum number of bytes to read.
-                If None, READ_SIZE will be used as the default chunk size.
+                If None, DEFAULT_CHUNK_SIZE will be used as the default chunk size.
 
         Returns:
             A chunk of bytes read from the file object and encoded using the payload's
@@ -797,7 +805,7 @@ def _read(self, remaining_content_len: int | None) -> bytes:
         the specified encoding (or UTF-8 if none was provided).
 
         """
-        chunk = self._value.read(remaining_content_len or READ_SIZE)
+        chunk = self._value.read(remaining_content_len or DEFAULT_CHUNK_SIZE)
         return chunk.encode(self._encoding) if self._encoding else chunk.encode()
 
     def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str:
@@ -881,7 +889,7 @@ async def write_with_length(
         self._set_or_restore_start_position()
         loop_count = 0
         remaining_bytes = content_length
-        while chunk := self._value.read(READ_SIZE):
+        while chunk := self._value.read(DEFAULT_CHUNK_SIZE):
             if loop_count > 0:
                 # Avoid blocking the event loop
                 # if they pass a large BytesIO object

aiohttp/streams.py

@@ -7,6 +7,7 @@
 from .base_protocol import BaseProtocol
 from .helpers import (
     _EXC_SENTINEL,
+    DEFAULT_CHUNK_SIZE,
     BaseTimerContext,
     TimerNoop,
     set_exception,
@@ -167,7 +168,7 @@ def __repr__(self) -> str:
             info.append("%d bytes" % self._size)
         if self._eof:
             info.append("eof")
-        if self._low_water != 2**18:  # default limit
+        if self._low_water != DEFAULT_CHUNK_SIZE:
             info.append("low=%d high=%d" % (self._low_water, self._high_water))
         if self._waiter:
             info.append("w=%r" % self._waiter)

aiohttp/web_fileresponse.py

@@ -26,7 +26,7 @@
 
 from . import hdrs
 from .abc import AbstractStreamWriter
-from .helpers import ETAG_ANY, ETag, must_be_empty_body
+from .helpers import DEFAULT_CHUNK_SIZE, ETAG_ANY, ETag, must_be_empty_body
 from .typedefs import LooseHeaders, PathLike
 from .web_exceptions import (
     HTTPForbidden,
@@ -95,7 +95,7 @@ class FileResponse(StreamResponse):
     def __init__(
         self,
         path: PathLike,
-        chunk_size: int = 256 * 1024,
+        chunk_size: int = DEFAULT_CHUNK_SIZE,
         status: int = 200,
         reason: str | None = None,
         headers: LooseHeaders | None = None,