From 968d5e232aa322044c76b61a69800a3fb36543eb Mon Sep 17 00:00:00 2001 From: Ashutosh Kumar Singh Date: Thu, 21 May 2026 23:47:40 +0530 Subject: [PATCH 1/7] Fix ZLibDecompressor dropping data past the first gzip member MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a response body contains concatenated gzip members (RFC 1952 §2.2), zlib sets eof and moves the remaining bytes to unused_data once the first member is fully consumed. decompress_sync() was not checking unused_data, so every member after the first was silently discarded. Apply the same while-eof-and-unused_data loop that ZSTDDecompressor already uses for multi-frame zstd streams. Add unused_data to ZLibDecompressObjProtocol so the attribute is typed. Include three tests mirroring the existing ZSTD multi-frame test suite. Fixes #7157 Signed-off-by: Ashutosh Kumar Singh --- CHANGES/7157.bugfix.rst | 6 ++++++ CONTRIBUTORS.txt | 1 + aiohttp/compression_utils.py | 21 +++++++++++++++++++++ tests/test_compression_utils.py | 25 +++++++++++++++++++++++++ 4 files changed, 53 insertions(+) create mode 100644 CHANGES/7157.bugfix.rst diff --git a/CHANGES/7157.bugfix.rst b/CHANGES/7157.bugfix.rst new file mode 100644 index 00000000000..83c145f45d2 --- /dev/null +++ b/CHANGES/7157.bugfix.rst @@ -0,0 +1,6 @@ +Fixed :class:`~aiohttp.ZLibDecompressor` silently dropping data past the first +member when decompressing concatenated gzip/deflate streams. Each subsequent +member is now handed to a fresh decompressor, matching the behaviour already +implemented for ZSTD multi-frame streams. + +-- by :user:`Ashutosh-177` diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index b65e45ddc4a..aee356ca061 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -49,6 +49,7 @@ Andrew Top Andrew Zhou Andrii Soldatenko Anes Abismail +Ashutosh Kumar Singh Antoine Pietri Anton Kasyanov Anton Zhdan-Pushkin diff --git a/aiohttp/compression_utils.py b/aiohttp/compression_utils.py index 9373344e65c..e619041bf7f 100644 --- a/aiohttp/compression_utils.py +++ b/aiohttp/compression_utils.py @@ -55,6 +55,9 @@ def eof(self) -> bool: ... @property def unconsumed_tail(self) -> bytes: ... + @property + def unused_data(self) -> bytes: ... + class ZLibBackendProtocol(Protocol): MAX_WBITS: int @@ -284,6 +287,24 @@ def decompress_sync( ) # Only way to know that isal has no further data is checking we get no output self._last_empty = result == b"" + + # Handle concatenated gzip/deflate streams (multi-member). + # After a member ends, unused_data holds the start of the next member. + # Create a fresh decompressor for each subsequent member. + while self._decompressor.eof and self._decompressor.unused_data: + unused = self._decompressor.unused_data + self._decompressor = self._zlib_backend.decompressobj(wbits=self._mode) + remaining = ( + max_length - len(result) + if max_length != ZLIB_MAX_LENGTH_UNLIMITED + else ZLIB_MAX_LENGTH_UNLIMITED + ) + if max_length != ZLIB_MAX_LENGTH_UNLIMITED and remaining <= 0: + break + chunk = self._decompressor.decompress(unused, remaining) + self._last_empty = chunk == b"" + result += chunk + return result def flush(self, length: int = 0) -> bytes: diff --git a/tests/test_compression_utils.py b/tests/test_compression_utils.py index 3362b8feed0..607644cc100 100644 --- a/tests/test_compression_utils.py +++ b/tests/test_compression_utils.py @@ -1,5 +1,6 @@ """Tests for compression utils.""" +import gzip import sys import pytest @@ -87,3 +88,27 @@ def test_zstd_multi_frame_max_length_exhausted_preserves_unused_data() -> None: assert result1 == b"AAAA" result2 = d.decompress_sync(frame3) assert result2 == b"BBBBCCCC" + + +def test_zlib_gzip_multi_member_unlimited() -> None: + d = ZLibDecompressor(encoding="gzip") + member1 = gzip.compress(b"AAAA") + member2 = gzip.compress(b"BBBB") + result = d.decompress_sync(member1 + member2) + assert result == b"AAAABBBB" + + +def test_zlib_gzip_multi_member_max_length_partial() -> None: + d = ZLibDecompressor(encoding="gzip") + member1 = gzip.compress(b"AAAA") + member2 = gzip.compress(b"BBBB") + result = d.decompress_sync(member1 + member2, max_length=6) + assert result == b"AAAABB" + + +def test_zlib_gzip_multi_member_max_length_exhausted() -> None: + d = ZLibDecompressor(encoding="gzip") + member1 = gzip.compress(b"AAAA") + member2 = gzip.compress(b"BBBB") + result = d.decompress_sync(member1 + member2, max_length=4) + assert result == b"AAAA" From 1aaf370def992537ded48b73bdb1cd3c577eb891 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 21 May 2026 18:18:49 +0000 Subject: [PATCH 2/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- CONTRIBUTORS.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index aee356ca061..ba5a94eb2ea 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -49,7 +49,6 @@ Andrew Top Andrew Zhou Andrii Soldatenko Anes Abismail -Ashutosh Kumar Singh Antoine Pietri Anton Kasyanov Anton Zhdan-Pushkin @@ -58,6 +57,7 @@ Arie Bovenberg Arseny Timoniq Artem Yushkovskiy Arthur Darcet +Ashutosh Kumar Singh Austin Scola Bai Haoran Ben Bader From 79ac7911dff1afdac43cc5685507a6646f55f448 Mon Sep 17 00:00:00 2001 From: Ashutosh Kumar Singh Date: Thu, 21 May 2026 23:57:42 +0530 Subject: [PATCH 3/7] Fix doc-spelling failures in changelog entry Add gzip and decompressor to the spelling wordlist, and replace the unresolvable Sphinx class cross-reference with a plain code literal. --- CHANGES/7157.bugfix.rst | 2 +- docs/spelling_wordlist.txt | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES/7157.bugfix.rst b/CHANGES/7157.bugfix.rst index 83c145f45d2..60f06d8da9f 100644 --- a/CHANGES/7157.bugfix.rst +++ b/CHANGES/7157.bugfix.rst @@ -1,4 +1,4 @@ -Fixed :class:`~aiohttp.ZLibDecompressor` silently dropping data past the first +Fixed ``ZLibDecompressor`` silently dropping data past the first member when decompressing concatenated gzip/deflate streams. Each subsequent member is now handed to a fresh decompressor, matching the behaviour already implemented for ZSTD multi-frame streams. diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 185d9ebfdf5..75d3d0c8323 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -99,6 +99,7 @@ Cython Cythonize cythonized de +decompressor deduplicate defs Dependabot @@ -145,6 +146,7 @@ github google gunicorn gunicorn’s +gzip gzipped hackish highlevel From 2daddc799e93c769bacf6274d3bef7d2b5dbba07 Mon Sep 17 00:00:00 2001 From: Ashutosh Kumar Singh Date: Fri, 22 May 2026 00:11:30 +0530 Subject: [PATCH 4/7] Preserve unused bytes when max_length is hit mid-member When the output budget is exhausted in the multi-member loop, store the leftover compressed bytes in _pending_unused_data so the next decompress_sync() call picks them up, matching the ZSTDDecompressor behaviour. Also expose _pending_unused_data in data_available and add a test that mirrors test_zstd_multi_frame_max_length_exhausted_preserves_unused_data. --- aiohttp/compression_utils.py | 11 ++++++++++- tests/test_compression_utils.py | 11 +++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/aiohttp/compression_utils.py b/aiohttp/compression_utils.py index e619041bf7f..b0e958d0b32 100644 --- a/aiohttp/compression_utils.py +++ b/aiohttp/compression_utils.py @@ -278,10 +278,14 @@ def __init__( self._zlib_backend: Final = ZLibBackendWrapper(ZLibBackend._zlib_backend) self._decompressor = self._zlib_backend.decompressobj(wbits=self._mode) self._last_empty = False + self._pending_unused_data: bytes | None = None def decompress_sync( self, data: Buffer, max_length: int = ZLIB_MAX_LENGTH_UNLIMITED ) -> bytes: + if self._pending_unused_data is not None: + data = self._pending_unused_data + bytes(data) + self._pending_unused_data = None result = self._decompressor.decompress( self._decompressor.unconsumed_tail + data, max_length ) @@ -300,6 +304,7 @@ def decompress_sync( else ZLIB_MAX_LENGTH_UNLIMITED ) if max_length != ZLIB_MAX_LENGTH_UNLIMITED and remaining <= 0: + self._pending_unused_data = unused break chunk = self._decompressor.decompress(unused, remaining) self._last_empty = chunk == b"" @@ -316,7 +321,11 @@ def flush(self, length: int = 0) -> bytes: @property def data_available(self) -> bool: - return bool(self._decompressor.unconsumed_tail) or not self._last_empty + return ( + bool(self._decompressor.unconsumed_tail) + or not self._last_empty + or self._pending_unused_data is not None + ) @property def eof(self) -> bool: diff --git a/tests/test_compression_utils.py b/tests/test_compression_utils.py index 607644cc100..5deebc8470d 100644 --- a/tests/test_compression_utils.py +++ b/tests/test_compression_utils.py @@ -112,3 +112,14 @@ def test_zlib_gzip_multi_member_max_length_exhausted() -> None: member2 = gzip.compress(b"BBBB") result = d.decompress_sync(member1 + member2, max_length=4) assert result == b"AAAA" + + +def test_zlib_gzip_multi_member_max_length_exhausted_preserves_unused_data() -> None: + d = ZLibDecompressor(encoding="gzip") + member1 = gzip.compress(b"AAAA") + member2 = gzip.compress(b"BBBB") + member3 = gzip.compress(b"CCCC") + result1 = d.decompress_sync(member1 + member2, max_length=4) + assert result1 == b"AAAA" + result2 = d.decompress_sync(member3) + assert result2 == b"BBBBCCCC" From 12751fdca7fa9a854daa9626092b6be88350c99e Mon Sep 17 00:00:00 2001 From: Ashutosh Kumar Singh Date: Sun, 31 May 2026 00:37:28 +0530 Subject: [PATCH 5/7] Add HTTP parser gzip multi-member tests and align loop with ZSTD Align the multi-member loop in ZLibDecompressor.decompress_sync() with the cleaner ZSTD pattern: update max_length in-place rather than computing a separate remaining variable each iteration. Also add the eof-at-boundary reset that ZSTD already has: when a gzip member ends exactly at a chunk boundary, unused_data is empty so the while loop never runs, but the spent decompressor would error on the next feed_data() call. Reset it to a fresh decompressobj the same way the ZSTD path does. Add four HTTP parser-level tests mirroring the ZSTD multi-frame suite: all-at-once, chunked, split mid-member, and many small members. --- aiohttp/compression_utils.py | 21 ++++++---- tests/test_http_parser.py | 79 ++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 9 deletions(-) diff --git a/aiohttp/compression_utils.py b/aiohttp/compression_utils.py index b0e958d0b32..32a5d2e0161 100644 --- a/aiohttp/compression_utils.py +++ b/aiohttp/compression_utils.py @@ -298,18 +298,21 @@ def decompress_sync( while self._decompressor.eof and self._decompressor.unused_data: unused = self._decompressor.unused_data self._decompressor = self._zlib_backend.decompressobj(wbits=self._mode) - remaining = ( - max_length - len(result) - if max_length != ZLIB_MAX_LENGTH_UNLIMITED - else ZLIB_MAX_LENGTH_UNLIMITED - ) - if max_length != ZLIB_MAX_LENGTH_UNLIMITED and remaining <= 0: - self._pending_unused_data = unused - break - chunk = self._decompressor.decompress(unused, remaining) + if max_length != ZLIB_MAX_LENGTH_UNLIMITED: + max_length -= len(result) + if max_length <= 0: + self._pending_unused_data = unused + break + chunk = self._decompressor.decompress(unused, max_length) self._last_empty = chunk == b"" result += chunk + # Member ended exactly at chunk boundary — no unused_data, but the + # next feed_data() call would fail on the spent decompressor. + # Prepare a fresh one for the next chunk. + if self._decompressor.eof: + self._decompressor = self._zlib_backend.decompressobj(wbits=self._mode) + return result def flush(self, length: int = 0) -> bytes: diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index bb9ef3393bb..3dbd3170f79 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -1,6 +1,7 @@ # Tests for aiohttp/protocol.py import asyncio +import gzip import platform import re import sys @@ -2639,6 +2640,84 @@ async def test_http_payload_zstandard_many_small_frames( assert out.is_eof() + async def test_http_payload_gzip_multi_member( + self, protocol: BaseProtocol + ) -> None: + member1 = gzip.compress(b"first") + member2 = gzip.compress(b"second") + payload = member1 + member2 + out = aiohttp.StreamReader( + protocol, DEFAULT_CHUNK_SIZE, loop=asyncio.get_running_loop() + ) + p = HttpPayloadParser( + out, + length=len(payload), + compression="gzip", + headers_parser=HeadersParser(), + ) + p.feed_data(payload) + assert b"firstsecond" == b"".join(out._buffer) + assert out.is_eof() + + async def test_http_payload_gzip_multi_member_chunked( + self, protocol: BaseProtocol + ) -> None: + member1 = gzip.compress(b"chunk1") + member2 = gzip.compress(b"chunk2") + out = aiohttp.StreamReader( + protocol, DEFAULT_CHUNK_SIZE, loop=asyncio.get_running_loop() + ) + p = HttpPayloadParser( + out, + length=len(member1) + len(member2), + compression="gzip", + headers_parser=HeadersParser(), + ) + p.feed_data(member1) + p.feed_data(member2) + assert b"chunk1chunk2" == b"".join(out._buffer) + assert out.is_eof() + + async def test_http_payload_gzip_member_split_mid_chunk( + self, protocol: BaseProtocol + ) -> None: + member1 = gzip.compress(b"AAAA") + member2 = gzip.compress(b"BBBB") + combined = member1 + member2 + split_point = len(member1) + 3 # 3 bytes into member2 + out = aiohttp.StreamReader( + protocol, DEFAULT_CHUNK_SIZE, loop=asyncio.get_running_loop() + ) + p = HttpPayloadParser( + out, + length=len(combined), + compression="gzip", + headers_parser=HeadersParser(), + ) + p.feed_data(combined[:split_point]) + p.feed_data(combined[split_point:]) + assert b"AAAABBBB" == b"".join(out._buffer) + assert out.is_eof() + + async def test_http_payload_gzip_many_small_members( + self, protocol: BaseProtocol + ) -> None: + parts = [f"part{i}".encode() for i in range(10)] + payload = b"".join(gzip.compress(p) for p in parts) + out = aiohttp.StreamReader( + protocol, DEFAULT_CHUNK_SIZE, loop=asyncio.get_running_loop() + ) + p = HttpPayloadParser( + out, + length=len(payload), + compression="gzip", + headers_parser=HeadersParser(), + ) + p.feed_data(payload) + assert b"".join(parts) == b"".join(out._buffer) + assert out.is_eof() + + class TestDeflateBuffer: async def test_feed_data(self, protocol: BaseProtocol) -> None: buf = aiohttp.StreamReader( From d91fdce1ecaf73ad1e46167516274e72dd96bfe6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 30 May 2026 19:08:53 +0000 Subject: [PATCH 6/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_http_parser.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 3dbd3170f79..e57b5912f54 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -2639,10 +2639,7 @@ async def test_http_payload_zstandard_many_small_frames( assert b"".join(parts) == b"".join(out._buffer) assert out.is_eof() - - async def test_http_payload_gzip_multi_member( - self, protocol: BaseProtocol - ) -> None: + async def test_http_payload_gzip_multi_member(self, protocol: BaseProtocol) -> None: member1 = gzip.compress(b"first") member2 = gzip.compress(b"second") payload = member1 + member2 From 6a5a3545f21157200e24d106770846d4d5150c1d Mon Sep 17 00:00:00 2001 From: Ashutosh Kumar Singh Date: Sun, 31 May 2026 18:53:24 +0530 Subject: [PATCH 7/7] Fix eof reset breaking deflate after stream ends The chunk-boundary reset was unconditionally replacing the spent decompressor with a fresh one, which set eof=False. DeflateBuffer.feed_eof() checks not decompressor.eof for deflate encoding and raises ContentEncodingError when it sees False, breaking all deflate responses. Guard the reset on gzip mode (wbits > MAX_WBITS) only. Deflate has a single stream and relies on eof=True to signal completion; gzip multi-member streams need the fresh decompressor for the next chunk. --- aiohttp/compression_utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/aiohttp/compression_utils.py b/aiohttp/compression_utils.py index 32a5d2e0161..400df141470 100644 --- a/aiohttp/compression_utils.py +++ b/aiohttp/compression_utils.py @@ -309,8 +309,9 @@ def decompress_sync( # Member ended exactly at chunk boundary — no unused_data, but the # next feed_data() call would fail on the spent decompressor. - # Prepare a fresh one for the next chunk. - if self._decompressor.eof: + # Only reset for gzip; deflate's feed_eof() relies on eof=True to + # confirm the stream is complete. + if self._decompressor.eof and self._mode > self._zlib_backend.MAX_WBITS: self._decompressor = self._zlib_backend.decompressobj(wbits=self._mode) return result