multipart file upload support for async file

Author: tsimoshka

httpx/_compat.py

@@ -0,0 +1,28 @@
+import sys
+
+if sys.version_info >= (3, 10):
+    from contextlib import aclosing
+else:
+    from contextlib import asynccontextmanager
+    from typing import Any, AsyncIterator, Awaitable, Protocol, TypeVar
+
+    class _SupportsAclose(Protocol):
+        def aclose(self) -> Awaitable[object]: ...
+
+    _SupportsAcloseT = TypeVar("_SupportsAcloseT", bound=_SupportsAclose)
+
+    @asynccontextmanager
+    async def aclosing(thing: _SupportsAcloseT) -> AsyncIterator[Any]:
+        try:
+            yield thing
+        finally:
+            await thing.aclose()
+
+
+if sys.version_info >= (3, 13):
+    from typing import TypeIs
+else:
+    from typing_extensions import TypeIs
+
+
+__all__ = ["aclosing", "TypeIs"]

httpx/_multipart.py

@@ -7,13 +7,15 @@
 import typing
 from pathlib import Path
 
+from ._compat import aclosing
 from ._types import (
     AsyncByteStream,
     FileContent,
     FileTypes,
     RequestData,
     RequestFiles,
     SyncByteStream,
+    is_async_readable_binary_file,
 )
 from ._utils import (
     peek_filelike_length,
@@ -201,6 +203,11 @@ def render_headers(self) -> bytes:
         return self._headers
 
     def render_data(self) -> typing.Iterator[bytes]:
+        if is_async_readable_binary_file(self.file):
+            raise TypeError(
+                "Invalid type for file. AsyncReadableBinaryFile is not supported."
+            )
+
         if isinstance(self.file, (str, bytes)):
             yield to_bytes(self.file)
             return
@@ -216,10 +223,27 @@ def render_data(self) -> typing.Iterator[bytes]:
             yield to_bytes(chunk)
             chunk = self.file.read(self.CHUNK_SIZE)
 
+    async def arender_data(self) -> typing.AsyncGenerator[bytes]:
+        if not is_async_readable_binary_file(self.file):
+            for chunk in self.render_data():
+                yield chunk
+            return
+        await self.file.seek(0)
+        chunk = await self.file.read(self.CHUNK_SIZE)
+        while chunk:
+            yield to_bytes(chunk)
+            chunk = await self.file.read(self.CHUNK_SIZE)
+
     def render(self) -> typing.Iterator[bytes]:
         yield self.render_headers()
         yield from self.render_data()
 
+    async def arender(self) -> typing.AsyncGenerator[bytes]:
+        yield self.render_headers()
+        async with aclosing(self.arender_data()) as data:
+            async for chunk in data:
+                yield chunk
+
 
 class MultipartStream(SyncByteStream, AsyncByteStream):
     """
@@ -262,6 +286,19 @@ def iter_chunks(self) -> typing.Iterator[bytes]:
             yield b"\r\n"
         yield b"--%s--\r\n" % self.boundary
 
+    async def aiter_chunks(self) -> typing.AsyncGenerator[bytes]:
+        for field in self.fields:
+            yield b"--%s\r\n" % self.boundary
+            if isinstance(field, FileField):
+                async with aclosing(field.arender()) as data:
+                    async for chunk in data:
+                        yield chunk
+            else:
+                for chunk in field.render():
+                    yield chunk
+            yield b"\r\n"
+        yield b"--%s--\r\n" % self.boundary
+
     def get_content_length(self) -> int | None:
         """
         Return the length of the multipart encoded content, or `None` if
@@ -296,5 +333,6 @@ def __iter__(self) -> typing.Iterator[bytes]:
             yield chunk
 
     async def __aiter__(self) -> typing.AsyncIterator[bytes]:
-        for chunk in self.iter_chunks():
-            yield chunk
+        async with aclosing(self.aiter_chunks()) as data:
+            async for chunk in data:
+                yield chunk

httpx/_types.py

@@ -8,6 +8,7 @@
     IO,
     TYPE_CHECKING,
     Any,
+    AnyStr,
     AsyncIterable,
     AsyncIterator,
     Callable,
@@ -23,7 +24,7 @@
     Union,
 )
 
-from typing_extensions import TypeGuard
+from ._compat import TypeIs
 
 if TYPE_CHECKING:  # pragma: no cover
     from ._auth import Auth  # noqa: F401
@@ -75,7 +76,18 @@
 
 RequestData = Mapping[str, Any]
 
-FileContent = Union[IO[bytes], bytes, str]
+
+class AsyncReadableBinaryFile(Protocol):
+    async def __aiter__(self) -> AsyncIterator[AnyStr]: ...
+
+    async def read(self, size: int = -1) -> AnyStr: ...
+
+    def fileno(self) -> int: ...
+
+    async def seek(self, offset: int, whence: int | None = ...) -> int: ...
+
+
+FileContent = Union[IO[bytes], bytes, str, AsyncReadableBinaryFile]
 FileTypes = Union[
     # file (or bytes)
     FileContent,
@@ -118,22 +130,14 @@ async def aclose(self) -> None:
         pass
 
 
-class AsyncReadableBinaryFile(Protocol):
-    async def __aiter__(self) -> AsyncIterator[bytes]: ...
-
-    async def read(self, size: int = -1) -> bytes: ...
-
-    def fileno(self) -> int: ...
-
-
-def is_async_readable_binary_file(fp: Any) -> TypeGuard[AsyncReadableBinaryFile]:
+def is_async_readable_binary_file(fp: Any) -> TypeIs[AsyncReadableBinaryFile]:
     return (
         isinstance(fp, AsyncIterable)
         and hasattr(fp, "read")
         and inspect.iscoroutinefunction(fp.read)
         and hasattr(fp, "fileno")
         and callable(fp.fileno)
         and not inspect.iscoroutinefunction(fp.fileno)
-        and hasattr(fp, "mode")
-        and "b" in fp.mode
+        and hasattr(fp, "seek")
+        and inspect.iscoroutinefunction(fp.seek)
     )

tests/test_content.py

@@ -526,16 +526,13 @@ def test_allow_nan_false():
 @pytest.mark.parametrize("client_method", ["put", "post"])
 @pytest.mark.anyio
 async def test_chunked_async_file_content(
-    tmp_path, anyio_backend, monkeypatch, client_method
+    tmp_path, anyio_backend, monkeypatch, client_method, server
 ):
     total_chunks = 3
-
-    def echo_request_content(request: httpx.Request) -> httpx.Response:
-        return httpx.Response(200, content=request.content)
-
     content_bytes = b"".join([b"a" * AsyncIteratorByteStream.CHUNK_SIZE] * total_chunks)
     to_upload = tmp_path / "upload.txt"
     to_upload.write_bytes(content_bytes)
+    url = server.url.copy_with(path="/echo_body")
 
     async def checks(
         client: httpx.AsyncClient, async_file: AsyncReadableBinaryFile
@@ -557,9 +554,7 @@ def mock_fileno(*args):
 
         monkeypatch.setattr(async_file, "read", mock_read)
         monkeypatch.setattr(async_file, "fileno", mock_fileno)
-        response = await getattr(client, client_method)(
-            url="http://127.0.0.1:8000/", content=async_file
-        )
+        response = await getattr(client, client_method)(url=url, content=async_file)
         assert response.status_code == 200
         assert response.content == content_bytes
         assert response.request.headers["Content-Length"] == str(len(content_bytes))
@@ -570,19 +565,15 @@ def mock_fileno(*args):
         await anyio.open_file(to_upload, mode="rb")
         if anyio_backend != "trio"
         else await trio.open_file(to_upload, mode="rb") as async_file,
-        httpx.AsyncClient(
-            transport=httpx.MockTransport(echo_request_content)
-        ) as client,
+        httpx.AsyncClient() as client,
     ):
         assert is_async_readable_binary_file(async_file)
         await checks(client, async_file)
 
     if anyio_backend != "trio":  # aiofiles doesn't work with trio
         async with (
             aiofiles.open(to_upload, mode="rb") as aio_file,
-            httpx.AsyncClient(
-                transport=httpx.MockTransport(echo_request_content)
-            ) as client,
+            httpx.AsyncClient() as client,
         ):
             assert is_async_readable_binary_file(aio_file)
             await checks(client, aio_file)

tests/test_multipart.py

@@ -4,9 +4,13 @@
 import tempfile
 import typing
 
+import anyio
 import pytest
+import trio
 
 import httpx
+from httpx._multipart import FileField
+from httpx._types import AsyncReadableBinaryFile, is_async_readable_binary_file
 
 
 def echo_request_content(request: httpx.Request) -> httpx.Response:
@@ -467,3 +471,88 @@ def test_unicode_with_control_character(self):
         files = {"upload": (filename, b"<file content>")}
         request = httpx.Request("GET", "https://www.example.com", files=files)
         assert expected in request.read()
+
+
+@pytest.mark.anyio
+async def test_chunked_async_file_multipart(
+    tmp_path, anyio_backend, monkeypatch, server
+):
+    total_chunks = 3
+
+    content_bytes = b"".join([b"a" * FileField.CHUNK_SIZE] * total_chunks)
+    to_upload = tmp_path / "upload.txt"
+    to_upload.write_bytes(content_bytes)
+    url = server.url.copy_with(path="/echo_body")
+
+    async def checks(
+        client: httpx.AsyncClient, async_file: AsyncReadableBinaryFile
+    ) -> None:
+        read_called = 0
+        fileno_called = False
+        original_read = async_file.read
+        original_fileno = async_file.fileno
+
+        async def mock_read(*args, **kwargs):
+            nonlocal read_called
+            read_called += 1
+            return await original_read(*args, **kwargs)
+
+        def mock_fileno(*args):
+            nonlocal fileno_called
+            fileno_called = True
+            return original_fileno(*args)
+
+        monkeypatch.setattr(async_file, "read", mock_read)
+        monkeypatch.setattr(async_file, "fileno", mock_fileno)
+        response = await client.post(url=url, files={"file": async_file})
+        assert response.status_code == 200
+        boundary = response.request.headers["Content-Type"].split("boundary=")[-1]
+        boundary_bytes = boundary.encode("ascii")
+        pre_content = b"".join(
+            [
+                b"--" + boundary_bytes + b"\r\n",
+                b'Content-Disposition: form-data; name="file"; '
+                b'filename="upload.txt"\r\n',
+                b"Content-Type: text/plain\r\n",
+                b"\r\n",
+            ]
+        )
+        post_content = b"".join(
+            [
+                b"\r\n",
+                b"--" + boundary_bytes + b"--\r\n",
+            ]
+        )
+        assert response.content == b"".join(
+            [
+                pre_content,
+                content_bytes,
+                post_content,
+            ]
+        )
+        assert response.request.headers["Content-Length"] == str(
+            len(pre_content) + len(post_content) + len(content_bytes)
+        )
+        assert read_called == total_chunks + 1
+        assert fileno_called
+
+    async with (
+        await anyio.open_file(to_upload, mode="rb")
+        if anyio_backend != "trio"
+        else await trio.open_file(to_upload, mode="rb") as async_file,
+        httpx.AsyncClient() as client,
+    ):
+        assert is_async_readable_binary_file(async_file)
+
+        await checks(client, async_file)
+
+    async with (
+        await anyio.open_file(to_upload, mode="rb")
+        if anyio_backend != "trio"
+        else await trio.open_file(to_upload, mode="rb") as async_file,
+    ):
+        with (
+            httpx.Client() as sync_client,
+            pytest.raises(TypeError, match="AsyncReadableBinaryFile is not supported"),
+        ):
+            sync_client.post(url, files={"file": async_file})