request - streaming multipart/form-data requests using file like objects

Author: commonism

aiopenapi3/v30/glue.py

@@ -283,14 +283,38 @@ def _prepare_body(self, data, rbq):
             https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.1.0.md#media-type-object
             """
             media: aiopenapi3.v30.media.MediaType = self.operation.requestBody.content[ct]
-            if not media.schema_ or not isinstance(data, media.schema_.get_type()):
-                """expect the data to be a model"""
+            if media.schema_ and isinstance(data, media.schema_.get_type()):
+                """data is a model"""
+                params = parameters_from_multipart(data, media, rbq)
+                msg = encode_multipart_parameters(params)
+                self.req.content = msg.as_string()
+                self.req.headers["Content-Type"] = f'{msg.get_content_type()}; boundary="{msg.get_boundary()}"'
+            elif isinstance(data, list):
+                _files = list()
+                _data = dict()
+                for name, value in data:
+                    if isinstance(value, tuple):
+                        alias = fh = content_type = None
+                        headers = {}
+                        if len(value) == 4:
+                            (alias, fh, content_type, headers) = value
+                        elif len(value) == 3:
+                            (alias, fh, content_type) = value
+                        elif len(value) == 2:
+                            (alias, fh) = value
+                        elif len(value) == 1:
+                            (alias, fh) = value
+
+                        if (e := media.encoding.get(name)) is not None:
+                            headers.update({name: rbq[name] for name in e.headers.keys() if name in rbq})
+                        _value = (alias, fh, content_type, headers)
+                        _files.append((name, _value))
+                    else:
+                        _data[name] = value
+                self.req.files = _files
+                self.req.data = _data
+            else:
                 raise TypeError((type(data), media.schema_.get_type()))
-
-            params = parameters_from_multipart(data, media, rbq)
-            msg = encode_multipart_parameters(params)
-            self.req.content = msg.as_string()
-            self.req.headers["Content-Type"] = f'{msg.get_content_type()}; boundary="{msg.get_boundary()}"'
         elif (ct := "application/x-www-form-urlencoded") in self.operation.requestBody.content:
             self.req.headers["Content-Type"] = ct
             media: aiopenapi3.v30.media.MediaType = self.operation.requestBody.content[ct]

docs/source/advanced.rst

@@ -111,9 +111,12 @@ e.g.:
 mutualTLS
 ^^^^^^^^^
 MutualTLS authentication requires
+
     * certificate file
     * key file
-to authenticate to the remote server.
+    * (optional) password to keyfile
+
+to authenticate to the remote server, c.f. :ref:`httpx.Client.cert <https://www.python-httpx.org/api/#client>`_.
 
 .. code:: python
 
@@ -158,7 +161,7 @@ and
 Manual Requests
 ===============
 
-Creating a request manually allows accessing the httpx.Response as part of the :meth:`aiopenapi3.request.Request.request` return value.
+Creating a request manually allows accessing the httpx.Response as part of the :meth:`aiopenapi3.request.RequestBase.request` return value.
 
 .. code:: python
 
@@ -178,9 +181,65 @@ This can be used to provide certain header values (ETag), which are not paramete
     req.req.headers["If-Match"] = etag
     r = await req(parameters=parameters, data=kwargs)
 
+Request Streaming
+-----------------
+File uploads via "multipart/form-data" as mentioned in the httpx documentation
+(Multipart file `uploads <https://www.python-httpx.org/quickstart/#sending-multipart-file-uploads>`_ &
+`encoding <https://www.python-httpx.org/advanced/#multipart-file-encoding>`_)
+do not require the content of the request to be in memory but work with file-like-objects instead.
+
+httpx request streaming using file-like objects is limited to "multipart/form-data".
+It can not be used with "application/json" or "application/octet-stream".
+Additionally it does not support choice of encoding (such as base16, base64url or quoted-printable) as possible with OpenAPI v3.1 contentEncoding, which should not be a limitation.
+
+Use via `Manual Requests`_ using the :meth:`~aiopenapi3.request.RequestBase.request` API.
+
+.. code:: python
+
+    data = [
+        ("name",('form-data:name', file-like-object, content_type, headers))
+    ]
+
+A Request like
+
+.. code::
+
+    Content-Type: multipart/form-data; boundary=2a8ae6ad-f4ad-4d9a-a92c-6d217011fe0f
+    Content-Length: …
+
+    --2a8ae6ad-f4ad-4d9a-a92c-6d217011fe0f
+    Content-Disposition: form-data; name="datafile"; filename="r.gif"
+    Content-Type: image/gif
+
+    …
+
+would have to be created such as
+
+.. code:: python
+
+    data = [
+        ("datafile",('r.gif', Path('r.gif').open('rb'), "image/gif", {})
+    ]
+
+    req.request(data=data)
+
+Mixing file-like-objects and other form data fields is possible.
+
+.. code:: python
+
+    data = [
+        ("datafile",('r.gif', Path('r.gif').open('rb'), "image/gif", {}),
+        ("path", "media/images/r.gif"),
+    ]
+
+    req.request(data=data)
+
+
+See :aioai3:ref:`tests.stream_test.test_request`.
+
 
 Response Streaming
-==================
+------------------
 
 Responses exceeding the defined maximum content-length raise :class:`aiopenapi3.errors.ContentLengthExceededError` to prevent memory exhaustion.
 Though it is possible to increase the defined maximum content-length, it is preferable to use streaming for large responses, limiting the amount of memory required.
@@ -219,13 +278,13 @@ The main difference in the async use of the streaming is await & async for.
 
 
 Non-JSON Content
-----------------
+^^^^^^^^^^^^^^^^
 In case the content is not a model (application/octet-stream), the data can be read iteratively and written/processed.
 
 See :aioai3:ref:`tests.stream_test.test_stream_data`.
 
 JSON/Arrays of Models
----------------------
+^^^^^^^^^^^^^^^^^^^^^
 In case the large response is an array of models, iterative JSON parsing libraries can be used to process the data.
 
 See :aioai3:ref:`tests.stream_test.test_stream_array`.

docs/source/index.rst

@@ -41,7 +41,7 @@ While aiopenapi3 supports some of the more exotic features of the Swagger/OpenAP
 * :ref:`api:Parameter Encoding`
 * :ref:`advanced:Forms`
 * :ref:`advanced:mutualTLS` authentication
-* :ref:`advanced:Response Streaming`
+* :ref:`Request <advanced:Request Streaming>` and :ref:`Response <advanced:Response Streaming>` streaming to reduce memory usage
 
 some aspects of the specifications are implemented loose
 

pdm.lock

@@ -114,6 +114,7 @@ tests = [
     "nonecorn",
     "bootstrap-flask",
     "ijson",
+    "python-multipart>=0.0.6",
 ]
 
 [tool.pdm]

pyproject.toml

@@ -2,14 +2,20 @@
 import random
 import sys
 import string
-from typing import List
+
+if sys.version_info >= (3, 9):
+    from typing import List, Annotated
+else:
+    from typing import List
+    from typing_extensions import Annotated
+
 from pathlib import Path
 
 import uvloop
 from hypercorn.asyncio import serve
 from hypercorn.config import Config
 import pydantic
-from fastapi import FastAPI, Request, Response, Query
+from fastapi import FastAPI, Request, Response, Query, UploadFile, Body
 from fastapi.responses import PlainTextResponse
 
 import pytest
@@ -46,7 +52,6 @@ async def server(event_loop, config):
 @pytest_asyncio.fixture(scope="session")
 async def client(event_loop, server):
     api = await aiopenapi3.OpenAPI.load_async(f"http://{server.bind[0]}/openapi.json")
-
     return api
 
 
@@ -66,6 +71,22 @@ def files(request: Request, response: Response, number: int = Query(), size: int
         return [File(data=f.read(size)) for _ in range(number)]
 
 
+@app.post("/request-streaming", operation_id="request_streaming", response_model=int)
+def request_streaming(
+    request: Request, response: Response, files: List[UploadFile], path: Annotated[str, Body()]
+) -> int:
+    r = 0
+    for file in files:
+        if file.filename == "a.png":
+            assert file.headers["x-extra"] == "yes"
+
+        file.file.seek(0, 2)
+        offset = file.file.tell()
+        r += offset
+
+    return r + len(path)
+
+
 @pytest.mark.asyncio
 async def test_stream_data(event_loop, server, client):
     cl = client._max_response_content_length
@@ -81,6 +102,19 @@ async def test_stream_data(event_loop, server, client):
     assert l == cl
 
 
+@pytest.mark.asyncio
+async def test_request(event_loop, server, client):
+    import io
+
+    data = [
+        ("files", ("a.png", io.BytesIO(b"data:a"), "image/png", {"x-extra": "yes"})),
+        ("files", ("b.png", io.BytesIO(b"data:b"))),
+        ("path", "media/images"),
+    ]
+    size = await client._.request_streaming(data=data)
+    assert size == 24
+
+
 @pytest.mark.asyncio
 async def test_stream_array(event_loop, server, client):
     import ijson