Support chunked encoding in requests

examples/ChunkRequest/ChunkRequest.ino

@@ -0,0 +1,202 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+// Copyright 2016-2026 Hristo Gochkov, Mathieu Carbou, Emil Muratov, Mitch Bradley
+
+//
+// - Test for chunked encoding in requests
+//
+
+#include <Arduino.h>
+#if defined(ESP32) || defined(LIBRETINY)
+#include <AsyncTCP.h>
+#include <WiFi.h>
+#elif defined(ESP8266)
+#include <ESP8266WiFi.h>
+#include <ESPAsyncTCP.h>
+#elif defined(TARGET_RP2040) || defined(TARGET_RP2350) || defined(PICO_RP2040) || defined(PICO_RP2350)
+#include <RPAsyncTCP.h>
+#include <WiFi.h>
+#endif
+
+#include <ESPAsyncWebServer.h>
+#include <LittleFS.h>
+
+using namespace asyncsrv;
+
+// Tests:
+//
+// Upload a file  with PUT
+//   curl -T myfile.txt http://192.168.4.1/
+//
+// Upload a file with PUT using chunked encoding
+//   curl -T bigfile.txt -H 'Transfer-Encoding: chunked' http://192.168.4.1/
+//   ** Note: If the file will not fit in the available space, the server
+//   ** does not know that in advance due to the lack of a Content-Length header.
+//   ** The transfer will proceed until the filesystem fills up, then the transfer
+//   ** will fail and the partial file will be deleted.  This works correctly with
+//   ** recent versions (e.g. pioarduino) of the arduinoespressif32 framework, but
+//   ** fails with the stale 3.20017.241212+sha.dcc1105b version due to a LittleFS
+//   ** bug that has since been fixed.
+//
+// Immediately reject a chunked PUT that will not fit in available space
+//   curl -T bigfile.txt -H 'Transfer-Encoding: chunked' -H 'X-Expected-Entity-Length: 99999999' http://192.168.4.1/
+//   ** Note: MacOS WebDAVFS supplies the X-Expected-Entity-Length header with its
+//   ** chunked PUTs
+
+// This struct is used with _tempObject to communicate between handleBody and a subsequent handleRequest
+struct RequestState {
+  File outFile;
+};
+
+void handleRequest(AsyncWebServerRequest *request) {
+  Serial.print(request->methodToString());
+  Serial.print(" ");
+  Serial.println(request->url());
+
+  if (request->method() != HTTP_PUT) {
+    request->send(400);  // Bad Request
+    return;
+  }
+
+  // If request->_tempObject is not null, handleBody already
+  // did the necessary work for a PUT operation
+  auto state = static_cast<RequestState *>(request->_tempObject);
+  if (state) {
+    if (state->outFile) {
+      // The file was already opened and written in handleBody so
+      // we are done.  We will handle PUT without body data below.
+      state->outFile.close();
+      request->send(201);  // Created
+    }
+    delete state;
+    request->_tempObject = nullptr;
+    return;
+  }
+
+  String path = request->url();
+
+  if (request->method() == HTTP_PUT) {
+    // This PUT code executes if the body was empty, which
+    // can happen if the client creates a zero-length file.
+    // MacOS WebDAVFS does that, then later LOCKs the file
+    // and issues a subsequent PUT with body contents.
+
+#ifdef ESP32
+    File file = LittleFS.open(path, "w", true);
+#else
+    File file = LittleFS.open(path, "w");
+#endif
+
+    if (file) {
+      file.close();
+      request->send(201);  // Created
+      return;
+    }
+    request->send(403);
+    return;
+  }
+
+  request->send(404);
+}
+
+void handleBody(AsyncWebServerRequest *request, unsigned char *data, size_t len, size_t index, size_t total) {
+  if (request->method() == HTTP_PUT) {
+    auto state = static_cast<RequestState *>(request->_tempObject);
+    if (index == 0) {
+      // parse the url to a proper path
+      String path = request->url();
+
+      state = new RequestState{File()};
+      request->_tempObject = static_cast<void *>(state);
+
+      if (total) {
+#ifdef ESP32
+        size_t avail = LittleFS.totalBytes() - LittleFS.usedBytes();
+#else
+        FSInfo info;
+        LittleFS.info(info);
+        auto avail = info.totalBytes - info.usedBytes;
+#endif
+        avail -= 4096;  // Reserve a block for overhead
+        if (total > avail) {
+          Serial.printf("PUT %d bytes will not fit in available space (%d).\n", total, avail);
+          request->send(507, "text/plain", "Too large for available storage\r\n");
+          return;
+        }
+      }
+      Serial.print("Opening ");
+      Serial.print(path);
+      Serial.println(" from handleBody");
+#ifdef ESP32
+      File file = LittleFS.open(path, "w", true);
+#else
+      File file = LittleFS.open(path, "w");
+#endif
+      if (!file) {
+        request->send(500, "text/plain", "Cannot create the file");
+        return;
+      }
+      if (file.isDirectory()) {
+        file.close();
+        Serial.println("Cannot PUT to a directory");
+        request->send(403, "text/plain", "Cannot PUT to a directory");
+        return;
+      }
+      // If we already returned, the File object in request->_tempObject
+      // is the default-contructed one.  The presence of
+
+      std::swap(state->outFile, file);
+      // Now request->_tempObject contains the actual file object which owns it,
+      // and default-constructed File() object is in file, which will
+      // go out of scope
+    }
+    if (state && state->outFile) {
+      Serial.printf("write %d at %d\n", len, index);
+      auto actual = state->outFile.write(data, len);
+      if (actual != len) {
+        Serial.println("WebDAV write failed.  Deleting file.");
+
+        // Replace the File object in state with a null one
+        File file{};
+        std::swap(state->outFile, file);
+        file.close();
+
+        String path = request->url();
+        LittleFS.remove(path);
+        request->send(507, "text/plain", "Too large for available storage\r\n");
+        return;
+      }
+    }
+  }
+}
+
+static AsyncWebServer server(80);
+
+void setup() {
+  Serial.begin(115200);
+
+#if ASYNCWEBSERVER_WIFI_SUPPORTED
+#define AP_SUBNET 100
+  IPAddress local_IP(192, 168, AP_SUBNET, 1);
+  IPAddress gateway(192, 168, AP_SUBNET, 1);
+  IPAddress subnet(255, 255, 255, 0);
+  WiFi.softAPConfig(local_IP, gateway, subnet);
+
+  WiFi.mode(WIFI_AP);
+  WiFi.softAP("esp-captive");
+#endif
+
+#ifdef ESP32
+  LittleFS.begin(true);
+#else
+  LittleFS.begin();
+#endif
+
+  server.onRequestBody(handleBody);
+  server.onNotFound(handleRequest);
+
+  server.begin();
+}
+
+void loop() {
+  delay(100);
+}

platformio.ini

@@ -6,6 +6,7 @@ lib_dir = .
 ; src_dir = examples/Auth
 ; src_dir = examples/CaptivePortal
 ; src_dir = examples/CatchAllHandler
+; src_dir = examples/ChunkRequest
 ; src_dir = examples/ChunkResponse
 ; src_dir = examples/ChunkRetryResponse
 ; src_dir = examples/CORS

src/ESPAsyncWebServer.h

@@ -280,6 +280,12 @@ class AsyncWebServerRequest {
   size_t _itemBufferIndex;
   bool _itemIsFile;
 
+  size_t _chunkStartIndex;  // Offset from start of the chunked data stream
+  size_t _chunkOffset;      // Offset into the current chunk
+  size_t _chunkSize;        // Size of the current chunk
+  uint8_t _chunkedParseState;
+  bool _parseChunkedBytes(uint8_t *data, size_t len);
+
   void _onPoll();
   void _onAck(size_t len, uint32_t time);
   void _onError(int8_t error);

src/WebRequest.cpp

@@ -29,11 +29,20 @@ enum {
   PARSE_REQ_FAIL = 4
 };
 
+enum {
+  CHUNK_NONE = 0,   // Body transfer encoding is not chunked
+  CHUNK_LENGTH,     // Getting chunk length - HHHH[;...] CR LF
+  CHUNK_EXTENSION,  // Getting chunk length - HHHH[;...] CR LF
+  CHUNK_DATA,       // Handling chunk data
+  CHUNK_END,        // Getting chunk end marker  - CR LF
+};
+
 AsyncWebServerRequest::AsyncWebServerRequest(AsyncWebServer *s, AsyncClient *c)
   : _client(c), _server(s), _handler(NULL), _response(NULL), _onDisconnectfn(NULL), _temp(), _parseState(PARSE_REQ_START), _version(0), _method(HTTP_ANY),
     _url(), _host(), _contentType(), _boundary(), _authorization(), _reqconntype(RCT_HTTP), _authMethod(AsyncAuthType::AUTH_NONE), _isMultipart(false),
     _isPlainPost(false), _expectingContinue(false), _contentLength(0), _parsedLength(0), _multiParseState(0), _boundaryPosition(0), _itemStartIndex(0),
-    _itemSize(0), _itemName(), _itemFilename(), _itemType(), _itemValue(), _itemBuffer(0), _itemBufferIndex(0), _itemIsFile(false), _tempObject(NULL) {
+    _itemSize(0), _itemName(), _itemFilename(), _itemType(), _itemValue(), _itemBuffer(0), _itemBufferIndex(0), _itemIsFile(false),
+    _chunkedParseState(CHUNK_NONE), _tempObject(NULL) {
   c->onError(
     [](void *r, AsyncClient *c, int8_t error) {
       (void)c;
@@ -164,6 +173,14 @@ void AsyncWebServerRequest::_onData(void *buf, size_t len) {
         }
       }
     } else if (_parseState == PARSE_REQ_BODY) {
+      if (_chunkedParseState != CHUNK_NONE) {
+        if (_parseChunkedBytes((uint8_t *)buf, len)) {
+          _parseState = PARSE_REQ_END;
+          _runMiddlewareChain();
+          _send();
+        }
+        break;
+      }
       // A handler should be already attached at this point in _parseLine function.
       // If handler does nothing (_onRequest is NULL), we don't need to really parse the body.
       const bool needParse = _handler && !_handler->isRequestHandlerTrivial();
@@ -334,6 +351,78 @@ bool AsyncWebServerRequest::_parseReqHead() {
   return true;
 }
 
+// Returns true when done
+bool AsyncWebServerRequest::_parseChunkedBytes(uint8_t *buf, size_t len) {
+  for (size_t i = 0; i < len;) {
+    if (_chunkedParseState == CHUNK_DATA) {
+      // In DATA state, we pass the bytes off to handleBody as a group
+
+      // In order to avoid allocating an extra buffer, the data
+      // blocks that we pass on do not necessarily correspond to
+      // whole chunks.  We just send however much we already have,
+      // anticipating that more will arrive later.  handleBody()
+      // cannot assume that it receives entire chunks at once.
+      // That should not be a problem because we do not attach
+      // any semantic meaning to chunks.  That might change if
+      // we were to support chunk extensions, but that seems
+      // unlikely since RFC9112 suggests that they are only
+      // useful for very specialized purposes.
+      size_t curLen = std::min(_chunkSize - _chunkOffset, len - i);
+
+      // On the final zero-length chunk, _chunkSize - _chunkOffset
+      // will be zero, so we will call handleBody with a zero size,
+      // marking the end of the data stream.
+
+      if (_handler) {
+        _handler->handleBody(this, buf + i, curLen, _chunkStartIndex, _contentLength);
+      }
+      _chunkOffset += curLen;
+      _chunkStartIndex += curLen;
+      i += curLen;
+      if (_chunkOffset == _chunkSize) {
+        _chunkedParseState = CHUNK_END;
+      }
+    } else {
+      // In other states we process the bytes one by one
+      uint8_t data = buf[i++];
+
+      if (_chunkedParseState == CHUNK_LENGTH) {
+        // Incrementally decode a hex number
+        if (data >= '0' && data <= '9') {
+          _chunkSize = (_chunkSize * 16) + (data - '0');
+        } else if (data >= 'A' && data <= 'F') {
+          _chunkSize = (_chunkSize * 16) + (data - 'A' + 10);
+        } else if (data >= 'a' && data <= 'f') {
+          _chunkSize = (_chunkSize * 16) + (data - 'a' + 10);
+        } else if (data == ';') {
+          _chunkedParseState = CHUNK_EXTENSION;
+        } else if (data == '\n') {
+          _chunkOffset = 0;
+          _chunkedParseState = CHUNK_DATA;
+        }
+      } else if (_chunkedParseState == CHUNK_EXTENSION) {
+        if (data == '\n') {
+          // A zero length chunk marks the end of the chunk stream
+          _chunkOffset = 0;
+          _chunkedParseState = CHUNK_DATA;
+        }
+      } else if (_chunkedParseState == CHUNK_END) {
+        if (data == '\n') {
+          if (_chunkSize == 0) {
+            // If we needed to support trailers, we would switch to
+            // TRAILER state, but since we have no use case for them,
+            // we just stop processing the body.
+            return true;
+          }
+          _chunkSize = 0;
+          _chunkedParseState = CHUNK_LENGTH;
+        }
+      }
+    }
+  }
+  return false;
+}
+
 bool AsyncWebServerRequest::_parseReqHeader() {
   AsyncWebHeader header = AsyncWebHeader::parse(_temp);
   if (header) {
@@ -348,7 +437,10 @@ bool AsyncWebServerRequest::_parseReqHeader() {
         _boundary.replace(String('"'), String());
         _isMultipart = true;
       }
-    } else if (name.equalsIgnoreCase(T_Content_Length)) {
+    } else if (name.equalsIgnoreCase(T_Content_Length) || name.equalsIgnoreCase(T_X_Expected_Entity_Length)) {
+      // MacOS WebDAVFS uses X-Expected-Entity-Length to indicate the
+      // total length of a chunked request body.  It is useful to
+      // determine if a PUT can possibly fit in the available space.
       _contentLength = atoi(value.c_str());
     } else if (name.equalsIgnoreCase(T_EXPECT) && value.equalsIgnoreCase(T_100_CONTINUE)) {
       _expectingContinue = true;
@@ -385,6 +477,17 @@ bool AsyncWebServerRequest::_parseReqHeader() {
         // WebEvent request can be uniquely identified by header:  [Accept: text/event-stream]
         _reqconntype = RCT_EVENT;
       }
+    } else if (name.equalsIgnoreCase(T_Transfer_Encoding)) {
+      String lowcase(value);
+      lowcase.toLowerCase();
+
+      if (lowcase.indexOf("chunked") != -1) {
+        _chunkSize = 0;
+        _chunkStartIndex = 0;
+        _chunkedParseState = CHUNK_LENGTH;
+        _itemIsFile = true;
+        _itemFilename = _url;
+      }
     }
     _headers.emplace_back(std::move(header));
   }
@@ -680,7 +783,7 @@ void AsyncWebServerRequest::_parseLine() {
         String response(T_HTTP_100_CONT);
         _client->write(response.c_str(), response.length());
       }
-      if (_contentLength) {
+      if (_contentLength || _chunkedParseState != CHUNK_NONE) {
         _parseState = PARSE_REQ_BODY;
       } else {
         _parseState = PARSE_REQ_END;

src/literals.h

@@ -101,6 +101,7 @@ static constexpr const char T_uri[] = "uri";
 static constexpr const char T_username[] = "username";
 static constexpr const char T_WS[] = "websocket";
 static constexpr const char T_WWW_AUTH[] = "WWW-Authenticate";
+static constexpr const char T_X_Expected_Entity_Length[] = "X-Expected-Entity-Length";
 
 // HTTP Methods
 static constexpr const char T_ANY[] = "ANY";
@@ -215,6 +216,7 @@ DECLARE_STR(T_HTTP_CODE_502, "Bad Gateway");
 DECLARE_STR(T_HTTP_CODE_503, "Service Unavailable");
 DECLARE_STR(T_HTTP_CODE_504, "Gateway Time-out");
 DECLARE_STR(T_HTTP_CODE_505, "HTTP Version Not Supported");
+DECLARE_STR(T_HTTP_CODE_507, "Insufficient storage");
 DECLARE_STR(T_HTTP_CODE_ANY, "Unknown code");
 
 static constexpr const char *T_only_once_headers[] = {