Skip to content

Commit 32e0df6

Browse files
MitchBradleyMitchBradley
committed
Support chunked encoding in requests
1 parent bf0869b commit 32e0df6

5 files changed

Lines changed: 317 additions & 3 deletions

File tree

‎examples/ChunkRequest/ChunkRequest.ino‎

Lines changed: 202 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,202 @@
1+
// SPDX-License-Identifier: LGPL-3.0-or-later
2+
// Copyright 2016-2026 Hristo Gochkov, Mathieu Carbou, Emil Muratov, Mitch Bradley
3+
4+
//
5+
// - Test for chunked encoding in requests
6+
//
7+
8+
#include <Arduino.h>
9+
#if defined(ESP32) || defined(LIBRETINY)
10+
#include <AsyncTCP.h>
11+
#include <WiFi.h>
12+
#elif defined(ESP8266)
13+
#include <ESP8266WiFi.h>
14+
#include <ESPAsyncTCP.h>
15+
#elif defined(TARGET_RP2040) || defined(TARGET_RP2350) || defined(PICO_RP2040) || defined(PICO_RP2350)
16+
#include <RPAsyncTCP.h>
17+
#include <WiFi.h>
18+
#endif
19+
20+
#include <ESPAsyncWebServer.h>
21+
#include <LittleFS.h>
22+
23+
using namespace asyncsrv;
24+
25+
// Tests:
26+
//
27+
// Upload a file with PUT
28+
// curl -T myfile.txt http://192.168.4.1/
29+
//
30+
// Upload a file with PUT using chunked encoding
31+
// curl -T bigfile.txt -H 'Transfer-Encoding: chunked' http://192.168.4.1/
32+
// ** Note: If the file will not fit in the available space, the server
33+
// ** does not know that in advance due to the lack of a Content-Length header.
34+
// ** The transfer will proceed until the filesystem fills up, then the transfer
35+
// ** will fail and the partial file will be deleted. This works correctly with
36+
// ** recent versions (e.g. pioarduino) of the arduinoespressif32 framework, but
37+
// ** fails with the stale 3.20017.241212+sha.dcc1105b version due to a LittleFS
38+
// ** bug that has since been fixed.
39+
//
40+
// Immediately reject a chunked PUT that will not fit in available space
41+
// curl -T bigfile.txt -H 'Transfer-Encoding: chunked' -H 'X-Expected-Entity-Length: 99999999' http://192.168.4.1/
42+
// ** Note: MacOS WebDAVFS supplies the X-Expected-Entity-Length header with its
43+
// ** chunked PUTs
44+
45+
// This struct is used with _tempObject to communicate between handleBody and a subsequent handleRequest
46+
struct RequestState {
47+
File outFile;
48+
};
49+
50+
void handleRequest(AsyncWebServerRequest *request) {
51+
Serial.print(request->methodToString());
52+
Serial.print(" ");
53+
Serial.println(request->url());
54+
55+
if (request->method() != HTTP_PUT) {
56+
request->send(400); // Bad Request
57+
return;
58+
}
59+
60+
// If request->_tempObject is not null, handleBody already
61+
// did the necessary work for a PUT operation
62+
auto state = static_cast<RequestState *>(request->_tempObject);
63+
if (state) {
64+
if (state->outFile) {
65+
// The file was already opened and written in handleBody so
66+
// we are done. We will handle PUT without body data below.
67+
state->outFile.close();
68+
request->send(201); // Created
69+
}
70+
delete state;
71+
request->_tempObject = nullptr;
72+
return;
73+
}
74+
75+
String path = request->url();
76+
77+
if (request->method() == HTTP_PUT) {
78+
// This PUT code executes if the body was empty, which
79+
// can happen if the client creates a zero-length file.
80+
// MacOS WebDAVFS does that, then later LOCKs the file
81+
// and issues a subsequent PUT with body contents.
82+
83+
#ifdef ESP32
84+
File file = LittleFS.open(path, "w", true);
85+
#else
86+
File file = LittleFS.open(path, "w");
87+
#endif
88+
89+
if (file) {
90+
file.close();
91+
request->send(201); // Created
92+
return;
93+
}
94+
request->send(403);
95+
return;
96+
}
97+
98+
request->send(404);
99+
}
100+
101+
void handleBody(AsyncWebServerRequest *request, unsigned char *data, size_t len, size_t index, size_t total) {
102+
if (request->method() == HTTP_PUT) {
103+
auto state = static_cast<RequestState *>(request->_tempObject);
104+
if (index == 0) {
105+
// parse the url to a proper path
106+
String path = request->url();
107+
108+
state = new RequestState{File()};
109+
request->_tempObject = static_cast<void *>(state);
110+
111+
if (total) {
112+
#ifdef ESP32
113+
size_t avail = LittleFS.totalBytes() - LittleFS.usedBytes();
114+
#else
115+
FSInfo info;
116+
LittleFS.info(info);
117+
auto avail = info.totalBytes - info.usedBytes;
118+
#endif
119+
avail -= 4096; // Reserve a block for overhead
120+
if (total > avail) {
121+
Serial.printf("PUT %d bytes will not fit in available space (%d).\n", total, avail);
122+
request->send(507, "text/plain", "Too large for available storage\r\n");
123+
return;
124+
}
125+
}
126+
Serial.print("Opening ");
127+
Serial.print(path);
128+
Serial.println(" from handleBody");
129+
#ifdef ESP32
130+
File file = LittleFS.open(path, "w", true);
131+
#else
132+
File file = LittleFS.open(path, "w");
133+
#endif
134+
if (!file) {
135+
request->send(500, "text/plain", "Cannot create the file");
136+
return;
137+
}
138+
if (file.isDirectory()) {
139+
file.close();
140+
Serial.println("Cannot PUT to a directory");
141+
request->send(403, "text/plain", "Cannot PUT to a directory");
142+
return;
143+
}
144+
// If we already returned, the File object in request->_tempObject
145+
// is the default-contructed one. The presence of
146+
147+
std::swap(state->outFile, file);
148+
// Now request->_tempObject contains the actual file object which owns it,
149+
// and default-constructed File() object is in file, which will
150+
// go out of scope
151+
}
152+
if (state && state->outFile) {
153+
Serial.printf("write %d at %d\n", len, index);
154+
auto actual = state->outFile.write(data, len);
155+
if (actual != len) {
156+
Serial.println("WebDAV write failed. Deleting file.");
157+
158+
// Replace the File object in state with a null one
159+
File file{};
160+
std::swap(state->outFile, file);
161+
file.close();
162+
163+
String path = request->url();
164+
LittleFS.remove(path);
165+
request->send(507, "text/plain", "Too large for available storage\r\n");
166+
return;
167+
}
168+
}
169+
}
170+
}
171+
172+
static AsyncWebServer server(80);
173+
174+
void setup() {
175+
Serial.begin(115200);
176+
177+
#if ASYNCWEBSERVER_WIFI_SUPPORTED
178+
#define AP_SUBNET 100
179+
IPAddress local_IP(192, 168, AP_SUBNET, 1);
180+
IPAddress gateway(192, 168, AP_SUBNET, 1);
181+
IPAddress subnet(255, 255, 255, 0);
182+
WiFi.softAPConfig(local_IP, gateway, subnet);
183+
184+
WiFi.mode(WIFI_AP);
185+
WiFi.softAP("esp-captive");
186+
#endif
187+
188+
#ifdef ESP32
189+
LittleFS.begin(true);
190+
#else
191+
LittleFS.begin();
192+
#endif
193+
194+
server.onRequestBody(handleBody);
195+
server.onNotFound(handleRequest);
196+
197+
server.begin();
198+
}
199+
200+
void loop() {
201+
delay(100);
202+
}

‎platformio.ini‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ lib_dir = .
66
; src_dir = examples/Auth
77
; src_dir = examples/CaptivePortal
88
; src_dir = examples/CatchAllHandler
9+
; src_dir = examples/ChunkRequest
910
; src_dir = examples/ChunkResponse
1011
; src_dir = examples/ChunkRetryResponse
1112
; src_dir = examples/CORS

‎src/ESPAsyncWebServer.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -280,6 +280,12 @@ class AsyncWebServerRequest {
280280
size_t _itemBufferIndex;
281281
bool _itemIsFile;
282282

283+
size_t _chunkStartIndex; // Offset from start of the chunked data stream
284+
size_t _chunkOffset; // Offset into the current chunk
285+
size_t _chunkSize; // Size of the current chunk
286+
uint8_t _chunkedParseState;
287+
bool _parseChunkedBytes(uint8_t *data, size_t len);
288+
283289
void _onPoll();
284290
void _onAck(size_t len, uint32_t time);
285291
void _onError(int8_t error);

‎src/WebRequest.cpp‎

Lines changed: 106 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,20 @@ enum {
2929
PARSE_REQ_FAIL = 4
3030
};
3131

32+
enum {
33+
CHUNK_NONE = 0, // Body transfer encoding is not chunked
34+
CHUNK_LENGTH, // Getting chunk length - HHHH[;...] CR LF
35+
CHUNK_EXTENSION, // Getting chunk length - HHHH[;...] CR LF
36+
CHUNK_DATA, // Handling chunk data
37+
CHUNK_END, // Getting chunk end marker - CR LF
38+
};
39+
3240
AsyncWebServerRequest::AsyncWebServerRequest(AsyncWebServer *s, AsyncClient *c)
3341
: _client(c), _server(s), _handler(NULL), _response(NULL), _onDisconnectfn(NULL), _temp(), _parseState(PARSE_REQ_START), _version(0), _method(HTTP_ANY),
3442
_url(), _host(), _contentType(), _boundary(), _authorization(), _reqconntype(RCT_HTTP), _authMethod(AsyncAuthType::AUTH_NONE), _isMultipart(false),
3543
_isPlainPost(false), _expectingContinue(false), _contentLength(0), _parsedLength(0), _multiParseState(0), _boundaryPosition(0), _itemStartIndex(0),
36-
_itemSize(0), _itemName(), _itemFilename(), _itemType(), _itemValue(), _itemBuffer(0), _itemBufferIndex(0), _itemIsFile(false), _tempObject(NULL) {
44+
_itemSize(0), _itemName(), _itemFilename(), _itemType(), _itemValue(), _itemBuffer(0), _itemBufferIndex(0), _itemIsFile(false),
45+
_chunkedParseState(CHUNK_NONE), _tempObject(NULL) {
3746
c->onError(
3847
[](void *r, AsyncClient *c, int8_t error) {
3948
(void)c;
@@ -164,6 +173,14 @@ void AsyncWebServerRequest::_onData(void *buf, size_t len) {
164173
}
165174
}
166175
} else if (_parseState == PARSE_REQ_BODY) {
176+
if (_chunkedParseState != CHUNK_NONE) {
177+
if (_parseChunkedBytes((uint8_t *)buf, len)) {
178+
_parseState = PARSE_REQ_END;
179+
_runMiddlewareChain();
180+
_send();
181+
}
182+
break;
183+
}
167184
// A handler should be already attached at this point in _parseLine function.
168185
// If handler does nothing (_onRequest is NULL), we don't need to really parse the body.
169186
const bool needParse = _handler && !_handler->isRequestHandlerTrivial();
@@ -334,6 +351,78 @@ bool AsyncWebServerRequest::_parseReqHead() {
334351
return true;
335352
}
336353

354+
// Returns true when done
355+
bool AsyncWebServerRequest::_parseChunkedBytes(uint8_t *buf, size_t len) {
356+
for (size_t i = 0; i < len;) {
357+
if (_chunkedParseState == CHUNK_DATA) {
358+
// In DATA state, we pass the bytes off to handleBody as a group
359+
360+
// In order to avoid allocating an extra buffer, the data
361+
// blocks that we pass on do not necessarily correspond to
362+
// whole chunks. We just send however much we already have,
363+
// anticipating that more will arrive later. handleBody()
364+
// cannot assume that it receives entire chunks at once.
365+
// That should not be a problem because we do not attach
366+
// any semantic meaning to chunks. That might change if
367+
// we were to support chunk extensions, but that seems
368+
// unlikely since RFC9112 suggests that they are only
369+
// useful for very specialized purposes.
370+
size_t curLen = std::min(_chunkSize - _chunkOffset, len - i);
371+
372+
// On the final zero-length chunk, _chunkSize - _chunkOffset
373+
// will be zero, so we will call handleBody with a zero size,
374+
// marking the end of the data stream.
375+
376+
if (_handler) {
377+
_handler->handleBody(this, buf + i, curLen, _chunkStartIndex, _contentLength);
378+
}
379+
_chunkOffset += curLen;
380+
_chunkStartIndex += curLen;
381+
i += curLen;
382+
if (_chunkOffset == _chunkSize) {
383+
_chunkedParseState = CHUNK_END;
384+
}
385+
} else {
386+
// In other states we process the bytes one by one
387+
uint8_t data = buf[i++];
388+
389+
if (_chunkedParseState == CHUNK_LENGTH) {
390+
// Incrementally decode a hex number
391+
if (data >= '0' && data <= '9') {
392+
_chunkSize = (_chunkSize * 16) + (data - '0');
393+
} else if (data >= 'A' && data <= 'F') {
394+
_chunkSize = (_chunkSize * 16) + (data - 'A' + 10);
395+
} else if (data >= 'a' && data <= 'f') {
396+
_chunkSize = (_chunkSize * 16) + (data - 'a' + 10);
397+
} else if (data == ';') {
398+
_chunkedParseState = CHUNK_EXTENSION;
399+
} else if (data == '\n') {
400+
_chunkOffset = 0;
401+
_chunkedParseState = CHUNK_DATA;
402+
}
403+
} else if (_chunkedParseState == CHUNK_EXTENSION) {
404+
if (data == '\n') {
405+
// A zero length chunk marks the end of the chunk stream
406+
_chunkOffset = 0;
407+
_chunkedParseState = CHUNK_DATA;
408+
}
409+
} else if (_chunkedParseState == CHUNK_END) {
410+
if (data == '\n') {
411+
if (_chunkSize == 0) {
412+
// If we needed to support trailers, we would switch to
413+
// TRAILER state, but since we have no use case for them,
414+
// we just stop processing the body.
415+
return true;
416+
}
417+
_chunkSize = 0;
418+
_chunkedParseState = CHUNK_LENGTH;
419+
}
420+
}
421+
}
422+
}
423+
return false;
424+
}
425+
337426
bool AsyncWebServerRequest::_parseReqHeader() {
338427
AsyncWebHeader header = AsyncWebHeader::parse(_temp);
339428
if (header) {
@@ -348,7 +437,10 @@ bool AsyncWebServerRequest::_parseReqHeader() {
348437
_boundary.replace(String('"'), String());
349438
_isMultipart = true;
350439
}
351-
} else if (name.equalsIgnoreCase(T_Content_Length)) {
440+
} else if (name.equalsIgnoreCase(T_Content_Length) || name.equalsIgnoreCase(T_X_Expected_Entity_Length)) {
441+
// MacOS WebDAVFS uses X-Expected-Entity-Length to indicate the
442+
// total length of a chunked request body. It is useful to
443+
// determine if a PUT can possibly fit in the available space.
352444
_contentLength = atoi(value.c_str());
353445
} else if (name.equalsIgnoreCase(T_EXPECT) && value.equalsIgnoreCase(T_100_CONTINUE)) {
354446
_expectingContinue = true;
@@ -385,6 +477,17 @@ bool AsyncWebServerRequest::_parseReqHeader() {
385477
// WebEvent request can be uniquely identified by header: [Accept: text/event-stream]
386478
_reqconntype = RCT_EVENT;
387479
}
480+
} else if (name.equalsIgnoreCase(T_Transfer_Encoding)) {
481+
String lowcase(value);
482+
lowcase.toLowerCase();
483+
484+
if (lowcase.indexOf("chunked") != -1) {
485+
_chunkSize = 0;
486+
_chunkStartIndex = 0;
487+
_chunkedParseState = CHUNK_LENGTH;
488+
_itemIsFile = true;
489+
_itemFilename = _url;
490+
}
388491
}
389492
_headers.emplace_back(std::move(header));
390493
}
@@ -680,7 +783,7 @@ void AsyncWebServerRequest::_parseLine() {
680783
String response(T_HTTP_100_CONT);
681784
_client->write(response.c_str(), response.length());
682785
}
683-
if (_contentLength) {
786+
if (_contentLength || _chunkedParseState != CHUNK_NONE) {
684787
_parseState = PARSE_REQ_BODY;
685788
} else {
686789
_parseState = PARSE_REQ_END;

‎src/literals.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ static constexpr const char T_uri[] = "uri";
101101
static constexpr const char T_username[] = "username";
102102
static constexpr const char T_WS[] = "websocket";
103103
static constexpr const char T_WWW_AUTH[] = "WWW-Authenticate";
104+
static constexpr const char T_X_Expected_Entity_Length[] = "X-Expected-Entity-Length";
104105

105106
// HTTP Methods
106107
static constexpr const char T_ANY[] = "ANY";
@@ -215,6 +216,7 @@ DECLARE_STR(T_HTTP_CODE_502, "Bad Gateway");
215216
DECLARE_STR(T_HTTP_CODE_503, "Service Unavailable");
216217
DECLARE_STR(T_HTTP_CODE_504, "Gateway Time-out");
217218
DECLARE_STR(T_HTTP_CODE_505, "HTTP Version Not Supported");
219+
DECLARE_STR(T_HTTP_CODE_507, "Insufficient storage");
218220
DECLARE_STR(T_HTTP_CODE_ANY, "Unknown code");
219221

220222
static constexpr const char *T_only_once_headers[] = {

0 commit comments

Comments
 (0)