Skip to content

Commit 9d5763b

Browse files
committed
Initial support to proxy request with Transfer-Encoding: chunked
1 parent fdf5b2e commit 9d5763b

File tree

3 files changed

+210
-24
lines changed

3 files changed

+210
-24
lines changed

gateway/src/apicast/configuration/service.lua

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,8 @@ local function get_request_params(method)
176176

177177
if method == "GET" then
178178
return params
179+
elseif ngx.req.get_headers()["Transfer-Encoding"] == "chunked" then
180+
return params
179181
else
180182
ngx.req.read_body()
181183
local body_params, err = ngx.req.get_post_args()

gateway/src/apicast/http_proxy.lua

Lines changed: 47 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ local resty_resolver = require 'resty.resolver'
55
local round_robin = require 'resty.balancer.round_robin'
66
local http_proxy = require 'resty.http.proxy'
77
local file_reader = require("resty.file").file_reader
8+
local chunked_reader = require('resty.http.chunked').chunked_reader
9+
local chunked_writer = require('resty.http.chunked').chunked_writer
810

911
local _M = { }
1012

@@ -82,14 +84,23 @@ local function absolute_url(uri)
8284
end
8385

8486
local function forward_https_request(proxy_uri, uri, skip_https_connect)
85-
-- This is needed to call ngx.req.get_body_data() below.
86-
ngx.req.read_body()
87-
88-
local request = {
89-
uri = uri,
90-
method = ngx.req.get_method(),
91-
headers = ngx.req.get_headers(0, true),
92-
path = format('%s%s%s', ngx.var.uri, ngx.var.is_args, ngx.var.query_string or ''),
87+
local sock, err
88+
local chunksize = 32 * 1024
89+
local body
90+
91+
if ngx.req.get_headers()["Transfer-Encoding"] == "chunked" then
92+
-- The default ngx reader does not support chunked request
93+
-- so we will need to get the raw request socket and manually
94+
-- decode the chunked request
95+
sock, err = ngx.req.socket(true)
96+
97+
if not sock then
98+
return ngx.exit(ngx.HTTP_SERVICE_UNAVAILABLE)
99+
end
100+
body = chunked_reader(sock, chunksize)
101+
else
102+
-- This is needed to call ngx.req.get_body_data() below.
103+
ngx.req.read_body()
93104

94105
-- We cannot use resty.http's .get_client_body_reader().
95106
-- In POST requests with HTTPS, the result of that call is nil, and it
@@ -100,24 +111,31 @@ local function forward_https_request(proxy_uri, uri, skip_https_connect)
100111
-- read and need to be cached in a local file. This request will return
101112
-- nil, so after this we need to read the temp file.
102113
-- https://github.com/openresty/lua-nginx-module#ngxreqget_body_data
103-
body = ngx.req.get_body_data(),
104-
proxy_uri = proxy_uri
105-
}
106-
107-
if not request.body then
108-
local temp_file_path = ngx.req.get_body_file()
109-
ngx.log(ngx.INFO, "HTTPS Proxy: Request body is bigger than client_body_buffer_size, read the content from path='", temp_file_path, "'")
110-
111-
if temp_file_path then
112-
local body, err = file_reader(temp_file_path)
113-
if err then
114-
ngx.log(ngx.ERR, "HTTPS proxy: Failed to read temp body file, err: ", err)
115-
ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
116-
end
117-
request.body = body
114+
body = ngx.req.get_body_data()
115+
116+
if not body then
117+
local temp_file_path = ngx.req.get_body_file()
118+
ngx.log(ngx.INFO, "HTTPS Proxy: Request body is bigger than client_body_buffer_size, read the content from path='", temp_file_path, "'")
119+
120+
if temp_file_path then
121+
body, err = file_reader(temp_file_path)
122+
if err then
123+
ngx.log(ngx.ERR, "HTTPS proxy: Failed to read temp body file, err: ", err)
124+
ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
125+
end
126+
end
118127
end
119128
end
120129

130+
local request = {
131+
uri = uri,
132+
method = ngx.req.get_method(),
133+
headers = ngx.req.get_headers(0, true),
134+
path = format('%s%s%s', ngx.var.uri, ngx.var.is_args, ngx.var.query_string or ''),
135+
body = body,
136+
proxy_uri = proxy_uri
137+
}
138+
121139
local httpc, err = http_proxy.new(request, skip_https_connect)
122140

123141
if not httpc then
@@ -130,7 +148,12 @@ local function forward_https_request(proxy_uri, uri, skip_https_connect)
130148
res, err = httpc:request(request)
131149

132150
if res then
133-
httpc:proxy_response(res)
151+
-- if we are using raw socket we will need to send the response back with sock:send
152+
if sock then
153+
chunked_writer(sock, res ,chunksize)
154+
else
155+
httpc:proxy_response(res)
156+
end
134157
httpc:set_keepalive()
135158
else
136159
ngx.log(ngx.ERR, 'failed to proxy request to: ', proxy_uri, ' err : ', err)

gateway/src/resty/http/chunked.lua

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
local co_wrap_iter = require("resty.coroutines").co_wrap_iter
2+
local co_yield = coroutine._yield
3+
4+
local _M = {
5+
}
6+
7+
local cr_lf = "\r\n"
8+
local default_max_chunk_size = 32 * 1024 -- 32K
9+
10+
local function send(socket, data)
11+
if not data or data == '' then
12+
ngx.log(ngx.DEBUG, 'skipping sending nil')
13+
return
14+
end
15+
16+
return socket:send(data)
17+
end
18+
19+
local function print_err(error_code, ...)
20+
ngx.log(ngx.ERR, ...)
21+
return ngx.exit(error_code)
22+
end
23+
24+
-- TODO: support Trailers
25+
-- chunked_reader return a body reader that translates the data read from sock
26+
-- out of HTTP "chunked" format before returning it
27+
--
28+
-- The chunked reader return nil when the final 0-length chunk is read
29+
function _M.chunked_reader(sock, max_chunk_size)
30+
max_chunk_size = max_chunk_size or default_max_chunk_size
31+
32+
if not sock then
33+
return nil, "chunked_reader: invalid sock"
34+
end
35+
36+
return co_wrap_iter(function()
37+
local eof = false
38+
local remaining = 0
39+
local size = 0
40+
repeat
41+
-- If we still have data on this chunk
42+
if max_chunk_size and remaining > 0 then
43+
if remaining > max_chunk_size then
44+
-- Consume up to max_chunk_size
45+
size = max_chunk_size
46+
remaining = remaining - max_chunk_size
47+
else
48+
-- Consume all remaining
49+
size = remaining
50+
remaining = 0
51+
end
52+
else
53+
-- read a line from socket
54+
-- chunk-size CRLF
55+
local line, err = sock:receive()
56+
if not line then
57+
co_yield(nil, "chunked_reader: failed to receive chunk size, err: " .. err)
58+
end
59+
60+
size = tonumber(line, 16)
61+
if not size then
62+
co_yield(nil, "chunked_reader: unable to read chunksize")
63+
end
64+
65+
if max_chunk_size and size > max_chunk_size then
66+
-- Consume up to max_chunk_size
67+
remaining = size - max_chunk_size
68+
size = max_chunk_size
69+
end
70+
end
71+
72+
73+
if size > 0 then
74+
-- Receive the chunk
75+
local chunk, err = sock:receive(size)
76+
if not chunk then
77+
co_yield(nil, "chunked_reader: failed to receive chunk of size " .. size .. " err: " .. err)
78+
end
79+
80+
-- We're at the end of a chunk, read the next two bytes
81+
-- and verify they are "\r\n"
82+
local data, err = sock:receive(2)
83+
if not data then
84+
co_yield(nil, "chunked_reader: failed to receive chunk terminator, err: " .. err)
85+
end
86+
87+
chunk = string.format("%x\r\n", size) .. chunk .. data
88+
89+
co_yield(chunk)
90+
else
91+
-- we're at the end of a chunk, read the next two
92+
-- bytes to verify they are "\r\n".
93+
local chunk, err = sock:receive(2)
94+
if not chunk then
95+
co_yield(nil, "chunked_reader: failed to receive chunk terminator, err: " .. err)
96+
end
97+
98+
if chunk ~= "\r\n" then
99+
co_yield(nil, "chunked_reader: bad chunk terminator: " .. chunk)
100+
end
101+
102+
eof = true
103+
co_yield("0\r\n\r\n")
104+
break
105+
end
106+
until eof
107+
end)
108+
end
109+
110+
-- chunked_writer writes response body reader to sock in the HTTP/1.x server response format,
111+
-- including the status line, headers, body, and optional trailer.
112+
function _M.chunked_writer(sock, res, chunksize)
113+
local bytes, err
114+
chunksize = chunksize or 65536
115+
116+
-- Status line
117+
-- FIXME: should get protocol version from res?
118+
local status = "HTTP/1.1 " .. res.status .. " " .. res.reason .. cr_lf
119+
bytes, err = send(sock, status)
120+
if not bytes then
121+
print_err(503, "chunked_writer: failed to send status line, err: ", err)
122+
end
123+
124+
-- Rest of header
125+
for k, v in pairs(res.headers) do
126+
local header = k .. ": " .. v .. cr_lf
127+
bytes, err = send(sock, header)
128+
if not bytes then
129+
print_err(503, "chunked_writer: failed to send header, err: ", err)
130+
end
131+
end
132+
133+
-- End-of-header
134+
bytes, err = send(sock, cr_lf)
135+
if not bytes then
136+
print_err(503, "chunked_writer: failed to send end of header, err: ", err)
137+
end
138+
139+
-- Write body and trailer
140+
-- TODO: handle trailer
141+
if res.has_body then
142+
local reader = res.body_reader
143+
repeat
144+
local chunk, read_err
145+
146+
chunk, read_err = reader(chunksize)
147+
if read_err then
148+
print_err(503, "chunked_writer: failed to read body, err: ", err)
149+
end
150+
151+
if chunk then
152+
bytes, err = send(sock, chunk)
153+
if not bytes then
154+
print_err("chunked_writer: failed to send body, err: ", err)
155+
end
156+
end
157+
until not chunk
158+
end
159+
end
160+
161+
return _M

0 commit comments

Comments
 (0)