Skip to content

Commit 996fd95

Browse files
authored
feat(stream): support upstream mTLS client cert via C-API (#114)
1 parent f88812c commit 996fd95

12 files changed

Lines changed: 490 additions & 9 deletions

.github/workflows/ci.yml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,10 @@ jobs:
1515
- name: apisix-runtime
1616
script_url: "https://raw.githubusercontent.com/api7/apisix-build-tools/master/build-apisix-runtime.sh"
1717
script_name: "build-apisix-runtime.sh"
18-
exclude_tests: ""
18+
# These assert stream-lua internal buffer-allocation debug logs that
19+
# changed in OpenResty 1.29.2.4 (stream-lua 0.0.19); they still run on
20+
# api7ee-runtime (OR 1.21, stream-lua 0.0.16) for functional coverage.
21+
exclude_tests: "t/stream/xrpc/downstream.t t/stream/xrpc/upstream.t"
1922
- name: api7ee-runtime
2023
script_url: "https://raw.githubusercontent.com/api7/apisix-build-tools/release/api7ee-runtime/build-api7ee-runtime.sh"
2124
script_name: "build-api7ee-runtime.sh"

lib/resty/apisix/stream/upstream.lua

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ ffi.cdef([[
1212
typedef intptr_t ngx_int_t;
1313
ngx_int_t
1414
ngx_stream_apisix_upstream_enable_tls(ngx_stream_lua_request_t *r);
15+
ngx_int_t
16+
ngx_stream_apisix_upstream_set_cert_and_key(ngx_stream_lua_request_t *r, void *cert, void *key);
1517
]])
1618
local _M = {}
1719

@@ -30,4 +32,19 @@ function _M.set_tls()
3032
end
3133

3234

35+
function _M.set_cert_and_key(cert, key)
36+
if not cert or not key then
37+
return nil, "both client certificate and private key should be given"
38+
end
39+
40+
local r = get_request()
41+
local ret = C.ngx_stream_apisix_upstream_set_cert_and_key(r, cert, key)
42+
if ret == NGX_ERROR then
43+
return nil, "error while setting upstream client cert and key"
44+
end
45+
46+
return true
47+
end
48+
49+
3350
return _M
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
diff --git src/stream/ngx_stream_proxy_module.c src/stream/ngx_stream_proxy_module.c
2+
--- src/stream/ngx_stream_proxy_module.c
3+
+++ src/stream/ngx_stream_proxy_module.c
4+
@@ -1219,7 +1219,11 @@ ngx_stream_proxy_ssl_init_connection(ngx_stream_session_t *s)
5+
}
6+
7+
s->connection->log->action = "SSL handshaking to upstream";
8+
9+
+#if (NGX_STREAM_APISIX)
10+
+ ngx_stream_apisix_set_upstream_ssl(s, pc);
11+
+#endif
12+
+
13+
rc = ngx_ssl_handshake(pc);
14+
15+
if (rc == NGX_AGAIN) {
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
diff --git src/stream/ngx_stream_proxy_module.c src/stream/ngx_stream_proxy_module.c
2+
--- src/stream/ngx_stream_proxy_module.c
3+
+++ src/stream/ngx_stream_proxy_module.c
4+
@@ -1219,7 +1219,11 @@ ngx_stream_proxy_ssl_init_connection(ngx_stream_session_t *s)
5+
}
6+
7+
s->connection->log->action = "SSL handshaking to upstream";
8+
9+
+#if (NGX_STREAM_APISIX)
10+
+ ngx_stream_apisix_set_upstream_ssl(s, pc);
11+
+#endif
12+
+
13+
rc = ngx_ssl_handshake(pc);
14+
15+
if (rc == NGX_AGAIN) {
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
diff --git src/stream/ngx_stream_proxy_module.c src/stream/ngx_stream_proxy_module.c
2+
--- src/stream/ngx_stream_proxy_module.c
3+
+++ src/stream/ngx_stream_proxy_module.c
4+
@@ -1219,7 +1219,11 @@ ngx_stream_proxy_ssl_init_connection(ngx_stream_session_t *s)
5+
}
6+
7+
s->connection->log->action = "SSL handshaking to upstream";
8+
9+
+#if (NGX_STREAM_APISIX)
10+
+ ngx_stream_apisix_set_upstream_ssl(s, pc);
11+
+#endif
12+
+
13+
rc = ngx_ssl_handshake(pc);
14+
15+
if (rc == NGX_AGAIN) {
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
diff --git src/stream/ngx_stream_proxy_module.c src/stream/ngx_stream_proxy_module.c
2+
--- src/stream/ngx_stream_proxy_module.c
3+
+++ src/stream/ngx_stream_proxy_module.c
4+
@@ -1219,7 +1219,11 @@ ngx_stream_proxy_ssl_init_connection(ngx_stream_session_t *s)
5+
}
6+
7+
s->connection->log->action = "SSL handshaking to upstream";
8+
9+
+#if (NGX_STREAM_APISIX)
10+
+ ngx_stream_apisix_set_upstream_ssl(s, pc);
11+
+#endif
12+
+
13+
rc = ngx_ssl_handshake(pc);
14+
15+
if (rc == NGX_AGAIN) {
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
diff --git src/stream/ngx_stream_proxy_module.c src/stream/ngx_stream_proxy_module.c
2+
--- src/stream/ngx_stream_proxy_module.c
3+
+++ src/stream/ngx_stream_proxy_module.c
4+
@@ -1219,7 +1219,11 @@ ngx_stream_proxy_ssl_init_connection(ngx_stream_session_t *s)
5+
}
6+
7+
s->connection->log->action = "SSL handshaking to upstream";
8+
9+
+#if (NGX_STREAM_APISIX)
10+
+ ngx_stream_apisix_set_upstream_ssl(s, pc);
11+
+#endif
12+
+
13+
rc = ngx_ssl_handshake(pc);
14+
15+
if (rc == NGX_AGAIN) {
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
diff --git src/stream/ngx_stream_proxy_module.c src/stream/ngx_stream_proxy_module.c
2+
--- src/stream/ngx_stream_proxy_module.c
3+
+++ src/stream/ngx_stream_proxy_module.c
4+
@@ -1219,7 +1219,11 @@ ngx_stream_proxy_ssl_init_connection(ngx_stream_session_t *s)
5+
}
6+
7+
s->connection->log->action = "SSL handshaking to upstream";
8+
9+
+#if (NGX_STREAM_APISIX)
10+
+ ngx_stream_apisix_set_upstream_ssl(s, pc);
11+
+#endif
12+
+
13+
rc = ngx_ssl_handshake(pc);
14+
15+
if (rc == NGX_AGAIN) {

src/stream/ngx_stream_apisix_module.c

Lines changed: 203 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@
44

55

66
typedef struct {
7+
#if (NGX_STREAM_SSL)
8+
STACK_OF(X509) *upstream_cert;
9+
EVP_PKEY *upstream_pkey;
10+
#endif
711
unsigned proxy_ssl_enabled:1;
812
} ngx_stream_apisix_ctx_t;
913

@@ -36,21 +40,89 @@ ngx_module_t ngx_stream_apisix_module = {
3640
};
3741

3842

39-
ngx_int_t
40-
ngx_stream_apisix_upstream_enable_tls(ngx_stream_lua_request_t *r)
43+
#if (NGX_STREAM_SSL)
44+
45+
static X509 *
46+
ngx_stream_apisix_x509_copy(const X509 *in)
4147
{
42-
ngx_stream_apisix_ctx_t *ctx;
48+
return X509_up_ref((X509 *) in) == 0 ? NULL : (X509 *) in;
49+
}
50+
51+
52+
static void
53+
ngx_stream_apisix_flush_ssl_error(void)
54+
{
55+
ERR_clear_error();
56+
}
57+
58+
59+
static void
60+
ngx_stream_apisix_cleanup_cert_and_key(ngx_stream_apisix_ctx_t *ctx)
61+
{
62+
if (ctx->upstream_cert != NULL) {
63+
sk_X509_pop_free(ctx->upstream_cert, X509_free);
64+
EVP_PKEY_free(ctx->upstream_pkey);
65+
66+
ctx->upstream_cert = NULL;
67+
ctx->upstream_pkey = NULL;
68+
}
69+
}
70+
71+
72+
static void
73+
ngx_stream_apisix_cleanup(void *data)
74+
{
75+
ngx_stream_apisix_ctx_t *ctx = data;
76+
77+
ngx_stream_apisix_cleanup_cert_and_key(ctx);
78+
}
79+
80+
#endif
81+
82+
83+
static ngx_stream_apisix_ctx_t *
84+
ngx_stream_apisix_get_module_ctx(ngx_stream_lua_request_t *r)
85+
{
86+
ngx_stream_apisix_ctx_t *ctx;
87+
#if (NGX_STREAM_SSL)
88+
ngx_pool_cleanup_t *cln;
89+
#endif
4390

4491
ctx = ngx_stream_lua_get_module_ctx(r, ngx_stream_apisix_module);
92+
4593
if (ctx == NULL) {
4694
ctx = ngx_pcalloc(r->pool, sizeof(ngx_stream_apisix_ctx_t));
4795
if (ctx == NULL) {
48-
return NGX_ERROR;
96+
return NULL;
97+
}
98+
99+
#if (NGX_STREAM_SSL)
100+
cln = ngx_pool_cleanup_add(r->pool, 0);
101+
if (cln == NULL) {
102+
return NULL;
49103
}
50104

105+
cln->data = ctx;
106+
cln->handler = ngx_stream_apisix_cleanup;
107+
#endif
108+
51109
ngx_stream_lua_set_ctx(r, ctx, ngx_stream_apisix_module);
52110
}
53111

112+
return ctx;
113+
}
114+
115+
116+
ngx_int_t
117+
ngx_stream_apisix_upstream_enable_tls(ngx_stream_lua_request_t *r)
118+
{
119+
ngx_stream_apisix_ctx_t *ctx;
120+
121+
ctx = ngx_stream_apisix_get_module_ctx(r);
122+
if (ctx == NULL) {
123+
return NGX_ERROR;
124+
}
125+
54126
ctx->proxy_ssl_enabled = 1;
55127

56128
return NGX_OK;
@@ -66,3 +138,130 @@ ngx_stream_apisix_is_proxy_ssl_enabled(ngx_stream_session_t *s)
66138

67139
return ctx != NULL && ctx->proxy_ssl_enabled;
68140
}
141+
142+
143+
#if (NGX_STREAM_SSL)
144+
145+
ngx_int_t
146+
ngx_stream_apisix_upstream_set_cert_and_key(ngx_stream_lua_request_t *r,
147+
void *data_cert, void *data_key)
148+
{
149+
STACK_OF(X509) *cert = data_cert;
150+
EVP_PKEY *key = data_key;
151+
STACK_OF(X509) *new_chain;
152+
ngx_stream_apisix_ctx_t *ctx;
153+
154+
if (cert == NULL || key == NULL) {
155+
return NGX_ERROR;
156+
}
157+
158+
ctx = ngx_stream_apisix_get_module_ctx(r);
159+
160+
if (ctx == NULL) {
161+
return NGX_ERROR;
162+
}
163+
164+
if (ctx->upstream_cert != NULL) {
165+
ngx_stream_apisix_cleanup_cert_and_key(ctx);
166+
}
167+
168+
if (EVP_PKEY_up_ref(key) == 0) {
169+
goto failed;
170+
}
171+
172+
new_chain = sk_X509_deep_copy(cert, ngx_stream_apisix_x509_copy,
173+
X509_free);
174+
if (new_chain == NULL) {
175+
EVP_PKEY_free(key);
176+
goto failed;
177+
}
178+
179+
ctx->upstream_cert = new_chain;
180+
ctx->upstream_pkey = key;
181+
182+
return NGX_OK;
183+
184+
failed:
185+
186+
ngx_stream_apisix_flush_ssl_error();
187+
188+
return NGX_ERROR;
189+
}
190+
191+
192+
void
193+
ngx_stream_apisix_set_upstream_ssl(ngx_stream_session_t *s, ngx_connection_t *c)
194+
{
195+
ngx_ssl_conn_t *sc = c->ssl->connection;
196+
ngx_stream_apisix_ctx_t *ctx;
197+
STACK_OF(X509) *cert;
198+
EVP_PKEY *pkey;
199+
X509 *x509;
200+
#ifdef OPENSSL_IS_BORINGSSL
201+
size_t i;
202+
#else
203+
int i;
204+
#endif
205+
206+
ctx = ngx_stream_get_module_ctx(s, ngx_stream_apisix_module);
207+
208+
if (ctx == NULL) {
209+
ngx_log_debug0(NGX_LOG_DEBUG_STREAM, c->log, 0,
210+
"skip overriding upstream SSL configuration, "
211+
"module ctx not set");
212+
return;
213+
}
214+
215+
if (ctx->upstream_cert != NULL) {
216+
cert = ctx->upstream_cert;
217+
pkey = ctx->upstream_pkey;
218+
219+
if (sk_X509_num(cert) < 1) {
220+
ngx_ssl_error(NGX_LOG_ERR, c->log, 0,
221+
"invalid client certificate provided while "
222+
"handshaking with upstream");
223+
goto failed;
224+
}
225+
226+
x509 = sk_X509_value(cert, 0);
227+
if (x509 == NULL) {
228+
ngx_ssl_error(NGX_LOG_ERR, c->log, 0, "sk_X509_value() failed");
229+
goto failed;
230+
}
231+
232+
if (SSL_use_certificate(sc, x509) == 0) {
233+
ngx_ssl_error(NGX_LOG_ERR, c->log, 0,
234+
"SSL_use_certificate() failed");
235+
goto failed;
236+
}
237+
238+
for (i = 1; i < sk_X509_num(cert); i++) {
239+
x509 = sk_X509_value(cert, i);
240+
if (x509 == NULL) {
241+
ngx_ssl_error(NGX_LOG_ERR, c->log, 0,
242+
"sk_X509_value() failed");
243+
goto failed;
244+
}
245+
246+
if (SSL_add1_chain_cert(sc, x509) == 0) {
247+
ngx_ssl_error(NGX_LOG_ERR, c->log, 0,
248+
"SSL_add1_chain_cert() failed");
249+
goto failed;
250+
}
251+
}
252+
253+
if (SSL_use_PrivateKey(sc, pkey) == 0) {
254+
ngx_ssl_error(NGX_LOG_ERR, c->log, 0,
255+
"SSL_use_PrivateKey() failed");
256+
goto failed;
257+
}
258+
}
259+
260+
return;
261+
262+
failed:
263+
264+
ngx_stream_apisix_flush_ssl_error();
265+
}
266+
267+
#endif

src/stream/ngx_stream_apisix_module.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,10 @@
77

88
ngx_int_t ngx_stream_apisix_is_proxy_ssl_enabled(ngx_stream_session_t *s);
99

10+
#if (NGX_STREAM_SSL)
11+
void ngx_stream_apisix_set_upstream_ssl(ngx_stream_session_t *s,
12+
ngx_connection_t *c);
13+
#endif
14+
1015

1116
#endif /* _NGX_STREAM_APISIX_H_INCLUDED_ */

0 commit comments

Comments
 (0)