fix(stream): support upstream client certificate (mTLS) in L4 proxy (#13596)

Author: AlinsRan

.requirements

@@ -17,5 +17,5 @@
 
 APISIX_PACKAGE_NAME=apisix
 
-APISIX_RUNTIME=1.3.6
+APISIX_RUNTIME=1.3.8
 APISIX_DASHBOARD_COMMIT=c8d3466d3c36386d3888efbc8250cd8183c77298

apisix/init.lua

@@ -493,6 +493,32 @@ local function common_phase(phase_name)
 end
 
 
+-- Resolve the upstream client certificate referenced by `tls.client_cert_id`
+-- into `api_ctx.upstream_ssl`. Shared by the http and stream subsystems.
+-- Returns false on error (invalid/missing referenced ssl object).
+local function resolve_upstream_client_cert(api_ctx)
+    if not (api_ctx.matched_upstream and api_ctx.matched_upstream.tls and
+            api_ctx.matched_upstream.tls.client_cert_id) then
+        return true
+    end
+
+    local cert_id = api_ctx.matched_upstream.tls.client_cert_id
+    local upstream_ssl = router.router_ssl.get_by_id(cert_id)
+    if not upstream_ssl or upstream_ssl.type ~= "client" then
+        local err = upstream_ssl and
+            "ssl type should be 'client'" or
+            "ssl id [" .. cert_id .. "] not exits"
+        core.log.error("failed to get ssl cert: ", err)
+        return false
+    end
+
+    core.log.info("matched upstream client ssl object, id: ", cert_id,
+                  ", type: ", upstream_ssl.type)
+    api_ctx.upstream_ssl = upstream_ssl
+    return true
+end
+
+
 function _M.handle_upstream(api_ctx, route, enable_websocket)
     -- some plugins(ai-proxy...) request upstream by http client directly
     if api_ctx.bypass_nginx_upstream then
@@ -537,27 +563,12 @@ function _M.handle_upstream(api_ctx, route, enable_websocket)
         api_ctx.matched_upstream = route_val.upstream
     end
 
-    if api_ctx.matched_upstream and api_ctx.matched_upstream.tls and
-        api_ctx.matched_upstream.tls.client_cert_id then
-
-        local cert_id = api_ctx.matched_upstream.tls.client_cert_id
-        local upstream_ssl = router.router_ssl.get_by_id(cert_id)
-        if not upstream_ssl or upstream_ssl.type ~= "client" then
-            local err  = upstream_ssl and
-                "ssl type should be 'client'" or
-                "ssl id [" .. cert_id .. "] not exits"
-            core.log.error("failed to get ssl cert: ", err)
-
-            if is_http then
-                return core.response.exit(502)
-            end
-
-            return ngx_exit(1)
+    local ok = resolve_upstream_client_cert(api_ctx)
+    if not ok then
+        if is_http then
+            return core.response.exit(502)
         end
-
-        core.log.info("matched ssl: ",
-                  core.json.delay_encode(upstream_ssl, true))
-        api_ctx.upstream_ssl = upstream_ssl
+        return ngx_exit(1)
     end
 
     if enable_websocket then
@@ -1385,6 +1396,11 @@ function _M.stream_preread_phase()
         return
     end
 
+    local ok = resolve_upstream_client_cert(api_ctx)
+    if not ok then
+        return ngx_exit(1)
+    end
+
     local code, err = set_upstream(matched_route, api_ctx)
     if code then
         core.log.error("failed to set upstream: ", err)

apisix/upstream.lua

@@ -42,15 +42,25 @@ else
 end
 
 local set_stream_upstream_tls
+local set_stream_upstream_cert_and_key
 if not is_http then
     local ok, apisix_ngx_stream_upstream = pcall(require, "resty.apisix.stream.upstream")
     if ok then
         set_stream_upstream_tls = apisix_ngx_stream_upstream.set_tls
-    else
+        set_stream_upstream_cert_and_key = apisix_ngx_stream_upstream.set_cert_and_key
+    end
+    -- guard each function independently: an older runtime may expose the module
+    -- (set_tls) without the newer mTLS C-API (set_cert_and_key)
+    if not set_stream_upstream_tls then
         set_stream_upstream_tls = function ()
             return nil, "need to build APISIX-Runtime to support TLS over TCP upstream"
         end
     end
+    if not set_stream_upstream_cert_and_key then
+        set_stream_upstream_cert_and_key = function ()
+            return nil, "need to build APISIX-Runtime to support upstream mTLS over TCP"
+        end
+    end
 end
 
 
@@ -160,6 +170,54 @@ local function fill_node_info(up_conf, scheme, is_stream)
 end
 
 
+-- Set upstream client certificate (mTLS) for the stream (L4) subsystem.
+-- Mirrors the http subsystem: the cert/key are parsed and cached once (the key
+-- is AES-decrypted at rest by fetch_pkey) and applied to the upstream SSL
+-- handshake through the apisix-nginx-module stream C API, so the plaintext key
+-- is never stringified into an nginx variable.
+local function set_stream_upstream_client_cert(api_ctx, up_conf)
+    local tls = up_conf.tls
+    if not (tls and (tls.client_cert or tls.client_cert_id)) then
+        return true
+    end
+
+    local client_cert, client_key
+    if tls.client_cert_id then
+        if not api_ctx.upstream_ssl then
+            return nil, "failed to find upstream ssl object for client_cert_id"
+        end
+        client_cert = api_ctx.upstream_ssl.cert
+        client_key = api_ctx.upstream_ssl.key
+    else
+        client_cert = tls.client_cert
+        client_key = tls.client_key
+    end
+
+    if not (client_cert and client_key) then
+        return nil, "missing client certificate or key for upstream mTLS"
+    end
+
+    -- the sni here is just for logging
+    local sni = api_ctx.var.upstream_host
+    local cert, err = apisix_ssl.fetch_cert(sni, client_cert)
+    if not cert then
+        return nil, err
+    end
+
+    local key, err = apisix_ssl.fetch_pkey(sni, client_key)
+    if not key then
+        return nil, err
+    end
+
+    local ok, err = set_stream_upstream_cert_and_key(cert, key)
+    if not ok then
+        return nil, err
+    end
+
+    return true
+end
+
+
 function _M.set_by_route(route, api_ctx)
     if api_ctx.upstream_conf then
         -- upstream_conf has been set by traffic-split plugin
@@ -246,6 +304,11 @@ function _M.set_by_route(route, api_ctx)
             if sni then
                 ngx_var.upstream_sni = sni
             end
+
+            local ok, err = set_stream_upstream_client_cert(api_ctx, up_conf)
+            if not ok then
+                return 503, err
+            end
         end
         local node_ver = resource.get_nodes_ver(up_conf.resource_key)
         local resource_version = upstream_util.version(up_conf.resource_version,

ci/linux-install-openresty.sh

@@ -61,19 +61,19 @@ else
     sudo apt-get -y update --fix-missing
     sudo apt-get install -y build-essential gcc g++ cpanminus libxml2-dev libxslt-dev
 
-    if [ "$APISIX_RUNTIME" != "1.3.6" ]; then
+    if [ "$APISIX_RUNTIME" != "1.3.8" ]; then
         echo "Please update the apisix-runtime-debug checksum for APISIX_RUNTIME=$APISIX_RUNTIME" >&2
         exit 1
     fi
 
     case "$ARCH" in
         x86_64|amd64)
             DEB_ARCH="amd64"
-            EXPECTED_SHA256="f3c3836270e4d71c7154bea3dd13005cacad5b489eacf9fab7b048907fa4d641"
+            EXPECTED_SHA256="d617eb9dbabdaa97c9722c7b48260aa26d121c280ecbb2c5e1bdeebc6fbeeb8e"
             ;;
         arm64|aarch64)
             DEB_ARCH="arm64"
-            EXPECTED_SHA256="6f5ba1e4dee34f9c2593687b3e97dad53cbc1f2b90283961fd87eba62e4c9bc4"
+            EXPECTED_SHA256="4e263650a6bfb773b53ebf5643fed791d21115e92b4b370d0cd6d43c58fd870c"
             ;;
         *)
             echo "Unsupported architecture: $ARCH" >&2

docs/en/latest/mtls.md

@@ -208,3 +208,8 @@ curl http://127.0.0.1:9180/apisix/admin/upstreams/1 \
     }
 }'
 ```
+
+This also works in the stream (L4) subsystem: when an upstream uses the `tls`
+scheme and configures `tls.client_cert`/`tls.client_key` (or
+`tls.client_cert_id`), APISIX presents the client certificate while
+establishing the TLS connection to the upstream.