From b645ee24d10d913f00412a994447fbc72346961f Mon Sep 17 00:00:00 2001 From: Qoder CLI Date: Thu, 28 May 2026 03:35:52 +0000 Subject: [PATCH 1/3] [ISSUE #10398] Fix TLS certificate hot-reload native memory leak by releasing old SslContext --- .../grpc/ProxyAndTlsProtocolNegotiator.java | 18 +++++++++++++++--- .../remoting/netty/NettyRemotingServer.java | 13 ++++++++++++- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java index 4222dacaad2..21132f82ff3 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java @@ -45,6 +45,7 @@ import io.grpc.netty.shaded.io.netty.handler.ssl.util.SelfSignedCertificate; import io.grpc.netty.shaded.io.netty.util.AsciiString; import io.grpc.netty.shaded.io.netty.util.CharsetUtil; +import io.grpc.netty.shaded.io.netty.util.ReferenceCountUtil; import java.io.IOException; import java.io.InputStream; @@ -77,7 +78,7 @@ public class ProxyAndTlsProtocolNegotiator implements InternalProtocolNegotiator */ private static final int SSL_RECORD_HEADER_LENGTH = 5; - private static SslContext sslContext; + private static volatile SslContext sslContext; public ProxyAndTlsProtocolNegotiator() { try { @@ -113,9 +114,10 @@ public static void loadSslContext() throws CertificateException, IOException { provider = SslProvider.JDK; log.info("Using JDK SSL provider"); } + SslContext newSslContext; if (proxyConfig.isTlsTestModeEnable()) { SelfSignedCertificate selfSignedCertificate = new SelfSignedCertificate(); - sslContext = GrpcSslContexts.forServer(selfSignedCertificate.certificate(), selfSignedCertificate.privateKey()) + newSslContext = GrpcSslContexts.forServer(selfSignedCertificate.certificate(), selfSignedCertificate.privateKey()) .sslProvider(provider) .trustManager(InsecureTrustManagerFactory.INSTANCE) .clientAuth(ClientAuth.NONE) @@ -128,7 +130,7 @@ public static void loadSslContext() throws CertificateException, IOException { Paths.get(tlsKeyPath)); InputStream serverCertificateStream = Files.newInputStream( Paths.get(tlsCertPath))) { - sslContext = GrpcSslContexts.forServer(serverCertificateStream, + newSslContext = GrpcSslContexts.forServer(serverCertificateStream, serverKeyInputStream, StringUtils.isNotBlank(tlsKeyPassword) ? tlsKeyPassword : null) .trustManager(InsecureTrustManagerFactory.INSTANCE) @@ -136,6 +138,16 @@ public static void loadSslContext() throws CertificateException, IOException { .build(); } } + SslContext oldSslContext = sslContext; + sslContext = newSslContext; + if (oldSslContext != null) { + try { + ReferenceCountUtil.release(oldSslContext); + log.info("Old SslContext released for proxy server"); + } catch (Exception e) { + log.warn("Failed to release old SslContext for proxy server", e); + } + } } private class ProxyAndTlsProtocolHandler extends ByteToMessageDecoder { diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java index 578c102daa4..cddb7afda3f 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java @@ -53,6 +53,7 @@ import io.netty.util.HashedWheelTimer; import io.netty.util.Timeout; import io.netty.util.TimerTask; +import io.netty.util.ReferenceCountUtil; import io.netty.util.concurrent.DefaultEventExecutorGroup; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -183,7 +184,17 @@ public void loadSslContext() { if (tlsMode != TlsMode.DISABLED) { try { - sslContext = TlsHelper.buildSslContext(false); + SslContext newSslContext = TlsHelper.buildSslContext(false); + SslContext oldSslContext = this.sslContext; + this.sslContext = newSslContext; + if (oldSslContext != null) { + try { + ReferenceCountUtil.release(oldSslContext); + log.info("Old SslContext released for server"); + } catch (Exception e) { + log.warn("Failed to release old SslContext for server", e); + } + } log.info("SslContext created for server"); } catch (CertificateException | IOException e) { log.error("Failed to create SslContext for server", e); From a48a96fd793f3b91ab9d3829c2e9a82e3d0c3092 Mon Sep 17 00:00:00 2001 From: Qoder CLI Date: Thu, 28 May 2026 04:01:09 +0000 Subject: [PATCH 2/3] fix: add missing SslContext import in NettyRemotingServer for PR #10399 CI --- .../org/apache/rocketmq/remoting/netty/NettyRemotingServer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java index cddb7afda3f..c059f5347f2 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java @@ -45,6 +45,7 @@ import io.netty.handler.codec.haproxy.HAProxyMessageDecoder; import io.netty.handler.codec.haproxy.HAProxyProtocolVersion; import io.netty.handler.codec.haproxy.HAProxyTLV; +import io.netty.handler.ssl.SslContext; import io.netty.handler.timeout.IdleState; import io.netty.handler.timeout.IdleStateEvent; import io.netty.handler.timeout.IdleStateHandler; From 2d98883bd32f904954935e9fd0f13e87a7c23b0b Mon Sep 17 00:00:00 2001 From: qianye Date: Mon, 1 Jun 2026 10:31:50 +0800 Subject: [PATCH 3/3] docs: add comments explaining race condition and JDK SslContext no-op behavior Co-Authored-By: Claude Opus 4.6 --- .../proxy/grpc/ProxyAndTlsProtocolNegotiator.java | 8 ++++++++ .../rocketmq/remoting/netty/NettyRemotingServer.java | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java index 21132f82ff3..fbe6d846340 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java @@ -141,6 +141,14 @@ public static void loadSslContext() throws CertificateException, IOException { SslContext oldSslContext = sslContext; sslContext = newSslContext; if (oldSslContext != null) { + // Release the old SslContext to free native memory (OpenSSL provider only). + // ReferenceCountUtil.release() is a no-op for JDK SslContext since it does not + // implement ReferenceCounted. + // Note: there is a theoretical race where an event-loop thread could read the old + // sslContext (volatile) and call newHandler() after release. In practice this is + // negligible because cert reload is very infrequent and the window is nanoseconds. + // Worst case: the single new connection gets an IllegalReferenceCountException and + // the client retries successfully — no pod crash or service disruption. try { ReferenceCountUtil.release(oldSslContext); log.info("Old SslContext released for proxy server"); diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java index c059f5347f2..ab2e208f484 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java @@ -189,6 +189,14 @@ public void loadSslContext() { SslContext oldSslContext = this.sslContext; this.sslContext = newSslContext; if (oldSslContext != null) { + // Release the old SslContext to free native memory (OpenSSL provider only). + // ReferenceCountUtil.release() is a no-op for JDK SslContext since it does not + // implement ReferenceCounted. + // Note: there is a theoretical race where an event-loop thread could read the old + // sslContext (volatile) and call newHandler() after release. In practice this is + // negligible because cert reload is very infrequent and the window is nanoseconds. + // Worst case: the single new connection gets an IllegalReferenceCountException and + // the client retries successfully — no pod crash or service disruption. try { ReferenceCountUtil.release(oldSslContext); log.info("Old SslContext released for server");