DSP-24887: Enable SAN-aware peer identity lookup for verifications

Enable SAN-aware peer identity lookup for Netty client endpoint verification based
on hostname.

Added a new opt-in SAN-aware peer identity feature via `SAN_PEER_IDENTITY_LOOKUP`
in Netty SSL context configuration. Implemented `SanPeerIdentityTrustManager` to
dynamically select the verification identity based on certificate SAN types,
including optional reverse-DNS lookup for IP-based peers with DNS SAN certs.

Integrated the feature into both JDK and OpenSSL client TLS paths by wrapping
`X509ExtendedTrustManager` during SSL context/session creation. Updated
`SslContextBuilder`, `OpenSslClientContext`, `ReferenceCountedOpenSslClientContext`,
and `JdkSslClientContext` so the SAN-aware verification behavior is propagated
consistently across providers.

Author: piyushk010

handler/src/main/java/io/netty/handler/ssl/JdkSslClientContext.java

@@ -20,6 +20,7 @@
 import java.security.Provider;
 import javax.net.ssl.KeyManager;
 import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.X509ExtendedTrustManager;
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLException;
 import javax.net.ssl.SSLSessionContext;
@@ -176,8 +177,8 @@ public JdkSslClientContext(
                         long sessionCacheSize, long sessionTimeout) throws SSLException {
         super(newSSLContext(provider, toX509CertificatesInternal(trustCertCollectionFile),
                 trustManagerFactory, null, null,
-                null, null, sessionCacheSize, sessionTimeout, null, KeyStore.getDefaultType(), null), true,
-                ciphers, cipherFilter, apn, ClientAuth.NONE, null, false);
+                null, null, sessionCacheSize, sessionTimeout, null, KeyStore.getDefaultType(),
+                null, false), true, ciphers, cipherFilter, apn, ClientAuth.NONE, null, false);
     }
 
     /**
@@ -260,7 +261,7 @@ public JdkSslClientContext(File trustCertCollectionFile, TrustManagerFactory tru
                 trustCertCollectionFile), trustManagerFactory,
                 toX509CertificatesInternal(keyCertChainFile), toPrivateKeyInternal(keyFile, keyPassword),
                 keyPassword, keyManagerFactory, sessionCacheSize, sessionTimeout,
-                        null, KeyStore.getDefaultType(), null), true,
+                null, KeyStore.getDefaultType(), null, false), true,
                 ciphers, cipherFilter, apn, ClientAuth.NONE, null, false);
     }
 
@@ -270,11 +271,11 @@ public JdkSslClientContext(File trustCertCollectionFile, TrustManagerFactory tru
                         KeyManagerFactory keyManagerFactory, Iterable<String> ciphers, CipherSuiteFilter cipherFilter,
                         ApplicationProtocolConfig apn, String[] protocols, long sessionCacheSize, long sessionTimeout,
                         SecureRandom secureRandom, String keyStoreType, String endpointIdentificationAlgorithm,
-                        ResumptionController resumptionController)
+                        ResumptionController resumptionController, boolean sanPeerIdentityLookup)
             throws SSLException {
         super(newSSLContext(sslContextProvider, trustCertCollection, trustManagerFactory,
                             keyCertChain, key, keyPassword, keyManagerFactory, sessionCacheSize,
-                            sessionTimeout, secureRandom, keyStoreType, resumptionController),
+                            sessionTimeout, secureRandom, keyStoreType, resumptionController, sanPeerIdentityLookup),
                 true, ciphers, cipherFilter, toNegotiator(apn, false), ClientAuth.NONE, protocols, false,
                 endpointIdentificationAlgorithm, resumptionController);
     }
@@ -285,7 +286,8 @@ private static SSLContext newSSLContext(Provider sslContextProvider,
                                             PrivateKey key, String keyPassword, KeyManagerFactory keyManagerFactory,
                                             long sessionCacheSize, long sessionTimeout,
                                             SecureRandom secureRandom, String keyStore,
-                                            ResumptionController resumptionController) throws SSLException {
+                                            ResumptionController resumptionController,
+                                            boolean sanPeerIdentityLookup) throws SSLException {
         try {
             if (trustCertCollection != null) {
                 trustManagerFactory = buildTrustManagerFactory(trustCertCollection, trustManagerFactory, keyStore);
@@ -297,8 +299,8 @@ private static SSLContext newSSLContext(Provider sslContextProvider,
             SSLContext ctx = sslContextProvider == null ? SSLContext.getInstance(PROTOCOL)
                 : SSLContext.getInstance(PROTOCOL, sslContextProvider);
             ctx.init(keyManagerFactory == null ? null : keyManagerFactory.getKeyManagers(),
-                     trustManagerFactory == null ? null :
-                             wrapIfNeeded(trustManagerFactory.getTrustManagers(), resumptionController),
+                     trustManagerFactory == null ? null : wrapIfNeeded(
+                             trustManagerFactory.getTrustManagers(), resumptionController, sanPeerIdentityLookup),
                      secureRandom);
 
             SSLSessionContext sessCtx = ctx.getClientSessionContext();
@@ -317,9 +319,11 @@ private static SSLContext newSSLContext(Provider sslContextProvider,
         }
     }
 
-    private static TrustManager[] wrapIfNeeded(TrustManager[] tms, ResumptionController resumptionController) {
-        if (resumptionController != null) {
-            for (int i = 0; i < tms.length; i++) {
+    private static TrustManager[] wrapIfNeeded(TrustManager[] tms, ResumptionController resumptionController,
+                                               boolean sanPeerIdentityLookup) {
+        for (int i = 0; i < tms.length; i++) {
+            tms[i] = SanPeerIdentityTrustManager.wrapIfNeeded(tms[i], sanPeerIdentityLookup);
+            if (resumptionController != null) {
                 tms[i] = resumptionController.wrapIfNeeded(tms[i]);
             }
         }

handler/src/main/java/io/netty/handler/ssl/OpenSslClientContext.java

@@ -196,7 +196,8 @@ public OpenSslClientContext(File trustCertCollectionFile, TrustManagerFactory tr
             OpenSslKeyMaterialProvider.validateKeyMaterialSupported(keyCertChain, key, keyPassword);
             sessionContext = newSessionContext(this, ctx, engineMap, trustCertCollection, trustManagerFactory,
                                                keyCertChain, key, keyPassword, keyManagerFactory, keyStore,
-                                               sessionCacheSize, sessionTimeout, resumptionController);
+                                               sessionCacheSize, sessionTimeout, endpointIdentificationAlgorithm,
+                                               resumptionController, options);
             success = true;
         } finally {
             if (!success) {

handler/src/main/java/io/netty/handler/ssl/ReferenceCountedOpenSslClientContext.java

@@ -73,7 +73,8 @@ public final class ReferenceCountedOpenSslClientContext extends ReferenceCounted
         try {
             sessionContext = newSessionContext(this, ctx, engineMap, trustCertCollection, trustManagerFactory,
                                                keyCertChain, key, keyPassword, keyManagerFactory, keyStore,
-                                               sessionCacheSize, sessionTimeout, resumptionController);
+                                               sessionCacheSize, sessionTimeout, endpointIdentificationAlgorithm,
+                                               resumptionController, options);
             success = true;
         } finally {
             if (!success) {
@@ -94,7 +95,9 @@ static OpenSslSessionContext newSessionContext(ReferenceCountedOpenSslContext th
                                                    X509Certificate[] keyCertChain, PrivateKey key,
                                                    String keyPassword, KeyManagerFactory keyManagerFactory,
                                                    String keyStore, long sessionCacheSize, long sessionTimeout,
-                                                   ResumptionController resumptionController)
+                                                   String endpointIdentificationAlgorithm,
+                                                   ResumptionController resumptionController,
+                                                   Map.Entry<SslContextOption<?>, Object>... options)
             throws SSLException {
         if (key == null && keyCertChain != null || key != null && keyCertChain == null) {
             throw new IllegalArgumentException(
@@ -155,8 +158,11 @@ static OpenSslSessionContext newSessionContext(ReferenceCountedOpenSslContext th
                             TrustManagerFactory.getDefaultAlgorithm());
                     trustManagerFactory.init((KeyStore) null);
                 }
-                final X509TrustManager manager = chooseTrustManager(
-                        trustManagerFactory.getTrustManagers(), resumptionController);
+                final boolean sanPeerIdentityLookup = isSanPeerIdentityLookupEnabled(
+                        endpointIdentificationAlgorithm, options);
+                final X509TrustManager manager = wrapTrustManagerIfNeeded(
+                        chooseTrustManager(trustManagerFactory.getTrustManagers(), resumptionController),
+                        sanPeerIdentityLookup);
 
                 // IMPORTANT: The callbacks set for verification must be static to prevent memory leak as
                 //            otherwise the context can never be collected. This is because the JNI code holds
@@ -204,6 +210,37 @@ private static void setVerifyCallback(long ctx, OpenSslEngineMap engineMap, X509
         }
     }
 
+    private static boolean isSanPeerIdentityLookupEnabled(String endpointIdentificationAlgorithm,
+                                                          Map.Entry<SslContextOption<?>, Object>[] options) {
+        if (endpointIdentificationAlgorithm == null) {
+            return false;
+        }
+        if (options == null) {
+            return false;
+        }
+        for (Map.Entry<SslContextOption<?>, Object> option : options) {
+            if (option == null) {
+                continue;
+            }
+            if (SanPeerIdentityConfig.SAN_PEER_IDENTITY_LOOKUP.equals(option.getKey())) {
+                return Boolean.TRUE.equals(option.getValue());
+            }
+        }
+        return false;
+    }
+
+    @SuppressJava6Requirement(reason = "Usage guarded by java version check")
+    private static X509TrustManager wrapTrustManagerIfNeeded(X509TrustManager manager,
+                                                             boolean sanPeerIdentityLookup) {
+        if (!sanPeerIdentityLookup) {
+            return manager;
+        }
+        if (useExtendedTrustManager(manager)) {
+            return new SanPeerIdentityTrustManager((X509ExtendedTrustManager) manager);
+        }
+        return manager;
+    }
+
     static final class OpenSslClientSessionContext extends OpenSslSessionContext {
         OpenSslClientSessionContext(ReferenceCountedOpenSslContext context, OpenSslKeyMaterialProvider provider) {
             super(context, provider, SSL.SSL_SESS_CACHE_CLIENT, new OpenSslClientSessionCache(context.engineMap));

handler/src/main/java/io/netty/handler/ssl/SanPeerIdentityConfig.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2026 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package io.netty.handler.ssl;
+
+final class SanPeerIdentityConfig {
+    static final SslContextOption<Boolean> SAN_PEER_IDENTITY_LOOKUP =
+            new SslContextOption<Boolean>("SAN_PEER_IDENTITY_LOOKUP");
+
+    private SanPeerIdentityConfig() {
+    }
+}
+