KNOX-3377: KNOX-3378: KNOX-3379: Support secure LDAP (LDAPS) for the Knox embedded LDAP service (#1305)

Author: smolnar82

.github/workflows/build/conf/topologies/knoxldap.xml

@@ -35,7 +35,7 @@ limitations under the License.
 			</param>
 			<param>
 				<name>main.ldapRealm.contextFactory.url</name>
-				<value>ldap://localhost:33390</value>  <!-- Local Knox LDAP service which is configured to proxy to the demo LDAP as a backend -->
+				<value>ldaps://localhost:33390</value>  <!-- Local Knox LDAP service (LDAPS); configured to proxy to the demo LDAP as a backend -->
 			</param>
 			<param>
 				<name>main.ldapRealm.contextFactory.authenticationMechanism</name>

.github/workflows/build/gateway-site.xml

@@ -153,6 +153,20 @@ limitations under the License.
         <name>gateway.ldap.interceptor.names</name>
         <value>demoldap</value>
     </property>
+    <!-- Expose the embedded Knox LDAP service over LDAPS using the dev keystore that
+         gateway.sh provisions; its password is resolved from the credential store. -->
+    <property>
+        <name>gateway.ldap.ssl.enabled</name>
+        <value>true</value>
+    </property>
+    <property>
+        <name>gateway.ldap.ssl.keystore.path</name>
+        <value>/knox-runtime/conf/ldaps-keystore.p12</value>
+    </property>
+    <property>
+        <name>gateway.ldap.ssl.keystore.password.alias</name>
+        <value>gateway_ldap_ssl_keystore_password</value>
+    </property>
 
     <!-- LDAP Backend specific configuration (proxying to demo ldap) -->
     <property>
@@ -165,7 +179,13 @@ limitations under the License.
     </property>
     <property>
         <name>gateway.ldap.interceptor.demoldap.url</name>
-        <value>ldap://ldap:33389</value>
+        <value>ldaps://ldap:33389</value>
+    </property>
+    <!-- The demo LDAP presents a self-signed dev certificate; trust it rather than
+         wiring a truststore for this dev/CI fixture. -->
+    <property>
+        <name>gateway.ldap.interceptor.demoldap.trustAllCertificates</name>
+        <value>true</value>
     </property>
     <property>
         <name>gateway.ldap.interceptor.demoldap.remoteBaseDn</name>

.github/workflows/build/gateway.sh

@@ -13,7 +13,37 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+set -e
 
+# The embedded Knox LDAP service is exposed over LDAPS (see gateway-site.xml:
+# gateway.ldap.ssl.enabled). Provision a dev-only certificate for it and make the
+# JVM trust that certificate so the knoxldap Shiro realm - which authenticates over
+# ldaps://localhost:33390 via JNDI - can validate it.
+KEYSTORE=/knox-runtime/conf/ldaps-keystore.p12
+KEYSTORE_PASSWORD=knox-ldap-ssl-password
+KEYSTORE_PASSWORD_ALIAS=gateway_ldap_ssl_keystore_password
 
-# Start Knox
-java -jar /knox-runtime/bin/gateway.jar
+# 1) Self-signed dev certificate for the embedded LDAP service's secure transport.
+#    Store and key passwords must match (ApacheDS opens the store and recovers the
+#    key with a single password). PKCS12 matches the JVM default keystore type.
+keytool -genkeypair -alias ldaps -keyalg RSA -keysize 2048 \
+  -dname "CN=localhost" -ext "san=dns:localhost,dns:knox" -validity 3650 \
+  -keystore "$KEYSTORE" -storetype PKCS12 \
+  -storepass "$KEYSTORE_PASSWORD" -keypass "$KEYSTORE_PASSWORD"
+
+# 2) Store the keystore password under the alias the LDAP SSL config resolves.
+/knox-runtime/bin/knoxcli.sh create-alias "$KEYSTORE_PASSWORD_ALIAS" --value "$KEYSTORE_PASSWORD"
+
+# 3) Trust that certificate in the JVM default truststore (cacerts) so the JNDI-based
+#    Shiro LDAP realm accepts it. This is additive - it does not remove the default CAs.
+keytool -exportcert -alias ldaps -rfc \
+  -keystore "$KEYSTORE" -storepass "$KEYSTORE_PASSWORD" -file /tmp/ldaps-cert.pem
+keytool -importcert -noprompt -alias knox-ldaps \
+  -keystore "${JAVA_HOME}/lib/security/cacerts" -storepass changeit \
+  -file /tmp/ldaps-cert.pem
+rm -f /tmp/ldaps-cert.pem
+
+# Start Knox. Endpoint (hostname) identification is disabled for the embedded LDAPS
+# connection - the dev cert is self-signed - but trust is still enforced via cacerts.
+java -Dcom.sun.jndi.ldap.object.disableEndpointIdentification=true \
+  -jar /knox-runtime/bin/gateway.jar

.github/workflows/build/ldap.sh

@@ -13,4 +13,22 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-java -jar /knox-runtime/bin/ldap.jar /knox-runtime/conf
+set -e
+
+# Expose the demo LDAP over a secure (LDAPS) transport so the Knox backend can be
+# configured to proxy to it over TLS. Generate a self-signed, dev-only certificate on
+# each start (the demo LDAP is a throwaway fixture, not a secret). The store and key
+# passwords must match: ApacheDS opens the keystore and recovers the key with a single
+# password. PKCS12 matches the JVM default keystore type ApacheDS loads it with.
+KEYSTORE=/knox-runtime/conf/ldap-keystore.p12
+KEYSTORE_PASSWORD=ldap-keystore-password
+
+keytool -genkeypair -alias ldaps -keyalg RSA -keysize 2048 \
+  -dname "CN=ldap" -ext "san=dns:ldap,dns:localhost" -validity 3650 \
+  -keystore "$KEYSTORE" -storetype PKCS12 \
+  -storepass "$KEYSTORE_PASSWORD" -keypass "$KEYSTORE_PASSWORD"
+
+java -Dldap.ssl.enabled=true \
+  -Dldap.keystore.file="$KEYSTORE" \
+  -Dldap.keystore.password="$KEYSTORE_PASSWORD" \
+  -jar /knox-runtime/bin/ldap.jar /knox-runtime/conf

.github/workflows/tests/test_knox_ldap_proxy_search.py

@@ -18,17 +18,25 @@
 These exercise the LdapProxyBackend.search() path (KNOX-3341): clients can query
 the embedded Knox LDAP service - which proxies to the demo LDAP backend - by
 objectClass, cn and uid (including wildcards), not just by a single uid lookup.
+
+The connection is made over LDAPS: the embedded Knox LDAP service is configured
+with gateway.ldap.ssl.enabled=true, and it in turn proxies to the demo LDAP
+backend over LDAPS as well (gateway.ldap.interceptor.demoldap.url=ldaps://...).
+The gateway presents a self-signed dev certificate in CI, so certificate
+validation is disabled on the client side.
 """
 
 from __future__ import annotations
 
 import os
+import ssl
 import unittest
 from urllib.parse import urlparse
 
 import ldap3
 
-# The embedded Knox LDAP service port (see gateway-site.xml: gateway.ldap.port).
+# The embedded Knox LDAP service secure port (see gateway-site.xml: gateway.ldap.port
+# with gateway.ldap.ssl.enabled=true).
 KNOX_LDAP_PORT = 33390
 
 BASE_DN = "dc=hadoop,dc=apache,dc=org"
@@ -50,7 +58,11 @@ class TestKnoxLdapProxySearch(unittest.TestCase):
     """Verify general search requests are proxied to the demo LDAP backend."""
 
     def setUp(self) -> None:
-        server = ldap3.Server(knox_host(), port=KNOX_LDAP_PORT, get_info=ldap3.NONE)
+        # Self-signed dev certificate in CI: connect over TLS but skip validation.
+        tls = ldap3.Tls(validate=ssl.CERT_NONE)
+        server = ldap3.Server(
+            knox_host(), port=KNOX_LDAP_PORT, use_ssl=True, tls=tls, get_info=ldap3.NONE
+        )
         self.connection = ldap3.Connection(
             server, user=BIND_DN, password=BIND_PASSWORD, auto_bind=True
         )

gateway-demo-ldap/pom.xml

@@ -90,6 +90,13 @@
             <groupId>org.bouncycastle</groupId>
             <artifactId>bcprov-jdk18on</artifactId>
         </dependency>
+        <!-- ApacheDS 2.0.0.AM27's LdapServer.start() references bcpkix (cert utilities)
+             even for a plaintext transport. The legacy jdk15on BouncyCastle it would pull
+             transitively is excluded project-wide, so declare the jdk18on artifact here. -->
+        <dependency>
+            <groupId>org.bouncycastle</groupId>
+            <artifactId>bcpkix-jdk18on</artifactId>
+        </dependency>
     </dependencies>
     <build>
         <plugins>

gateway-demo-ldap/src/main/java/org/apache/knox/gateway/security/ldap/SimpleLdapDirectoryServer.java

@@ -104,6 +104,24 @@ public SimpleLdapDirectoryServer( String rootDn, File usersLdif, Transport... tr
     server = new LdapServer();
     server.setTransports( transports );
     server.setDirectoryService( service );
+
+    // Optionally expose a secure (LDAPS) transport. Controlled by system properties so the
+    // demo server can be launched in either plaintext or secure mode without code changes.
+    if ( Boolean.parseBoolean( System.getProperty( "ldap.ssl.enabled", "false" ) ) ) {
+      for ( Transport transport : transports ) {
+        if ( transport instanceof TcpTransport ) {
+          ( (TcpTransport) transport ).setEnableSSL( true );
+        }
+      }
+      String keystoreFile = System.getProperty( "ldap.keystore.file" );
+      if ( keystoreFile != null && !keystoreFile.isEmpty() ) {
+        server.setKeystoreFile( keystoreFile );
+      }
+      String keystorePassword = System.getProperty( "ldap.keystore.password" );
+      if ( keystorePassword != null ) {
+        server.setCertificatePassword( keystorePassword );
+      }
+    }
   }
 
   private static void enabledPosixSchema( DirectoryService service ) throws LdapException {

gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java

@@ -1863,6 +1863,27 @@ public String getLdapRolesLookupFilePath() {
     return get(LDAP_ROLES_LOOKUP_FILE_PATH);
   }
 
+  @Override
+  public boolean isLDAPSSLEnabled() {
+    return Boolean.parseBoolean(get(LDAP_SSL_ENABLED, "false"));
+  }
+
+  @Override
+  public String getLDAPSSLKeystorePath() {
+    return get(LDAP_SSL_KEYSTORE_PATH, null);
+  }
+
+  @Override
+  public String getLDAPSSLKeystorePasswordAlias() {
+    return get(LDAP_SSL_KEYSTORE_PASSWORD_ALIAS, null);
+  }
+
+  @Override
+  public List<String> getLDAPSSLEnabledCipherSuites() {
+    final List<String> cipherSuites = splitConfigValueToList(LDAP_SSL_ENABLED_CIPHER_SUITES);
+    return cipherSuites == null ? Collections.emptyList() : cipherSuites;
+  }
+
   @Override
   public boolean getGroupUIServicesOnHomepage() {
     return getBoolean(KNOX_HOMEPAGE_GROUP_UI_SERVICES, DEFAULT_GROUP_UI_SERVICES);

gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/KnoxLDAPServerManager.java

@@ -79,6 +79,11 @@ public class KnoxLDAPServerManager {
     private int port;
     private String baseDn;
     private String bindUser;
+    // Secure (LDAPS) transport configuration
+    private boolean sslEnabled;
+    private String sslKeystorePath;
+    private String sslKeystorePasswordAlias;
+    private List<String> sslEnabledCipherSuites;
     // Collection of DNs for the proxied backend LDAP servers
     private Set<String> baseDns;
 
@@ -108,6 +113,17 @@ public void initialize(GatewayConfig config) throws Exception {
         this.baseDn = config.getLDAPBaseDN();
         this.bindUser = config.getLDAPBindUser();
 
+        // Secure (LDAPS) transport configuration. When enabled but no dedicated keystore is
+        // configured, fall back to the gateway identity keystore so the embedded server can
+        // reuse the gateway's own TLS material out of the box.
+        this.sslEnabled = config.isLDAPSSLEnabled();
+        if (sslEnabled) {
+            final String configuredKeystore = config.getLDAPSSLKeystorePath();
+            this.sslKeystorePath = StringUtils.isNotBlank(configuredKeystore) ? configuredKeystore : config.getIdentityKeystorePath();
+            this.sslKeystorePasswordAlias = config.getLDAPSSLKeystorePasswordAlias();
+            this.sslEnabledCipherSuites = config.getLDAPSSLEnabledCipherSuites();
+        }
+
         createInterceptors(config);
 
         // Clean up previous run if it didn't shut down cleanly
@@ -215,7 +231,11 @@ public void start() throws Exception {
 
         // Create LDAP server on configured port
         ldapServer = new LdapServer();
-        ldapServer.setTransports(new TcpTransport(port));
+        final TcpTransport transport = new TcpTransport(port);
+        if (sslEnabled) {
+            configureSsl(transport);
+        }
+        ldapServer.setTransports(transport);
         ldapServer.setDirectoryService(directoryService);
 
         ldapServer.start();
@@ -280,6 +300,56 @@ private int fetchAuthenticationInterceptorIndex(final List<Interceptor> dsInterc
                 .orElse(-1);
     }
 
+    /**
+     * Enable the secure (LDAPS) transport on the embedded server. The keystore holding the
+     * server certificate and its password are resolved from the gateway configuration and
+     * credential store; the password defaults to the gateway identity keystore password when
+     * no dedicated alias is configured. The keystore must be in the JVM default format
+     * (see {@code java.security.KeyStore#getDefaultType()}), which is what ApacheDS uses to
+     * load it.
+     */
+    private void configureSsl(final TcpTransport transport) throws Exception {
+        if (StringUtils.isBlank(sslKeystorePath)) {
+            final String reason = "no keystore configured; set " + GatewayConfig.LDAP_SSL_KEYSTORE_PATH
+                    + " or configure the gateway identity keystore";
+            LOG.ldapSslConfigInvalid(reason);
+            throw new IllegalStateException("Cannot enable LDAPS: " + reason);
+        }
+
+        final File keystore = new File(sslKeystorePath);
+        if (!keystore.exists()) {
+            final String reason = "keystore file not found: " + keystore.getAbsolutePath();
+            LOG.ldapSslConfigInvalid(reason);
+            throw new IllegalStateException("Cannot enable LDAPS: " + reason);
+        }
+
+        transport.setEnableSSL(true);
+        ldapServer.setKeystoreFile(keystore.getAbsolutePath());
+
+        final char[] passwordChars = resolveSslKeystorePassword();
+        if (passwordChars != null) {
+            ldapServer.setCertificatePassword(new String(passwordChars));
+        }
+
+        if (sslEnabledCipherSuites != null && !sslEnabledCipherSuites.isEmpty()) {
+            ldapServer.setEnabledCipherSuites(new ArrayList<>(sslEnabledCipherSuites));
+        }
+
+        LOG.ldapSslEnabled(keystore.getAbsolutePath());
+    }
+
+    /**
+     * Resolve the password protecting the LDAPS keystore from the gateway credential store.
+     * Uses the configured alias when set, otherwise falls back to the gateway identity
+     * keystore password (matching the keystore-path fallback in {@link #initialize(GatewayConfig)}).
+     */
+    private char[] resolveSslKeystorePassword() throws Exception {
+        if (StringUtils.isNotBlank(sslKeystorePasswordAlias)) {
+            return aliasService.getPasswordFromAliasForGateway(sslKeystorePasswordAlias);
+        }
+        return aliasService.getGatewayIdentityKeystorePassword();
+    }
+
     /**
      * Stop the LDAP server
      */

gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/LdapMessages.java

@@ -33,6 +33,14 @@ public interface LdapMessages {
             text = "LDAP service started successfully on port {0}")
     void ldapServiceStarted(int port);
 
+    @Message(level = MessageLevel.INFO,
+            text = "Enabling secure (LDAPS) transport for LDAP service using keystore: {0}")
+    void ldapSslEnabled(String keystorePath);
+
+    @Message(level = MessageLevel.ERROR,
+            text = "Cannot enable secure (LDAPS) transport: {0}")
+    void ldapSslConfigInvalid(String reason);
+
     @Message(level = MessageLevel.INFO,
             text = "Anonymous access disabled; clients must bind as: {0}")
     void ldapBindUserConfigured(String bindDn);