Fix JMX RMI connector startup failure introduced by CVE-2026-46495 hardening (#651)

Author: vharseko

opendj-server-legacy/src/main/java/org/opends/server/protocols/jmx/RmiConnector.java

@@ -65,25 +65,23 @@ public class RmiConnector
 {
   private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
 
-  static final String JMX_REMOTE_RMI_SERVER_CREDENTIAL_TYPES =
-      "jmx.remote.rmi.server.credential.types";
-
   /**
    * JDK 10+ JMX environment property scoping a JEP 290 deserialization
    * filter to the credentials object passed during {@code newClient()}.
    * Using the credentials-scoped filter (instead of the connector-wide
    * {@code jmx.remote.rmi.server.serial.filter.pattern}) avoids breaking
    * legitimate JMX traffic such as MBean invocations and notifications,
    * which may legitimately carry non-String types.
+   * <p>
+   * Note: this property is mutually exclusive with
+   * {@code jmx.remote.rmi.server.credential.types}; specifying both makes
+   * {@code RMIJRMPServerImpl} throw an {@link IllegalArgumentException} and
+   * prevents the connector from starting. The filter pattern is preferred
+   * because it additionally constrains array length and nesting depth.
    */
   static final String JMX_REMOTE_RMI_SERVER_CREDENTIALS_FILTER_PATTERN =
       "jmx.remote.rmi.server.credentials.filter.pattern";
 
-  private static final String[] JMX_CREDENTIAL_TYPES =
-  {
-    String.class.getName(),
-    String[].class.getName()
-  };
 
   private static final String JMX_CREDENTIAL_SERIAL_FILTER =
       "maxdepth=3;maxarray=2;java.lang.String;!*";
@@ -392,11 +390,13 @@ private void startConnectorNoClientCertificate() throws Exception
 
   static void configureJmxDeserializationProtection(Map<String, Object> env)
   {
-    env.put(JMX_REMOTE_RMI_SERVER_CREDENTIAL_TYPES,
-        JMX_CREDENTIAL_TYPES.clone());
     // Scope the JEP 290 deserialization filter to the credentials object
     // only, so legitimate JMX RMI traffic (MBean operations, notifications,
     // etc.) is not affected by the restrictive allowlist.
+    //
+    // Do NOT also set "jmx.remote.rmi.server.credential.types": the JDK
+    // rejects an environment that defines both properties, which would
+    // prevent the RMI connector from starting.
     env.put(JMX_REMOTE_RMI_SERVER_CREDENTIALS_FILTER_PATTERN,
         JMX_CREDENTIAL_SERIAL_FILTER);
   }

opendj-server-legacy/src/test/java/org/opends/server/protocols/jmx/RmiAuthenticatorTest.java

@@ -67,29 +67,15 @@ public void configuresCredentialDeserializationProtection()
     Map<String, Object> env = new HashMap<>();
     RmiConnector.configureJmxDeserializationProtection(env);
 
-    assertEquals(env.get(RmiConnector.JMX_REMOTE_RMI_SERVER_CREDENTIAL_TYPES),
-        new String[] { String.class.getName(), String[].class.getName() });
     assertEquals(env.get(RmiConnector.JMX_REMOTE_RMI_SERVER_CREDENTIALS_FILTER_PATTERN),
         "maxdepth=3;maxarray=2;java.lang.String;!*");
     // The connector-wide filter must NOT be set, so legitimate JMX traffic
     // (MBean operations, notifications) is not affected by the allowlist.
     assertNull(env.get("jmx.remote.rmi.server.serial.filter.pattern"));
-  }
-
-  /** Verifies that each environment receives its own credential type array. */
-  @Test
-  public void credentialTypesAreDefensivelyCopied()
-  {
-    Map<String, Object> env = new HashMap<>();
-    RmiConnector.configureJmxDeserializationProtection(env);
-    String[] credentialTypes =
-        (String[]) env.get(RmiConnector.JMX_REMOTE_RMI_SERVER_CREDENTIAL_TYPES);
-    credentialTypes[0] = Date.class.getName();
-
-    Map<String, Object> env2 = new HashMap<>();
-    RmiConnector.configureJmxDeserializationProtection(env2);
-    assertEquals(((String[]) env2.get(RmiConnector.JMX_REMOTE_RMI_SERVER_CREDENTIAL_TYPES))[0],
-        String.class.getName());
+    // "jmx.remote.rmi.server.credential.types" is mutually exclusive with the
+    // credentials filter pattern: setting both prevents the connector from
+    // starting, so only the filter pattern must be configured.
+    assertNull(env.get("jmx.remote.rmi.server.credential.types"));
   }
 
   /** Verifies the configured filter allows only the expected credential payload. */