#858 Fix: Legacy authentication breaks non-ASCII characters in passwords

Author: mrotteveel

src/docs/asciidoc/release_notes.adoc

@@ -33,6 +33,17 @@ Changes per Jaybird 6 release.
 See also <<whats-new-in-jaybird-6>>.
 For known issues, consult <<known-issues>>.
 
+=== Jaybird 6.0.6
+
+The following was fixed or changed since Jaybird 6.0.6:
+
+* Fixed: Legacy authentication fails with non-ASCII characters in password (https://github.com/FirebirdSQL/jaybird/issues/858[#858])
++
+The hash that is sent instead of the actual password -- for authentication plugin `Legacy_Auth` or on Firebird 2.5 or older -- was incorrectly generated based on the `char` (UTF-16) value.
+We now use UTF-8 bytes instead.
++
+See also <<compat-legacy-auth>>.
+
 === Jaybird 6.0.5
 
 The following was fixed or changed since Jaybird 6.0.4:
@@ -2357,3 +2368,15 @@ Be aware that this dependency does not support `embedded`.
 For more information about this library, see https://github.com/mrotteveel/jaybird-fbclient[^].
 
 In the future we may provide JARs with the embedded libraries of a specific Firebird version.
+
+[#compat-legacy-auth]
+=== Legacy authentication in Jaybird 6.0.6 and higher
+
+Since Jaybird 6.0.6 (and Jaybird 5.0.13), legacy authentication -- Firebird 2.5 or older, or using authentication plugin `Legacy_Auth` -- uses UTF-8 to derive the bytes for calculating the hash of the password to send to the server.
+It is possible UTF-8 is not the right character set on Firebird 2.5 and older.
+This possibly also applies for Firebird 3.0 and higher, if the password was set using connection encoding NONE on (Windows) systems where windows-1252 or similar is the default encoding.
+
+You can use connection property `legacyAuthCharset` to specify a different Java character set name (for pure Java connections only).
+
+If authentication against Firebird 2.5 or older starts to fail for you, then likely you need to specify `legacyAuthCharset=iso-8859-1`.
+Resetting your password, using https://firebirdsql.org/file/documentation/chunk/en/refdocs/fblangref25/fblangref25-security.html#fblangref25-security-auth-alter-user[`ALTER USER`] with connection encoding UTF8, might also resolve such failures.

src/main/org/firebirdsql/ds/AbstractConnectionPropertiesDataSource.java

@@ -456,6 +456,16 @@ public void setSocketFactory(String socketFactory) {
         FirebirdConnectionProperties.super.setSocketFactory(socketFactory);
     }
 
+    @Override
+    public String getLegacyAuthCharset() {
+        return FirebirdConnectionProperties.super.getLegacyAuthCharset();
+    }
+
+    @Override
+    public void setLegacyAuthCharset(String legacyAuthCharset) {
+        FirebirdConnectionProperties.super.setLegacyAuthCharset(legacyAuthCharset);
+    }
+
     @Override
     public boolean isUseCatalogAsPackage() {
         return FirebirdConnectionProperties.super.isUseCatalogAsPackage();

src/main/org/firebirdsql/gds/ng/wire/auth/ClientAuthBlock.java

@@ -80,6 +80,10 @@ public String getPassword() {
         return attachProperties.getPassword();
     }
 
+    public String getLegacyAuthCharset() {
+        return attachProperties.getLegacyAuthCharset();
+    }
+
     public boolean isAuthComplete() {
         return authComplete;
     }

src/main/org/firebirdsql/gds/ng/wire/auth/legacy/LegacyAuthenticationPlugin.java

@@ -44,7 +44,7 @@ public AuthStatus authenticate(ClientAuthBlock clientAuthBlock) {
         if (clientAuthBlock.getLogin() == null || clientAuthBlock.getPassword() == null) {
             return AuthStatus.AUTH_CONTINUE;
         }
-        clientData = LegacyHash.fbCrypt(clientAuthBlock.getPassword());
+        clientData = LegacyHash.fbCrypt(clientAuthBlock.getPassword(), clientAuthBlock.getLegacyAuthCharset());
         return AuthStatus.AUTH_SUCCESS;
     }
 

src/main/org/firebirdsql/gds/ng/wire/auth/legacy/LegacyHash.java

@@ -22,6 +22,11 @@
  */
 package org.firebirdsql.gds.ng.wire.auth.legacy;
 
+import org.jspecify.annotations.Nullable;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+
 /**
  * Implements the one way password hash used by the legacy authentication of Firebird.
  * <p>
@@ -374,21 +379,25 @@ private static void initPerm(long[][] perm, byte[] p) {
     }
 
     /**
-     * Encrypts String into crypt (Unix) code as used by Firebird.
+     * Encrypts {@code key} into UnixCrypt hash as used by Firebird Legacy_Auth.
      *
      * @param key
      *         the key to be encrypted
+     * @param charsetName
+     *         character set to convert {@code key} to bytes
      * @return the encrypted String
+     * @since 6.0.6
      */
-    public static byte[] fbCrypt(final String key) {
+    public static byte[] fbCrypt(@Nullable String key, String charsetName) {
         if (key == null) {
-            return new byte[] { '*' }; // will NOT match under ANY circumstances!
+            // will NOT match under ANY circumstances!
+            return new byte[] { '*' };
         }
-
-        final int keyLen = key.length();
+        byte[] keyBytes = key.getBytes(toCharset(charsetName));
+        final int keyLen = keyBytes.length;
         long keyword = 0L;
         for (int i = 0; i < 8; i++) {
-            keyword = keyword << 8 | ((i < keyLen) ? 2 * key.charAt(i) : 0);
+            keyword = keyword << 8 | ((i < keyLen) ? 2 * (keyBytes[i] & 0xFF) : 0);
         }
 
         long rsltblock = desCipher(desSetKey(keyword));
@@ -404,5 +413,15 @@ public static byte[] fbCrypt(final String key) {
         return cryptResult;
     }
 
+    private static Charset toCharset(String charsetName) {
+        try {
+            return Charset.forName(charsetName);
+        } catch (IllegalArgumentException e) {
+            System.getLogger(LegacyHash.class.getName()).log(System.Logger.Level.WARNING,
+                    () -> "Invalid character set %s, falling back to UTF-8".formatted(charsetName), e);
+            return StandardCharsets.UTF_8;
+        }
+    }
+
 }
 

src/main/org/firebirdsql/gds/ng/wire/version10/V10ParameterConverter.java

@@ -46,7 +46,8 @@ protected void populateAuthenticationProperties(final AbstractConnection<?, ?> c
             pb.addArgument(tagMapping.getUserNameTag(), props.getUser());
         }
         if (props.getPassword() != null) {
-            pb.addArgument(tagMapping.getEncryptedPasswordTag(), LegacyHash.fbCrypt(props.getPassword()));
+            pb.addArgument(tagMapping.getEncryptedPasswordTag(),
+                    LegacyHash.fbCrypt(props.getPassword(), props.getLegacyAuthCharset()));
         }
     }
 

src/main/org/firebirdsql/jaybird/props/AttachmentProperties.java

@@ -27,6 +27,8 @@
 import org.firebirdsql.gds.ng.FbAttachment;
 import org.firebirdsql.gds.ng.WireCrypt;
 
+import java.nio.charset.Charset;
+
 import static org.firebirdsql.jaybird.props.PropertyConstants.DEFAULT_WIRE_COMPRESSION;
 
 /**
@@ -495,4 +497,40 @@ default void setSocketFactory(String socketFactory) {
         setProperty(PropertyNames.socketFactory, socketFactory);
     }
 
+    /**
+     * The Java character set to use when processing a password for Legacy_Auth in the pure Java protocol.
+     * <p>
+     * In general, this should be {@code UTF-8} (the default). Using a different character set may be needed when
+     * connecting to Firebird 2.5 and older when using passwords with non-ASCII characters.
+     * </p>
+     *
+     * @return Java character set name (default {@code UTF-8})
+     * @since 6.0.6
+     */
+    default String getLegacyAuthCharset() {
+        return getProperty(PropertyNames.legacyAuthCharset, PropertyConstants.DEFAULT_LEGACY_AUTH_CHARSET);
+    }
+
+    /**
+     * Sets the Java character set to use when processing a password for Legacy_Auth in the pure Java protocol.
+     * <p>
+     * In general, this should be {@code UTF-8} (the default). Using a different character set may be needed when
+     * connecting to Firebird 2.5 and older when using passwords with non-ASCII characters.
+     * </p>
+     *
+     * @param legacyAuthCharset
+     *         Java character set name (use {@code null} to reset to the default, {@code UTF-8})
+     * @throws java.nio.charset.IllegalCharsetNameException
+     *         if {@code legacyAuthCharset} is an illegal character set name
+     * @throws java.nio.charset.UnsupportedCharsetException
+     *         if {@code legacyAuth} is not supported
+     * @since 6.0.6
+     */
+    default void setLegacyAuthCharset(String legacyAuthCharset) {
+        if (legacyAuthCharset != null) {
+            Charset.forName(legacyAuthCharset);
+        }
+        setProperty(PropertyNames.legacyAuthCharset, legacyAuthCharset);
+    }
+
 }

src/main/org/firebirdsql/jaybird/props/PropertyConstants.java

@@ -84,6 +84,7 @@ public final class PropertyConstants {
     public static final String SESSION_TIME_ZONE_SERVER = "server";
 
     public static final String DEFAULT_AUTH_PLUGINS = "Srp256,Srp";
+    static final String DEFAULT_LEGACY_AUTH_CHARSET = "UTF-8";
 
     private PropertyConstants() {
         // no instances

src/main/org/firebirdsql/jaybird/props/PropertyNames.java

@@ -63,6 +63,7 @@ public final class PropertyNames {
     public static final String enableProtocol = "enableProtocol";
     public static final String parallelWorkers = "parallelWorkers";
     public static final String socketFactory = "socketFactory";
+    public static final String legacyAuthCharset = "legacyAuthCharset";
 
     // database connection
     public static final String sqlDialect = "sqlDialect";

src/test/org/firebirdsql/common/extension/DatabaseUserExtension.java

@@ -66,7 +66,7 @@ public static DatabaseUserExtension withDatabaseUser() {
      *         for errors creating the user
      */
     public void createUser(String username, String password, String plugin) throws SQLException {
-        try (var connection = getConnectionViaDriverManager();
+        try (var connection = getConnectionViaDriverManager("lc_ctype", "UTF8");
              var statement = connection.createStatement()) {
             var createUserSql = new StringBuilder("CREATE USER ").append(statement.enquoteIdentifier(username, false))
                     .append(" PASSWORD ").append(statement.enquoteLiteral(password));
@@ -84,7 +84,7 @@ public void afterEach(ExtensionContext context) {
         if (createdUsers.isEmpty()) {
             return;
         }
-        try (var connection = getConnectionViaDriverManager()) {
+        try (var connection = getConnectionViaDriverManager("lc_ctype", "UTF8")) {
             connection.setAutoCommit(false);
             try (var statement = connection.createStatement()) {
                 for (User user : createdUsers) {