ssl: narrow TLS 1.2 RSA-PSS handling and add interop coverage

Signed-off-by: Viktor Sokolovskiy <maokaman@gmail.com>

Author: Maokaman1

ChangeLog.d/fix-tls12-rsa-pss-sigalgs.txt

@@ -1,4 +1,4 @@
 Bugfix
    * Fix a TLS 1.2 regression that caused clients to reject valid
-     ServerKeyExchange signatures using RSA-PSS signature scheme values.
+     ServerKeyExchange signatures using RSA-PSS signature algorithms.
      Fixes #10668.

library/ssl_misc.h

@@ -2428,28 +2428,6 @@ static inline int mbedtls_ssl_get_pk_sigalg_and_md_alg_from_sig_alg(
 static inline int mbedtls_ssl_tls12_sig_alg_is_supported(
     const uint16_t sig_alg)
 {
-#if defined(PSA_WANT_ALG_RSA_PSS)
-    /* RFC 8446 Section 4.2.3 requires implementations that support RSA-PSS in
-     * TLS 1.3 to also accept rsa_pss_rsae_* in TLS 1.2.
-     */
-    switch (sig_alg) {
-#if defined(PSA_WANT_ALG_SHA_256)
-        case MBEDTLS_TLS1_3_SIG_RSA_PSS_RSAE_SHA256:
-            return 1;
-#endif
-#if defined(PSA_WANT_ALG_SHA_384)
-        case MBEDTLS_TLS1_3_SIG_RSA_PSS_RSAE_SHA384:
-            return 1;
-#endif
-#if defined(PSA_WANT_ALG_SHA_512)
-        case MBEDTLS_TLS1_3_SIG_RSA_PSS_RSAE_SHA512:
-            return 1;
-#endif
-        default:
-            break;
-    }
-#endif /* PSA_WANT_ALG_RSA_PSS */
-
     /* High byte is hash */
     unsigned char hash = MBEDTLS_BYTE_1(sig_alg);
     unsigned char sig = MBEDTLS_BYTE_0(sig_alg);

library/ssl_tls12_client.c

@@ -12,6 +12,7 @@
 #include "mbedtls/platform.h"
 
 #include "mbedtls/ssl.h"
+#include "ssl_debug_helpers.h"
 #include "ssl_client.h"
 #include "debug_internal.h"
 #include "mbedtls/error.h"
@@ -1742,32 +1743,73 @@ static int ssl_parse_signature_algorithm(mbedtls_ssl_context *ssl,
 {
     if (mbedtls_ssl_get_pk_sigalg_and_md_alg_from_sig_alg(sig_alg, pk_alg, md_alg) != 0) {
         MBEDTLS_SSL_DEBUG_MSG(1,
-                              ("Server used unsupported value in SigAlg extension 0x%04x",
-                               sig_alg));
+                              ("Server used unsupported %s signature algorithm",
+                               mbedtls_ssl_sig_alg_to_str(sig_alg)));
         return MBEDTLS_SSL_ALERT_MSG_ILLEGAL_PARAMETER;
     }
 
     /*
-     * mbedtls_ssl_get_pk_sigalg_and_md_alg_from_sig_alg() understands sig_alg code points across
-     * TLS versions. Make sure that the received sig_alg extension is valid in TLS 1.2.
+     * mbedtls_ssl_get_pk_sigalg_and_md_alg_from_sig_alg() understands
+     * signature algorithm code points from both TLS 1.2 and TLS 1.3. Make sure
+     * that the selected signature algorithm is acceptable when TLS 1.2 is
+     * negotiated.
+     *
+     * In TLS 1.2, RSA-PSS signature algorithms (rsa_pss_rsae_*) are not
+     * defined by RFC 5246. However, RFC 8446 Section 4.2.3 requires that
+     * implementations which advertise support for RSASSA-PSS must be
+     * prepared to accept such signatures even when TLS 1.2 is negotiated,
+     * provided they were offered in the signature_algorithms extension.
+     *
+     * Therefore, we allow rsa_pss_rsae_* here if:
+     *  - the implementation supports them, and
+     *  - they were offered in the signature_algorithms extension (checked by
+     *    `mbedtls_ssl_sig_alg_is_offered()` below).
+     *
+     * If we were to add full support for rsa_pss_rsae_* signature algorithms
+     * in TLS 1.2 (not defined by RFC 5246; RFC 8446 requires implementations
+     * that advertise RSASSA-PSS to accept such signatures even when TLS 1.2
+     * is negotiated; in practice, several TLS implementations also offer and
+     * use these algorithms in TLS 1.2-only configurations), we should then
+     * integrate RSA-PSS into the TLS 1.2 signature algorithm support logic
+     * (`mbedtls_ssl_tls12_sig_alg_is_supported()`) instead of handling it as a
+     * special case here.
      */
     if (!mbedtls_ssl_sig_alg_is_supported(ssl, sig_alg)) {
-        MBEDTLS_SSL_DEBUG_MSG(1,
-                              ("Server used unsupported value in SigAlg extension 0x%04x",
-                               sig_alg));
-        return MBEDTLS_SSL_ALERT_MSG_ILLEGAL_PARAMETER;
+        switch (sig_alg) {
+#if defined(PSA_WANT_ALG_RSA_PSS)
+#if defined(PSA_WANT_ALG_SHA_256)
+            case MBEDTLS_TLS1_3_SIG_RSA_PSS_RSAE_SHA256:
+                break;
+#endif
+#if defined(PSA_WANT_ALG_SHA_384)
+            case MBEDTLS_TLS1_3_SIG_RSA_PSS_RSAE_SHA384:
+                break;
+#endif
+#if defined(PSA_WANT_ALG_SHA_512)
+            case MBEDTLS_TLS1_3_SIG_RSA_PSS_RSAE_SHA512:
+                break;
+#endif
+#endif /* PSA_WANT_ALG_RSA_PSS */
+            default:
+                MBEDTLS_SSL_DEBUG_MSG(1,
+                                      ("Server used unsupported %s signature algorithm",
+                                       mbedtls_ssl_sig_alg_to_str(sig_alg)));
+                return MBEDTLS_SSL_ALERT_MSG_ILLEGAL_PARAMETER;
+        }
     }
 
     /*
      * Check if the signature algorithm is acceptable
      */
     if (!mbedtls_ssl_sig_alg_is_offered(ssl, sig_alg)) {
-        MBEDTLS_SSL_DEBUG_MSG(1, ("Server used SigAlg value 0x%04x that was not offered", sig_alg));
+        MBEDTLS_SSL_DEBUG_MSG(1,
+                              ("Server used the signature algorithm %s that was not offered",
+                               mbedtls_ssl_sig_alg_to_str(sig_alg)));
         return MBEDTLS_SSL_ALERT_MSG_ILLEGAL_PARAMETER;
     }
 
-    MBEDTLS_SSL_DEBUG_MSG(2, ("Server used SignatureAlgorithm %d", sig_alg & 0x00FF));
-    MBEDTLS_SSL_DEBUG_MSG(2, ("Server used HashAlgorithm %d", sig_alg >> 8));
+    MBEDTLS_SSL_DEBUG_MSG(2, ("Server used the signature algorithm %s",
+                              mbedtls_ssl_sig_alg_to_str(sig_alg)));
 
     return 0;
 }

tests/ssl-opt.sh

@@ -13938,39 +13938,55 @@ run_test    "TLS 1.2: Check rsa_pss_rsae compatibility issue, m->O" \
             -c "Protocol is TLSv1.2" \
             -c "HTTP/1.0 200 [Oo][Kk]"
 
+requires_gnutls_tls1_3
+requires_config_enabled MBEDTLS_SSL_PROTO_TLS1_2
+requires_config_enabled MBEDTLS_DEBUG_C
+requires_config_enabled MBEDTLS_SSL_CLI_C
+run_test    "TLS 1.2: Check rsa_pss_rsae compatibility issue, m->G" \
+            "$G_NEXT_SRV_NO_CERT --x509certfile $DATA_FILES_PATH/server2-sha256.crt --x509keyfile $DATA_FILES_PATH/server2.key
+                    -d 4
+                    --priority=NORMAL:-VERS-ALL:+VERS-TLS1.2" \
+            "$P_CLI debug_level=4 crt_file=$DATA_FILES_PATH/server2-sha256.crt key_file=$DATA_FILES_PATH/server2.key
+                    sig_algs=rsa_pss_rsae_sha512,rsa_pkcs1_sha512
+                    min_version=tls12 max_version=tls13 " \
+            0 \
+            -c "Protocol is TLSv1.2" \
+            -c "HTTP/1.0 200 [Oo][Kk]"
+
 requires_openssl_tls1_3_with_compatible_ephemeral
 requires_config_enabled MBEDTLS_SSL_PROTO_TLS1_2
+requires_config_enabled MBEDTLS_SSL_PROTO_TLS1_3
 requires_config_enabled MBEDTLS_DEBUG_C
 requires_config_enabled MBEDTLS_SSL_CLI_C
 requires_config_enabled PSA_WANT_ALG_RSA_PSS
 requires_config_enabled PSA_WANT_ALG_SHA_256
-run_test    "TLS 1.2: OpenSSL chooses rsa_pss_rsae_sha256 for SKE, m->O" \
+run_test    "TLS 1.2: Server forces TLS 1.2 and rsa_pss_rsae_sha256, m->O" \
             "$O_NEXT_SRV_NO_CERT -cert $DATA_FILES_PATH/server2-sha256.crt -key $DATA_FILES_PATH/server2.key
-                                 -msg -tls1_2
-                                 -sigalgs rsa_pss_rsae_sha256 " \
-            "$P_CLI debug_level=4 auth_mode=none ca_file=none ca_path=none crt_file=none key_file=none
-                    sig_algs=rsa_pss_rsae_sha256
-                    min_version=tls12 max_version=tls12 " \
+                                 -tls1_2 -sigalgs rsa_pss_rsae_sha256 " \
+            "$P_CLI debug_level=3" \
             0 \
+            -c "sent signature scheme \\[804\\] rsa_pss_rsae_sha256" \
             -c "Perform .* computation of digest of ServerKeyExchange" \
-            -c "got signature scheme \\[804\\] rsa_pss_rsae_sha256" \
-            -c "Ciphersuite is TLS-ECDHE-RSA" \
+            -c "Server used the signature algorithm rsa_pss_rsae_sha256" \
             -c "Protocol is TLSv1.2" \
             -c "HTTP/1.0 200 [Oo][Kk]"
 
-
 requires_gnutls_tls1_3
 requires_config_enabled MBEDTLS_SSL_PROTO_TLS1_2
+requires_config_enabled MBEDTLS_SSL_PROTO_TLS1_3
 requires_config_enabled MBEDTLS_DEBUG_C
 requires_config_enabled MBEDTLS_SSL_CLI_C
-run_test    "TLS 1.2: Check rsa_pss_rsae compatibility issue, m->G" \
+requires_config_enabled PSA_WANT_ALG_RSA_PSS
+requires_config_enabled PSA_WANT_ALG_SHA_256
+run_test    "TLS 1.2: Server forces TLS 1.2 and rsa_pss_rsae_sha256, m->G" \
             "$G_NEXT_SRV_NO_CERT --x509certfile $DATA_FILES_PATH/server2-sha256.crt --x509keyfile $DATA_FILES_PATH/server2.key
-                    -d 4
-                    --priority=NORMAL:-VERS-ALL:+VERS-TLS1.2" \
-            "$P_CLI debug_level=4 crt_file=$DATA_FILES_PATH/server2-sha256.crt key_file=$DATA_FILES_PATH/server2.key
-                    sig_algs=rsa_pss_rsae_sha512,rsa_pkcs1_sha512
-                    min_version=tls12 max_version=tls13 " \
+                    --disable-client-cert
+                    --priority=NORMAL:-VERS-ALL:+VERS-TLS1.2:-SIGN-ALL:+SIGN-RSA-PSS-RSAE-SHA256" \
+            "$P_CLI debug_level=3" \
             0 \
+            -c "sent signature scheme \\[804\\] rsa_pss_rsae_sha256" \
+            -c "Perform .* computation of digest of ServerKeyExchange" \
+            -c "Server used the signature algorithm rsa_pss_rsae_sha256" \
             -c "Protocol is TLSv1.2" \
             -c "HTTP/1.0 200 [Oo][Kk]"
 

tests/suites/test_suite_ssl.data

@@ -3549,27 +3549,3 @@ send_invalid_sig_alg:MBEDTLS_SSL_SIG_ECDSA:MBEDTLS_SSL_HASH_SHA512:0
 Negative Test: Server using sig_alg not offered by the client - ECDSA with SHA512
 depends_on:MBEDTLS_CAN_HANDLE_ECDSA_TEST_KEY:MBEDTLS_CAN_HANDLE_ECDSA_CLIENT_TEST_KEY:PSA_WANT_ALG_SHA_512
 send_invalid_sig_alg:MBEDTLS_SSL_SIG_ECDSA:MBEDTLS_SSL_HASH_SHA512:MBEDTLS_ERR_SSL_ILLEGAL_PARAMETER
-
-TLS 1.2 accepts rsa_pss_rsae_sha256 in signature_algorithm
-depends_on:PSA_WANT_ALG_RSA_PSS:PSA_WANT_ALG_SHA_256
-ssl_tls12_sig_alg_supported:MBEDTLS_TLS1_3_SIG_RSA_PSS_RSAE_SHA256:1
-
-TLS 1.2 rejects rsa_pss_rsae_sha256 in signature_algorithm when RSA-PSS is disabled
-depends_on:!PSA_WANT_ALG_RSA_PSS:PSA_WANT_ALG_SHA_256
-ssl_tls12_sig_alg_supported:MBEDTLS_TLS1_3_SIG_RSA_PSS_RSAE_SHA256:0
-
-TLS 1.2 accepts rsa_pss_rsae_sha384 in signature_algorithm
-depends_on:PSA_WANT_ALG_RSA_PSS:PSA_WANT_ALG_SHA_384
-ssl_tls12_sig_alg_supported:MBEDTLS_TLS1_3_SIG_RSA_PSS_RSAE_SHA384:1
-
-TLS 1.2 rejects rsa_pss_rsae_sha384 in signature_algorithm when RSA-PSS is disabled
-depends_on:!PSA_WANT_ALG_RSA_PSS:PSA_WANT_ALG_SHA_384
-ssl_tls12_sig_alg_supported:MBEDTLS_TLS1_3_SIG_RSA_PSS_RSAE_SHA384:0
-
-TLS 1.2 accepts rsa_pss_rsae_sha512 in signature_algorithm
-depends_on:PSA_WANT_ALG_RSA_PSS:PSA_WANT_ALG_SHA_512
-ssl_tls12_sig_alg_supported:MBEDTLS_TLS1_3_SIG_RSA_PSS_RSAE_SHA512:1
-
-TLS 1.2 rejects rsa_pss_rsae_sha512 in signature_algorithm when RSA-PSS is disabled
-depends_on:!PSA_WANT_ALG_RSA_PSS:PSA_WANT_ALG_SHA_512
-ssl_tls12_sig_alg_supported:MBEDTLS_TLS1_3_SIG_RSA_PSS_RSAE_SHA512:0

tests/suites/test_suite_ssl.function

@@ -5853,14 +5853,6 @@ exit:
 }
 /* END_CASE */
 
-/* BEGIN_CASE depends_on:MBEDTLS_SSL_PROTO_TLS1_2:MBEDTLS_SSL_HANDSHAKE_WITH_CERT_ENABLED */
-void ssl_tls12_sig_alg_supported(int sig_alg, int expected)
-{
-    TEST_EQUAL(mbedtls_ssl_tls12_sig_alg_is_supported((uint16_t) sig_alg),
-               expected);
-}
-/* END_CASE */
-
 /* BEGIN_CASE depends_on:MBEDTLS_SSL_KEYING_MATERIAL_EXPORT:MBEDTLS_SSL_HANDSHAKE_WITH_CERT_ENABLED:PSA_WANT_ECC_SECP_R1_384:PSA_WANT_ALG_SHA_256 */
 void ssl_tls_exporter_consistent_result(int proto, int exported_key_length, int use_context)
 {