Enforce distinct aes-xts keys in both fips and non-fips modes to match wolfssl master, and update the test accordingly.

Skip this check when the provider is not used, since the default nettle
library accepts the keys to be the same.

Author: gasbytes

wolfssl-gnutls-wrapper/src/cipher.c

@@ -438,7 +438,7 @@ int wolfssl_cipher_setkey(void *_ctx, const void *key, size_t keysize)
     }
 
 #ifdef WOLFSSL_AES_XTS
-    if (ctx->mode == XTS && gnutls_fips140_mode_enabled()) {
+    if (ctx->mode == XTS) {
         /* XTS has two AES keys that are no allowed to be the same. */
         if (XMEMCMP(key, key + exp_key_size / 2, exp_key_size / 2) == 0) {
             WGW_ERROR("XTS keys are the same");

wolfssl-gnutls-wrapper/tests/test_aesxts.c

@@ -1,5 +1,6 @@
 
 #include <gnutls/crypto.h>
+#include <stdlib.h>
 
 #include "test_util.h"
 
@@ -97,15 +98,16 @@ static int test_aesxts(gnutls_cipher_algorithm_t cipher,
     /* Copy plaintext to a non-const buffer for GnuTLS */
     memcpy(plaintext, plaintext_data, sizeof(plaintext_data));
 
-    /* Try bad key - same data for both keys or key too small. */
-    ret = gnutls_cipher_init(&encrypt_handle, cipher, &bad_key, &iv);
-    if (gnutls_fips140_mode_enabled() && ret == 0) {
-        print_gnutls_error("initializing cipher with bad key", ret);
-        return 1;
-    }
-    if (!gnutls_fips140_mode_enabled() && ret != 0) {
-        print_gnutls_error("initializing cipher with bad key", ret);
-        return 1;
+    /* Try bad key - same data for both keys. The wolfSSL provider rejects
+     * identical XTS key halves in all modes; native GnuTLS/Nettle (used when
+     * GNUTLS_NO_PROVIDER=1) accepts them in non-FIPS mode, so only enforce the
+     * rejection when the provider is in use. */
+    if (!getenv("GNUTLS_NO_PROVIDER")) {
+        ret = gnutls_cipher_init(&encrypt_handle, cipher, &bad_key, &iv);
+        if (ret == 0) {
+            print_gnutls_error("initializing cipher with bad key", ret);
+            return 1;
+        }
     }
 
     /********** ENCRYPTION TEST **********/