Ed25519 support in the OpenSSL compatibility layer

wolfCrypt has full Ed25519 (keygen, sign/verify, ASN.1 import/export) and
the TLS 1.3 stack already authenticates with Ed25519 certificates, but the
OpenSSL-compatibility surface was missing the dispatch for several common
operations, so an application driving Ed25519 purely through the OpenSSL
API (EVP_PKEY_keygen, i2d_PUBKEY, X509_sign of an in-memory self-signed
cert, X509_verify, then loading it into an SSL_CTX) could not use it.  Fill
those gaps, each mirroring the adjacent RSA/ECC/X25519 case:

- EVP_PKEY_keygen (wolfcrypt/src/evp.c): generate an Ed25519 key and cache
  the PKCS#8 PrivateKeyInfo DER, like the EC/RSA cases; if the DER cannot be
  cached the freshly generated key is freed so a failed keygen never leaves
  a partial key behind.
- wolfSSL_i2d_PublicKey / i2d_PUBKEY (wolfcrypt/src/evp_pk.c): encode an
  Ed25519 SubjectPublicKeyInfo.  i2d_PublicKey intentionally emits the full
  SPKI here (rather than the bare key) because i2d_PUBKEY aliases it and the
  PEM_write_PUBKEY / d2i_PUBKEY round-trip relies on it.
- pkcs8_encode (src/pk.c): return the already-PKCS#8 cached DER for Ed25519
  (as the DH case does) after checking it is present, so
  i2d_PKCS8_PRIV_KEY_INFO works and a public-only key (e.g. one obtained
  from X509_get_pubkey, which caches only the raw public key) is rejected
  rather than emitted as a bogus private key.
- d2iTryEd25519Key (wolfcrypt/src/evp_pk.c): derive the public key after
  decoding a PKCS#8 private key (v1 carries only the seed) so the decoded
  EVP_PKEY is complete.
- d2i_AutoPrivateKey (wolfcrypt/src/evp_pk.c): detect Ed25519 by its
  algorithm id and decode the full PKCS#8 (its inner key is an OCTET
  STRING, which the RSA/ECC sequence-counting heuristic cannot classify).
- X509_sign / X509_resign_cert / sigTypeFromPKEY / wolfssl_x509_make_der
  (src/x509.c): sign a certificate with an Ed25519 key (NULL digest),
  resolving the signature type as CTC_ED25519 and building the
  SubjectPublicKey from an ed25519_key.
- X509_verify (src/x509.c): verify a certificate's signature with an
  Ed25519 EVP_PKEY, exporting the raw public key for the check (the cached
  DER is a PKCS#8 private blob, not the public key the verify path wants).
- X509_set_pubkey / X509_get_pubkey (src/x509.c): set and retrieve an
  Ed25519 public key, keeping the X.509 public-key buffer as the raw key
  bytes to match how DecodeCert/StoreKey store it.

Ed25519 object lifetime goes through the existing wolfSSL_ED25519_new /
wolfSSL_ED25519_free helpers.  Their guards (wolfssl/openssl/ed25519.h,
src/pk.c) and the ssl.c include of openssl/ed25519.h are widened to also
cover OPENSSL_EXTRA_X509_SMALL, since the X509 paths above build in that
configuration too.  The individual operations are gated on
HAVE_ED25519_KEY_IMPORT / HAVE_ED25519_KEY_EXPORT / HAVE_ED25519_MAKE_KEY
so a build that disables one of those sub-features still compiles.

tests/api.c gains test_wolfSSL_EVP_PKEY_ED25519_openssl, which exercises
the whole sequence: keygen, i2d_PUBKEY, the raw-key constructors
(EVP_PKEY_new_raw_public_key / EVP_PKEY_new_raw_private_key), the PKCS#8
round-trip (EVP_PKEY2PKCS8, i2d_PKCS8_PKEY, d2i_AutoPrivateKey) including a
check that the decoded key re-exports the same SPKI, a negative check that
PKCS#8 export of a public-only key fails cleanly, a self-signed X509_sign
with a NULL digest followed by X509_verify of the result, an X509_get_pubkey
round-trip, and loading the key+cert into an SSL_CTX.

Requires --enable-ed25519 --enable-certgen (key import/export and key
generation are on by default with --enable-ed25519).

Signed-off-by: Antonio Quartulli <antonio@mandelbit.com>

Author: ordex

src/pk.c

@@ -5518,8 +5518,8 @@ int wolfSSL_ED25519_verify(const unsigned char *msg, unsigned int msgSz,
 
 #endif /* OPENSSL_EXTRA && HAVE_ED25519 */
 
-#if (defined(OPENSSL_EXTRA) || defined(WOLFSSL_WPAS_SMALL)) && \
-    defined(HAVE_ED25519)
+#if (defined(OPENSSL_EXTRA) || defined(WOLFSSL_WPAS_SMALL) || \
+    defined(OPENSSL_EXTRA_X509_SMALL)) && defined(HAVE_ED25519)
 /* Allocate and initialize a new ed25519_key.
  *
  * @param [in] heap   Heap hint for memory allocation.
@@ -5570,7 +5570,8 @@ void wolfSSL_ED25519_free(ed25519_key* key)
     #endif
     }
 }
-#endif /* (OPENSSL_EXTRA || WOLFSSL_WPAS_SMALL) && HAVE_ED25519 */
+#endif /* (OPENSSL_EXTRA || WOLFSSL_WPAS_SMALL || OPENSSL_EXTRA_X509_SMALL) &&
+        * HAVE_ED25519 */
 
 /*******************************************************************************
  * END OF ED25519 API
@@ -7373,6 +7374,27 @@ int pkcs8_encode(WOLFSSL_EVP_PKEY* pkey, byte* key, word32* keySz)
         curveOid = NULL;
         oidSz = 0;
     }
+#endif
+#if defined(HAVE_ED25519)
+    else if (pkey->type == WC_EVP_PKEY_ED25519) {
+        /* The cached DER is already a PKCS#8 PrivateKeyInfo (set when the
+         * key was generated or decoded), so return it as-is (same as the
+         * DH special case above). */
+        if (keySz == NULL)
+            return BAD_FUNC_ARG;
+        /* The cached DER must be present; e.g. an Ed25519 EVP_PKEY built
+         * from wolfSSL_X509_get_pubkey() holds only a raw public key, not a
+         * PKCS#8 private key, so reject rather than emit bogus output. */
+        if (pkey->pkey.ptr == NULL || pkey->pkey_sz <= 0)
+            return BAD_FUNC_ARG;
+
+        *keySz = (word32)pkey->pkey_sz;
+        if (key == NULL)
+            return LENGTH_ONLY_E;
+
+        XMEMCPY(key, pkey->pkey.ptr, (size_t)pkey->pkey_sz);
+        return pkey->pkey_sz;
+    }
 #endif
     else {
         ret = NOT_COMPILED_IN;

src/ssl.c

@@ -63,6 +63,9 @@
     || defined(OPENSSL_EXTRA_X509_SMALL)                    \
     || defined(HAVE_WEBSERVER) || defined(WOLFSSL_KEY_GEN))
     #include <wolfssl/openssl/evp.h>
+    /* ed25519 compat helpers (wolfSSL_ED25519_new/free) are used by the X509
+     * Ed25519 paths, which also build under OPENSSL_EXTRA_X509_SMALL. */
+    #include <wolfssl/openssl/ed25519.h>
     /* openssl headers end, wolfssl internal headers next */
 #endif
 

src/x509.c

@@ -6481,6 +6481,11 @@ WOLFSSL_EVP_PKEY* wolfSSL_X509_get_pubkey(WOLFSSL_X509* x509)
                      x509->pubKeyOID == ML_DSA_87k) {
                 key->type = WC_EVP_PKEY_DILITHIUM;
             }
+        #endif
+        #if defined(HAVE_ED25519) && defined(HAVE_ED25519_KEY_IMPORT)
+            else if (x509->pubKeyOID == ED25519k) {
+                key->type = WC_EVP_PKEY_ED25519;
+            }
         #endif
             else {
                 key->type = WC_EVP_PKEY_EC;
@@ -6572,6 +6577,29 @@ WOLFSSL_EVP_PKEY* wolfSSL_X509_get_pubkey(WOLFSSL_X509* x509)
                 }
             }
             #endif /* NO_DSA */
+
+            /* decode Ed25519 key */
+            #if defined(HAVE_ED25519) && defined(HAVE_ED25519_KEY_IMPORT)
+            if (key->type == WC_EVP_PKEY_ED25519) {
+                key->ed25519 = wolfSSL_ED25519_new(x509->heap, INVALID_DEVID);
+                if (key->ed25519 == NULL) {
+                    wolfSSL_EVP_PKEY_free(key);
+                    return NULL;
+                }
+                key->ownEd25519 = 1;
+
+                /* The X.509 public key buffer holds the raw Ed25519 key
+                 * (CopyDecodedToX509 / StoreKey store the BIT STRING
+                 * contents), so import it directly. */
+                if (wc_ed25519_import_public(
+                            (const unsigned char*)key->pkey.ptr,
+                            (word32)key->pkey_sz, key->ed25519) != 0) {
+                    WOLFSSL_MSG("wc_ed25519_import_public failed");
+                    wolfSSL_EVP_PKEY_free(key);
+                    return NULL;
+                }
+            }
+            #endif /* HAVE_ED25519 */
         }
     }
     return key;
@@ -9010,6 +9038,12 @@ static int verifyX509orX509REQ(WOLFSSL_X509* x509, WOLFSSL_EVP_PKEY* pkey,
     const byte* der;
     int derSz = 0;
     int type;
+    const byte* pubKey;
+    int pubKeySz;
+#if defined(HAVE_ED25519) && defined(HAVE_ED25519_KEY_EXPORT)
+    byte edPubKey[ED25519_PUB_KEY_SIZE];
+    word32 edPubKeySz = (word32)sizeof(edPubKey);
+#endif
 
     (void)req;
 
@@ -9023,6 +9057,10 @@ static int verifyX509orX509REQ(WOLFSSL_X509* x509, WOLFSSL_EVP_PKEY* pkey,
         return WOLFSSL_FATAL_ERROR;
     }
 
+    /* Most key types verify against the cached public-key DER. */
+    pubKey   = (const byte*)pkey->pkey.ptr;
+    pubKeySz = pkey->pkey_sz;
+
     switch (pkey->type) {
         case WC_EVP_PKEY_RSA:
             type = RSAk;
@@ -9036,6 +9074,23 @@ static int verifyX509orX509REQ(WOLFSSL_X509* x509, WOLFSSL_EVP_PKEY* pkey,
             type = DSAk;
             break;
 
+    #if defined(HAVE_ED25519) && defined(HAVE_ED25519_KEY_EXPORT)
+        case WC_EVP_PKEY_ED25519:
+            /* The signature check needs the raw public key; for Ed25519
+             * pkey->pkey.ptr holds a PKCS#8 private blob (or nothing), so
+             * export the public key from the ed25519_key instead. */
+            if (pkey->ed25519 == NULL ||
+                    wc_ed25519_export_public(pkey->ed25519, edPubKey,
+                        &edPubKeySz) != 0) {
+                WOLFSSL_MSG("Unable to export Ed25519 public key");
+                return WOLFSSL_FATAL_ERROR;
+            }
+            pubKey   = edPubKey;
+            pubKeySz = (int)edPubKeySz;
+            type     = ED25519k;
+            break;
+    #endif
+
         default:
             WOLFSSL_MSG("Unknown pkey key type");
             return WOLFSSL_FATAL_ERROR;
@@ -9044,11 +9099,11 @@ static int verifyX509orX509REQ(WOLFSSL_X509* x509, WOLFSSL_EVP_PKEY* pkey,
 #ifdef WOLFSSL_CERT_REQ
     if (req)
         ret = CheckCSRSignaturePubKey(der, (word32)derSz, x509->heap,
-                (unsigned char*)pkey->pkey.ptr, pkey->pkey_sz, type);
+                (unsigned char*)pubKey, pubKeySz, type);
     else
 #endif
         ret = CheckCertSignaturePubKey(der, (word32)derSz, x509->heap,
-                (unsigned char*)pkey->pkey.ptr, pkey->pkey_sz, type);
+                (unsigned char*)pubKey, pubKeySz, type);
     if (ret == 0) {
         return WOLFSSL_SUCCESS;
     }
@@ -12225,6 +12280,14 @@ static int CertFromX509(Cert* cert, WOLFSSL_X509* x509)
         int hashType;
         int sigType = WOLFSSL_FAILURE;
 
+    #if defined(HAVE_ED25519)
+        /* Ed25519 carries its own hash, so md is unused (and may be NULL).
+         * Resolve it before touching md. */
+        if (pkey->type == WC_EVP_PKEY_ED25519) {
+            return CTC_ED25519;
+        }
+    #endif
+
         /* Convert key type and hash algorithm to a signature algorithm */
         if (wolfSSL_EVP_get_hashinfo(md, &hashType, NULL)
             == WC_NO_ERR_TRACE(WOLFSSL_FAILURE))
@@ -12337,6 +12400,9 @@ static int CertFromX509(Cert* cert, WOLFSSL_X509* x509)
     #ifndef NO_DSA
         DsaKey* dsa = NULL;
     #endif
+    #if defined(HAVE_ED25519)
+        ed25519_key* ed25519 = NULL;
+    #endif
     #if defined(HAVE_FALCON)
         falcon_key* falcon = NULL;
     #endif
@@ -12472,6 +12538,28 @@ static int CertFromX509(Cert* cert, WOLFSSL_X509* x509)
             key = (void*)dsa;
         }
     #endif
+    #if defined(HAVE_ED25519) && defined(HAVE_ED25519_KEY_IMPORT)
+        if (x509->pubKeyOID == ED25519k) {
+            ed25519 = wolfSSL_ED25519_new(NULL, INVALID_DEVID);
+            if (ed25519 == NULL) {
+                WOLFSSL_MSG("Failed to allocate memory for ed25519_key");
+                XFREE(cert, NULL, DYNAMIC_TYPE_CERT);
+                return WOLFSSL_FAILURE;
+            }
+
+            type = ED25519_TYPE;
+            /* The X.509 public key buffer holds the raw Ed25519 key. */
+            ret = wc_ed25519_import_public(x509->pubKey.buffer,
+                                           x509->pubKey.length, ed25519);
+            if (ret != 0) {
+                WOLFSSL_ERROR_VERBOSE(ret);
+                wolfSSL_ED25519_free(ed25519);
+                XFREE(cert, NULL, DYNAMIC_TYPE_CERT);
+                return ret;
+            }
+            key = (void*)ed25519;
+        }
+    #endif
     #if defined(HAVE_FALCON)
         if ((x509->pubKeyOID == FALCON_LEVEL1k) ||
             (x509->pubKeyOID == FALCON_LEVEL5k)) {
@@ -12723,6 +12811,11 @@ static int CertFromX509(Cert* cert, WOLFSSL_X509* x509)
             XFREE(ecc, NULL, DYNAMIC_TYPE_ECC);
         }
     #endif
+    #if defined(HAVE_ED25519)
+        if (x509->pubKeyOID == ED25519k) {
+            wolfSSL_ED25519_free(ed25519);
+        }
+    #endif
     #ifndef NO_DSA
         if (x509->pubKeyOID == DSAk) {
             wc_FreeDsaKey(dsa);
@@ -12807,6 +12900,12 @@ static int CertFromX509(Cert* cert, WOLFSSL_X509* x509)
             key = pkey->ecc->internal;
         }
     #endif
+    #if defined(HAVE_ED25519)
+        if (pkey->type == WC_EVP_PKEY_ED25519) {
+            type = ED25519_TYPE;
+            key = pkey->ed25519;
+        }
+    #endif
 
         /* Sign the certificate (request) body. */
         ret = wc_InitRng(&rng);
@@ -12895,11 +12994,24 @@ int wolfSSL_X509_sign(WOLFSSL_X509* x509, WOLFSSL_EVP_PKEY* pkey,
 
     WOLFSSL_ENTER("wolfSSL_X509_sign");
 
-    if (x509 == NULL || pkey == NULL || md == NULL) {
+    if (x509 == NULL || pkey == NULL) {
         ret = WOLFSSL_FAILURE;
         goto out;
     }
 
+    /* Most key types require an explicit digest.  Ed25519 is the exception:
+     * it signs with a NULL digest (the key has a built-in hash).  Kept as a
+     * dedicated block since other algorithms may grow similar edge cases. */
+    if (md == NULL) {
+    #if defined(HAVE_ED25519)
+        if (pkey->type != WC_EVP_PKEY_ED25519)
+    #endif
+        {
+            ret = WOLFSSL_FAILURE;
+            goto out;
+        }
+    }
+
     x509->sigOID = wolfSSL_sigTypeFromPKEY((WOLFSSL_EVP_MD*)md, pkey);
     if ((ret = wolfssl_x509_make_der(x509, 0, der, &derSz, 0)) !=
             WOLFSSL_SUCCESS) {
@@ -16408,6 +16520,30 @@ int wolfSSL_X509_set_pubkey(WOLFSSL_X509 *cert, WOLFSSL_EVP_PKEY *pkey)
             cert->pubKeyOID = ECDSAk;
         }
         break;
+#endif
+#if defined(HAVE_ED25519) && defined(HAVE_ED25519_KEY_EXPORT)
+    case WC_EVP_PKEY_ED25519:
+        {
+            word32 rawLen = ED25519_PUB_KEY_SIZE;
+
+            if (pkey->ed25519 == NULL)
+                return WOLFSSL_FAILURE;
+
+            /* Store the RAW public key: wolfSSL keeps an X.509 Ed25519
+             * public key as the bare key bytes (see StoreKey /
+             * CopyDecodedToX509), not a SubjectPublicKeyInfo. */
+            p = (byte*)XMALLOC(rawLen, cert->heap, DYNAMIC_TYPE_PUBLIC_KEY);
+            if (p == NULL)
+                return WOLFSSL_FAILURE;
+
+            if (wc_ed25519_export_public(pkey->ed25519, p, &rawLen) != 0) {
+                XFREE(p, cert->heap, DYNAMIC_TYPE_PUBLIC_KEY);
+                return WOLFSSL_FAILURE;
+            }
+            derSz = (int)rawLen;
+            cert->pubKeyOID = ED25519k;
+        }
+        break;
 #endif
     default:
         return WOLFSSL_FAILURE;