Reject MLD_PREHASH_NONE in prehash APIs

The internal pre-hash sign/verify entry points now return
MLD_ERR_FAIL when called with MLD_PREHASH_NONE, matching the
documented set of accepted hash algorithms. Doc comments in
mldsa_native.h and sign.h are updated to state the rejection
explicitly.

test_sign_pre_hash is extended to cover both rejection paths.

Signed-off-by: Fredrik Dahlgren <fredrik.dahlgren@trailofbits.com>
[hbecker: squashed fixup commits, rewrote subject/body, folded
 the rejection checks into test_sign_pre_hash]
Signed-off-by: Hanno Becker <beckphan@amazon.co.uk>

Author: fegge

mldsa/mldsa_native.h

@@ -525,6 +525,8 @@ int MLD_API_NAMESPACE(verify_extmu)(
  *   MLD_PREHASH_SHA3_224, MLD_PREHASH_SHA3_256, MLD_PREHASH_SHA3_384,
  *   MLD_PREHASH_SHA3_512, MLD_PREHASH_SHAKE_128, MLD_PREHASH_SHAKE_256.
  *
+ * MLD_PREHASH_NONE is rejected by this API.
+ *
  * @warning This is an unstable API that may change in the future. If you need
  * a stable API use signature_pre_hash_shake256.
  *
@@ -581,6 +583,8 @@ int MLD_API_NAMESPACE(signature_pre_hash_internal)(
  *   MLD_PREHASH_SHA3_224, MLD_PREHASH_SHA3_256, MLD_PREHASH_SHA3_384,
  *   MLD_PREHASH_SHA3_512, MLD_PREHASH_SHAKE_128, MLD_PREHASH_SHAKE_256.
  *
+ * MLD_PREHASH_NONE is rejected by this API.
+ *
  * @warning This is an unstable API that may change in the future. If you need
  * a stable API use verify_pre_hash_shake256.
  *

mldsa/src/sign.c

@@ -1336,6 +1336,12 @@ int mld_sign_signature_pre_hash_internal(
   size_t pre_len;
   int ret;
 
+  if (hashalg == MLD_PREHASH_NONE)
+  {
+    ret = MLD_ERR_FAIL;
+    goto cleanup;
+  }
+
   pre_len = mld_prepare_domain_separation_prefix(pre, ph, phlen, ctx, ctxlen,
                                                  hashalg);
   if (pre_len == 0)
@@ -1378,6 +1384,12 @@ int mld_sign_verify_pre_hash_internal(
   size_t pre_len;
   int ret;
 
+  if (hashalg == MLD_PREHASH_NONE)
+  {
+    ret = MLD_ERR_FAIL;
+    goto cleanup;
+  }
+
   pre_len = mld_prepare_domain_separation_prefix(pre, ph, phlen, ctx, ctxlen,
                                                  hashalg);
   if (pre_len == 0)

mldsa/src/sign.h

@@ -488,6 +488,8 @@ __contract__(
  *   MLD_PREHASH_SHA3_224, MLD_PREHASH_SHA3_256, MLD_PREHASH_SHA3_384,
  *   MLD_PREHASH_SHA3_512, MLD_PREHASH_SHAKE_128, MLD_PREHASH_SHAKE_256.
  *
+ * MLD_PREHASH_NONE is rejected by this API.
+ *
  * @warning This is an unstable API that may change in the future. If you need
  * a stable API use mld_sign_signature_pre_hash_shake256.
  *
@@ -553,6 +555,8 @@ __contract__(
  *   MLD_PREHASH_SHA3_224, MLD_PREHASH_SHA3_256, MLD_PREHASH_SHA3_384,
  *   MLD_PREHASH_SHA3_512, MLD_PREHASH_SHAKE_128, MLD_PREHASH_SHAKE_256.
  *
+ * MLD_PREHASH_NONE is rejected by this API.
+ *
  * @warning This is an unstable API that may change in the future. If you need
  * a stable API use mld_sign_verify_pre_hash_shake256.
  *

test/src/test_mldsa.c

@@ -152,6 +152,13 @@ static int test_sign_pre_hash(void)
   CHECK(mld_sign_verify_pre_hash_shake256(sig, siglen, m, MLEN, ctx, CTXLEN,
                                           pk) == 0);
 
+  /* MLD_PREHASH_NONE must be rejected by the internal prehash APIs. */
+  CHECK(mld_sign_verify_pre_hash_internal(sig, siglen, m, MLEN, ctx, CTXLEN, pk,
+                                          MLD_PREHASH_NONE) == MLD_ERR_FAIL);
+  CHECK(mld_sign_signature_pre_hash_internal(sig, &siglen, m, MLEN, ctx, CTXLEN,
+                                             rnd, sk,
+                                             MLD_PREHASH_NONE) == MLD_ERR_FAIL);
+
   return 0;
 }
 #endif /* !MLD_CONFIG_NO_KEYPAIR_API && !MLD_CONFIG_NO_SIGN_API && \