WIP: swtpm_cert: Enable CAs with ML-DSA key to sign a certifcate

Test for GNUTLS_PK_MLDSA44 to detect whether GnuTLS supports ML-DSA.

Only SHAKE-256 can be used for hashing when ML-DSA is used for signing:

https://github.com/gnutls/gnutls/blob/df24a53136f188d77aaffe66316b0fb6ba720d40/lib/algorithms/sign.c#L405-L428

The problem is now that the size of NVRAM indices is limited to
MAX_NV_INDEX_SIZE = 2048, which is too small for a certificate created
even with ML-DSA-44, which is around 2757 bytes long.

Signed-off-by: Stefan Berger <stefanb@linux.ibm.com>

Author: stefanberger

configure.ac

@@ -220,6 +220,25 @@ AS_IF([test "x$with_gnutls" != "xno"],
         [with_gnutls=no]
     )
     AS_IF([test "x$with_gnutls" != "xno"],[with_gnutls="yes"])
+    AS_IF([test "x$with_gnutls" = "xyes"],
+          [AC_COMPILE_IFELSE(
+            [AC_LANG_PROGRAM(
+              [[
+                #include <gnutls/gnutls.h>
+                #include <stdio.h>
+              ]],
+              [[
+                printf("%d", GNUTLS_PK_MLDSA44);
+              ]]
+            )],
+            [
+             AC_MSG_RESULT("GnuTLS has ML-DSA support")
+             AC_DEFINE_UNQUOTED([GNUTLS_SUPPORTS_MLDSA], 1,
+                [whether GnuTLS support ML-DSA signing algorithm])
+            ],
+            [AC_MSG_RESULT("GnuTLS has no ML-DSA support")],
+          )]
+         )
     ]
 )
 

src/swtpm_cert/ek-cert.c

@@ -41,6 +41,8 @@
  *       in section 3.5
  */
 
+#include "config.h"
+
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
@@ -984,6 +986,27 @@ static char *get_password(const char *password)
     return result;
 }
 
+/*
+ * Get the hash algorithm for signing. TPM 1.2 used SHA1 and TPM 2 SHA256.
+ * For an ML-DSA signing key only shake-256 is allowed.
+ */
+static int get_signing_hashalgo(int flags, gnutls_x509_privkey_t sigkey)
+{
+    int keyalgo;
+
+    if (flags & CERT_TYPE_TPM2_F) {
+#ifdef GNUTLS_SUPPORTS_MLDSA
+        keyalgo = gnutls_x509_privkey_get_pk_algorithm2(sigkey, NULL);
+        if (keyalgo == GNUTLS_PK_MLDSA44 ||
+            keyalgo == GNUTLS_PK_MLDSA65 ||
+            keyalgo == GNUTLS_PK_MLDSA87)
+            return GNUTLS_DIG_SHAKE_256;
+#endif
+        return GNUTLS_DIG_SHA256;
+    }
+    return GNUTLS_DIG_SHA1;
+}
+
 static void capabilities_print_json(void)
 {
     fprintf(stdout,
@@ -1018,7 +1041,7 @@ main(int argc, char *argv[])
     int ecc_y_len = 0;
     const char *ecc_curveid = NULL;
     gnutls_datum_t datum = { NULL, 0},  out = { NULL, 0};
-    gnutls_digest_algorithm_t hashAlgo = GNUTLS_DIG_SHA1;
+    gnutls_digest_algorithm_t hashAlgo;
     mpz_t serial;
     time_t now;
     int err;
@@ -1261,9 +1284,6 @@ main(int argc, char *argv[])
         }
     }
 
-    if (flags & CERT_TYPE_TPM2_F)
-        hashAlgo = GNUTLS_DIG_SHA256;
-
     if ((mpz_sizeinbase(serial, 2) + 7) / 8 > sizeof(ser_number) - 1) {
         fprintf(stderr, "Serial number is too large.\n");
         goto cleanup;
@@ -1704,6 +1724,8 @@ if (_err != GNUTLS_E_SUCCESS) {             \
     CHECK_GNUTLS_ERROR(err, "Could not set public EK on CRT: %s\n",
                        gnutls_strerror(err))
 
+    hashAlgo = get_signing_hashalgo(flags, sigkey);
+
     /* sign cert */
     if (sigkey) {
         err = gnutls_x509_crt_sign2(crt, sigcert, sigkey, hashAlgo, 0);