Regenerate SP sources: constant-time sp_*_from_mp

Regenerated the single-precision C sources so that sp_<N>_from_mp() converts
secret inputs (ECDH/ECDSA private keys and nonces) in constant time: a
fixed-count loop bounded by the output size with masked reads at/after
a->used, instead of looping a->used times (which leaked the value's
magnitude through the executed-instruction count).

Fixes the ct-callgrind constant-time failures for P-521 (ec_p521_kex,
ec_p521_sign) and hardens the same pattern across all curves and key sizes
(108 sp_*_from_mp functions in sp_c32/sp_c64, sp_arm32/sp_arm64/sp_armthumb/
sp_cortexm and sp_x86_64).

Verified (--enable-sp, gcc 15.2): ec_p521_kex diff 156 -> 0, ec_p521_sign
diff 367 -> 55 (tol 300); testwolfcrypt RSA/ECC KATs pass.

Generated by wolfSSL/scripts#626

Author: danielinux

wolfcrypt/src/sp_arm32.c

@@ -206,27 +206,45 @@ static void sp_2048_from_mp(sp_digit* r, int size, const mp_int* a)
 #elif DIGIT_BIT > 32
     unsigned int i;
     int j = 0;
+    int o;
+    int last = (int)a->used - 1;
     word32 s = 0;
-
+    /* Digit holder and mask are full mp_digit width (the type of a->dp[]) so
+     * the wide-digit split shifts below are not truncated when DIGIT_BIT is
+     * wider than the sp word (e.g. sp_c32.c over a 64-bit mp_digit). */
+    mp_digit d;
+    mp_digit mask;
+
+    /* Last valid digit index; 0 when a->used == 0 (as in the == branch). */
+    last &= 0 - (a->used > 0);
     r[0] = 0;
-    for (i = 0; i < (unsigned int)a->used && j < size; i++) {
-        r[j] |= ((sp_uint32)a->dp[i] << s);
+    /* Loop a fixed number of times (bounded by the output size, not by
+     * a->used). Clamp the read index to a valid digit and mask digits at or
+     * after a->used to zero, so a secret value is converted in constant time
+     * without reading past the mp_int digit array. */
+    for (i = 0; j < size; i++) {
+        /* mask = all ones while i < a->used, else zero (branchless). */
+        mask = (mp_digit)0 - (((mp_digit)i - (mp_digit)(unsigned int)a->used) >>
+            (sizeof(mp_digit) * 8 - 1));
+        /* o = i while i < a->used, else the last valid index (in bounds). */
+        o = (int)(((unsigned int)i & (unsigned int)mask) |
+            ((unsigned int)last & ~(unsigned int)mask));
+        d = a->dp[o] & mask;
+        r[j] |= (sp_digit)(d << s);
         r[j] &= 0xffffffff;
         s = 32U - s;
         if (j + 1 >= size) {
             break;
         }
-        /* lint allow cast of mismatch word32 and mp_digit */
-        r[++j] = (sp_digit)(a->dp[i] >> s); /*lint !e9033*/
+        r[++j] = (sp_digit)(d >> s);
         while ((s + 32U) <= (word32)DIGIT_BIT) {
             s += 32U;
             r[j] &= 0xffffffff;
             if (j + 1 >= size) {
                 break;
             }
             if (s < (word32)DIGIT_BIT) {
-                /* lint allow cast of mismatch word32 and mp_digit */
-                r[++j] = (sp_digit)(a->dp[i] >> s); /*lint !e9033*/
+                r[++j] = (sp_digit)(d >> s);
             }
             else {
                 r[++j] = (sp_digit)0;
@@ -18967,27 +18985,45 @@ static void sp_3072_from_mp(sp_digit* r, int size, const mp_int* a)
 #elif DIGIT_BIT > 32
     unsigned int i;
     int j = 0;
+    int o;
+    int last = (int)a->used - 1;
     word32 s = 0;
-
+    /* Digit holder and mask are full mp_digit width (the type of a->dp[]) so
+     * the wide-digit split shifts below are not truncated when DIGIT_BIT is
+     * wider than the sp word (e.g. sp_c32.c over a 64-bit mp_digit). */
+    mp_digit d;
+    mp_digit mask;
+
+    /* Last valid digit index; 0 when a->used == 0 (as in the == branch). */
+    last &= 0 - (a->used > 0);
     r[0] = 0;
-    for (i = 0; i < (unsigned int)a->used && j < size; i++) {
-        r[j] |= ((sp_uint32)a->dp[i] << s);
+    /* Loop a fixed number of times (bounded by the output size, not by
+     * a->used). Clamp the read index to a valid digit and mask digits at or
+     * after a->used to zero, so a secret value is converted in constant time
+     * without reading past the mp_int digit array. */
+    for (i = 0; j < size; i++) {
+        /* mask = all ones while i < a->used, else zero (branchless). */
+        mask = (mp_digit)0 - (((mp_digit)i - (mp_digit)(unsigned int)a->used) >>
+            (sizeof(mp_digit) * 8 - 1));
+        /* o = i while i < a->used, else the last valid index (in bounds). */
+        o = (int)(((unsigned int)i & (unsigned int)mask) |
+            ((unsigned int)last & ~(unsigned int)mask));
+        d = a->dp[o] & mask;
+        r[j] |= (sp_digit)(d << s);
         r[j] &= 0xffffffff;
         s = 32U - s;
         if (j + 1 >= size) {
             break;
         }
-        /* lint allow cast of mismatch word32 and mp_digit */
-        r[++j] = (sp_digit)(a->dp[i] >> s); /*lint !e9033*/
+        r[++j] = (sp_digit)(d >> s);
         while ((s + 32U) <= (word32)DIGIT_BIT) {
             s += 32U;
             r[j] &= 0xffffffff;
             if (j + 1 >= size) {
                 break;
             }
             if (s < (word32)DIGIT_BIT) {
-                /* lint allow cast of mismatch word32 and mp_digit */
-                r[++j] = (sp_digit)(a->dp[i] >> s); /*lint !e9033*/
+                r[++j] = (sp_digit)(d >> s);
             }
             else {
                 r[++j] = (sp_digit)0;
@@ -46995,27 +47031,45 @@ static void sp_4096_from_mp(sp_digit* r, int size, const mp_int* a)
 #elif DIGIT_BIT > 32
     unsigned int i;
     int j = 0;
+    int o;
+    int last = (int)a->used - 1;
     word32 s = 0;
-
+    /* Digit holder and mask are full mp_digit width (the type of a->dp[]) so
+     * the wide-digit split shifts below are not truncated when DIGIT_BIT is
+     * wider than the sp word (e.g. sp_c32.c over a 64-bit mp_digit). */
+    mp_digit d;
+    mp_digit mask;
+
+    /* Last valid digit index; 0 when a->used == 0 (as in the == branch). */
+    last &= 0 - (a->used > 0);
     r[0] = 0;
-    for (i = 0; i < (unsigned int)a->used && j < size; i++) {
-        r[j] |= ((sp_uint32)a->dp[i] << s);
+    /* Loop a fixed number of times (bounded by the output size, not by
+     * a->used). Clamp the read index to a valid digit and mask digits at or
+     * after a->used to zero, so a secret value is converted in constant time
+     * without reading past the mp_int digit array. */
+    for (i = 0; j < size; i++) {
+        /* mask = all ones while i < a->used, else zero (branchless). */
+        mask = (mp_digit)0 - (((mp_digit)i - (mp_digit)(unsigned int)a->used) >>
+            (sizeof(mp_digit) * 8 - 1));
+        /* o = i while i < a->used, else the last valid index (in bounds). */
+        o = (int)(((unsigned int)i & (unsigned int)mask) |
+            ((unsigned int)last & ~(unsigned int)mask));
+        d = a->dp[o] & mask;
+        r[j] |= (sp_digit)(d << s);
         r[j] &= 0xffffffff;
         s = 32U - s;
         if (j + 1 >= size) {
             break;
         }
-        /* lint allow cast of mismatch word32 and mp_digit */
-        r[++j] = (sp_digit)(a->dp[i] >> s); /*lint !e9033*/
+        r[++j] = (sp_digit)(d >> s);
         while ((s + 32U) <= (word32)DIGIT_BIT) {
             s += 32U;
             r[j] &= 0xffffffff;
             if (j + 1 >= size) {
                 break;
             }
             if (s < (word32)DIGIT_BIT) {
-                /* lint allow cast of mismatch word32 and mp_digit */
-                r[++j] = (sp_digit)(a->dp[i] >> s); /*lint !e9033*/
+                r[++j] = (sp_digit)(d >> s);
             }
             else {
                 r[++j] = (sp_digit)0;
@@ -67752,27 +67806,45 @@ static void sp_256_from_mp(sp_digit* r, int size, const mp_int* a)
 #elif DIGIT_BIT > 32
     unsigned int i;
     int j = 0;
+    int o;
+    int last = (int)a->used - 1;
     word32 s = 0;
-
+    /* Digit holder and mask are full mp_digit width (the type of a->dp[]) so
+     * the wide-digit split shifts below are not truncated when DIGIT_BIT is
+     * wider than the sp word (e.g. sp_c32.c over a 64-bit mp_digit). */
+    mp_digit d;
+    mp_digit mask;
+
+    /* Last valid digit index; 0 when a->used == 0 (as in the == branch). */
+    last &= 0 - (a->used > 0);
     r[0] = 0;
-    for (i = 0; i < (unsigned int)a->used && j < size; i++) {
-        r[j] |= ((sp_uint32)a->dp[i] << s);
+    /* Loop a fixed number of times (bounded by the output size, not by
+     * a->used). Clamp the read index to a valid digit and mask digits at or
+     * after a->used to zero, so a secret value is converted in constant time
+     * without reading past the mp_int digit array. */
+    for (i = 0; j < size; i++) {
+        /* mask = all ones while i < a->used, else zero (branchless). */
+        mask = (mp_digit)0 - (((mp_digit)i - (mp_digit)(unsigned int)a->used) >>
+            (sizeof(mp_digit) * 8 - 1));
+        /* o = i while i < a->used, else the last valid index (in bounds). */
+        o = (int)(((unsigned int)i & (unsigned int)mask) |
+            ((unsigned int)last & ~(unsigned int)mask));
+        d = a->dp[o] & mask;
+        r[j] |= (sp_digit)(d << s);
         r[j] &= 0xffffffff;
         s = 32U - s;
         if (j + 1 >= size) {
             break;
         }
-        /* lint allow cast of mismatch word32 and mp_digit */
-        r[++j] = (sp_digit)(a->dp[i] >> s); /*lint !e9033*/
+        r[++j] = (sp_digit)(d >> s);
         while ((s + 32U) <= (word32)DIGIT_BIT) {
             s += 32U;
             r[j] &= 0xffffffff;
             if (j + 1 >= size) {
                 break;
             }
             if (s < (word32)DIGIT_BIT) {
-                /* lint allow cast of mismatch word32 and mp_digit */
-                r[++j] = (sp_digit)(a->dp[i] >> s); /*lint !e9033*/
+                r[++j] = (sp_digit)(d >> s);
             }
             else {
                 r[++j] = (sp_digit)0;
@@ -90882,27 +90954,45 @@ static void sp_384_from_mp(sp_digit* r, int size, const mp_int* a)
 #elif DIGIT_BIT > 32
     unsigned int i;
     int j = 0;
+    int o;
+    int last = (int)a->used - 1;
     word32 s = 0;
-
+    /* Digit holder and mask are full mp_digit width (the type of a->dp[]) so
+     * the wide-digit split shifts below are not truncated when DIGIT_BIT is
+     * wider than the sp word (e.g. sp_c32.c over a 64-bit mp_digit). */
+    mp_digit d;
+    mp_digit mask;
+
+    /* Last valid digit index; 0 when a->used == 0 (as in the == branch). */
+    last &= 0 - (a->used > 0);
     r[0] = 0;
-    for (i = 0; i < (unsigned int)a->used && j < size; i++) {
-        r[j] |= ((sp_uint32)a->dp[i] << s);
+    /* Loop a fixed number of times (bounded by the output size, not by
+     * a->used). Clamp the read index to a valid digit and mask digits at or
+     * after a->used to zero, so a secret value is converted in constant time
+     * without reading past the mp_int digit array. */
+    for (i = 0; j < size; i++) {
+        /* mask = all ones while i < a->used, else zero (branchless). */
+        mask = (mp_digit)0 - (((mp_digit)i - (mp_digit)(unsigned int)a->used) >>
+            (sizeof(mp_digit) * 8 - 1));
+        /* o = i while i < a->used, else the last valid index (in bounds). */
+        o = (int)(((unsigned int)i & (unsigned int)mask) |
+            ((unsigned int)last & ~(unsigned int)mask));
+        d = a->dp[o] & mask;
+        r[j] |= (sp_digit)(d << s);
         r[j] &= 0xffffffff;
         s = 32U - s;
         if (j + 1 >= size) {
             break;
         }
-        /* lint allow cast of mismatch word32 and mp_digit */
-        r[++j] = (sp_digit)(a->dp[i] >> s); /*lint !e9033*/
+        r[++j] = (sp_digit)(d >> s);
         while ((s + 32U) <= (word32)DIGIT_BIT) {
             s += 32U;
             r[j] &= 0xffffffff;
             if (j + 1 >= size) {
                 break;
             }
             if (s < (word32)DIGIT_BIT) {
-                /* lint allow cast of mismatch word32 and mp_digit */
-                r[++j] = (sp_digit)(a->dp[i] >> s); /*lint !e9033*/
+                r[++j] = (sp_digit)(d >> s);
             }
             else {
                 r[++j] = (sp_digit)0;
@@ -117735,27 +117825,45 @@ static void sp_521_from_mp(sp_digit* r, int size, const mp_int* a)
 #elif DIGIT_BIT > 32
     unsigned int i;
     int j = 0;
+    int o;
+    int last = (int)a->used - 1;
     word32 s = 0;
-
+    /* Digit holder and mask are full mp_digit width (the type of a->dp[]) so
+     * the wide-digit split shifts below are not truncated when DIGIT_BIT is
+     * wider than the sp word (e.g. sp_c32.c over a 64-bit mp_digit). */
+    mp_digit d;
+    mp_digit mask;
+
+    /* Last valid digit index; 0 when a->used == 0 (as in the == branch). */
+    last &= 0 - (a->used > 0);
     r[0] = 0;
-    for (i = 0; i < (unsigned int)a->used && j < size; i++) {
-        r[j] |= ((sp_uint32)a->dp[i] << s);
+    /* Loop a fixed number of times (bounded by the output size, not by
+     * a->used). Clamp the read index to a valid digit and mask digits at or
+     * after a->used to zero, so a secret value is converted in constant time
+     * without reading past the mp_int digit array. */
+    for (i = 0; j < size; i++) {
+        /* mask = all ones while i < a->used, else zero (branchless). */
+        mask = (mp_digit)0 - (((mp_digit)i - (mp_digit)(unsigned int)a->used) >>
+            (sizeof(mp_digit) * 8 - 1));
+        /* o = i while i < a->used, else the last valid index (in bounds). */
+        o = (int)(((unsigned int)i & (unsigned int)mask) |
+            ((unsigned int)last & ~(unsigned int)mask));
+        d = a->dp[o] & mask;
+        r[j] |= (sp_digit)(d << s);
         r[j] &= 0xffffffff;
         s = 32U - s;
         if (j + 1 >= size) {
             break;
         }
-        /* lint allow cast of mismatch word32 and mp_digit */
-        r[++j] = (sp_digit)(a->dp[i] >> s); /*lint !e9033*/
+        r[++j] = (sp_digit)(d >> s);
         while ((s + 32U) <= (word32)DIGIT_BIT) {
             s += 32U;
             r[j] &= 0xffffffff;
             if (j + 1 >= size) {
                 break;
             }
             if (s < (word32)DIGIT_BIT) {
-                /* lint allow cast of mismatch word32 and mp_digit */
-                r[++j] = (sp_digit)(a->dp[i] >> s); /*lint !e9033*/
+                r[++j] = (sp_digit)(d >> s);
             }
             else {
                 r[++j] = (sp_digit)0;
@@ -147538,27 +147646,45 @@ static void sp_1024_from_mp(sp_digit* r, int size, const mp_int* a)
 #elif DIGIT_BIT > 32
     unsigned int i;
     int j = 0;
+    int o;
+    int last = (int)a->used - 1;
     word32 s = 0;
-
+    /* Digit holder and mask are full mp_digit width (the type of a->dp[]) so
+     * the wide-digit split shifts below are not truncated when DIGIT_BIT is
+     * wider than the sp word (e.g. sp_c32.c over a 64-bit mp_digit). */
+    mp_digit d;
+    mp_digit mask;
+
+    /* Last valid digit index; 0 when a->used == 0 (as in the == branch). */
+    last &= 0 - (a->used > 0);
     r[0] = 0;
-    for (i = 0; i < (unsigned int)a->used && j < size; i++) {
-        r[j] |= ((sp_uint32)a->dp[i] << s);
+    /* Loop a fixed number of times (bounded by the output size, not by
+     * a->used). Clamp the read index to a valid digit and mask digits at or
+     * after a->used to zero, so a secret value is converted in constant time
+     * without reading past the mp_int digit array. */
+    for (i = 0; j < size; i++) {
+        /* mask = all ones while i < a->used, else zero (branchless). */
+        mask = (mp_digit)0 - (((mp_digit)i - (mp_digit)(unsigned int)a->used) >>
+            (sizeof(mp_digit) * 8 - 1));
+        /* o = i while i < a->used, else the last valid index (in bounds). */
+        o = (int)(((unsigned int)i & (unsigned int)mask) |
+            ((unsigned int)last & ~(unsigned int)mask));
+        d = a->dp[o] & mask;
+        r[j] |= (sp_digit)(d << s);
         r[j] &= 0xffffffff;
         s = 32U - s;
         if (j + 1 >= size) {
             break;
         }
-        /* lint allow cast of mismatch word32 and mp_digit */
-        r[++j] = (sp_digit)(a->dp[i] >> s); /*lint !e9033*/
+        r[++j] = (sp_digit)(d >> s);
         while ((s + 32U) <= (word32)DIGIT_BIT) {
             s += 32U;
             r[j] &= 0xffffffff;
             if (j + 1 >= size) {
                 break;
             }
             if (s < (word32)DIGIT_BIT) {
-                /* lint allow cast of mismatch word32 and mp_digit */
-                r[++j] = (sp_digit)(a->dp[i] >> s); /*lint !e9033*/
+                r[++j] = (sp_digit)(d >> s);
             }
             else {
                 r[++j] = (sp_digit)0;