fix F-4409: Integer Overflow in PKCS1 Sign Length Check Allows Heap Buffer Overflow

douzzer · douzzer · commit 1b22e4b502f3 · 2026-06-12T18:14:55.000-05:00
diff --git a/linuxkm/lkcapi_rsa_glue.c b/linuxkm/lkcapi_rsa_glue.c
@@ -1128,7 +1128,7 @@ static int km_pkcs1pad_sign(struct akcipher_request *req)
         goto pkcs1pad_sign_out;
     }
 
-    if (req->src_len + hash_enc_len + RSA_MIN_PAD_SZ > ctx->key_len) {
+    if ((word64)req->src_len + (word64)hash_enc_len + RSA_MIN_PAD_SZ > ctx->key_len) {
         err = -EOVERFLOW;
         goto pkcs1pad_sign_out;
     }
@@ -1378,7 +1378,7 @@ static int km_pkcs1_sign(struct crypto_sig *tfm,
         goto pkcs1_sign_out;
     }
 
-    if (slen + hash_enc_len + RSA_MIN_PAD_SZ > ctx->key_len) {
+    if ((word64)slen + (word64)hash_enc_len + RSA_MIN_PAD_SZ > ctx->key_len) {
         err = -EOVERFLOW;
         goto pkcs1_sign_out;
     }
@@ -1708,7 +1708,7 @@ static int km_pkcs1pad_enc(struct akcipher_request *req)
         goto pkcs1_enc_out;
     }
 
-    if (req->src_len + RSA_MIN_PAD_SZ > ctx->key_len) {
+    if ((word64)req->src_len + RSA_MIN_PAD_SZ > ctx->key_len) {
         err = -EOVERFLOW;
         goto pkcs1_enc_out;
     }

Original file line number	Diff line number	Diff line change
`@@ -1128,7 +1128,7 @@ static int km_pkcs1pad_sign(struct akcipher_request *req)`
`1128`	`1128`	`goto pkcs1pad_sign_out;`
`1129`	`1129`	`}`
`1130`	`1130`
`1131`		`- if (req->src_len + hash_enc_len + RSA_MIN_PAD_SZ > ctx->key_len) {`
	`1131`	`+ if ((word64)req->src_len + (word64)hash_enc_len + RSA_MIN_PAD_SZ > ctx->key_len) {`
`1132`	`1132`	`err = -EOVERFLOW;`
`1133`	`1133`	`goto pkcs1pad_sign_out;`
`1134`	`1134`	`}`
`@@ -1378,7 +1378,7 @@ static int km_pkcs1_sign(struct crypto_sig *tfm,`
`1378`	`1378`	`goto pkcs1_sign_out;`
`1379`	`1379`	`}`
`1380`	`1380`
`1381`		`- if (slen + hash_enc_len + RSA_MIN_PAD_SZ > ctx->key_len) {`
	`1381`	`+ if ((word64)slen + (word64)hash_enc_len + RSA_MIN_PAD_SZ > ctx->key_len) {`
`1382`	`1382`	`err = -EOVERFLOW;`
`1383`	`1383`	`goto pkcs1_sign_out;`
`1384`	`1384`	`}`
`@@ -1708,7 +1708,7 @@ static int km_pkcs1pad_enc(struct akcipher_request *req)`
`1708`	`1708`	`goto pkcs1_enc_out;`
`1709`	`1709`	`}`
`1710`	`1710`
`1711`		`- if (req->src_len + RSA_MIN_PAD_SZ > ctx->key_len) {`
	`1711`	`+ if ((word64)req->src_len + RSA_MIN_PAD_SZ > ctx->key_len) {`
`1712`	`1712`	`err = -EOVERFLOW;`
`1713`	`1713`	`goto pkcs1_enc_out;`
`1714`	`1714`	`}`