[PATCH 0/4] uadk_provider: several improvements and fixes

src/uadk_prov_ec_kmgmt.c

@@ -172,7 +172,7 @@ static handle_t ec_alloc_sess(EC_KEY *ec, struct wd_ecc_out **ec_out)
 	sess = uadk_prov_ecc_alloc_sess(ec, "ecdh");
 	if (!sess) {
 		UADK_ERR("failed to alloc ec sess!\n");
-		return ret;
+		return UADK_P_FAIL;
 	}
 
 	*ec_out = wd_ecxdh_new_out(sess);

src/uadk_prov_rsa.c

@@ -208,7 +208,7 @@ static void uadk_rsa_mutex_infork(void)
 	pthread_mutex_unlock(&rsa_mutex);
 }
 
-int uadk_prov_rsa_init(void)
+static int uadk_prov_rsa_init(void)
 {
 	char alg_name[] = "rsa";
 	int ret;
@@ -277,9 +277,14 @@ void rsa_free_eng_session(struct uadk_rsa_sess *rsa_sess)
 struct uadk_rsa_sess *rsa_get_eng_session(RSA *rsa, unsigned int bits,
 						 int is_crt)
 {
-	unsigned int key_size =  bits >> BIT_BYTES_SHIFT;
+	unsigned int key_size = bits >> BIT_BYTES_SHIFT;
 	struct sched_params params = {0};
 	struct uadk_rsa_sess *rsa_sess;
+	int ret;
+
+	ret = uadk_prov_rsa_init();
+	if (ret)
+		return NULL;
 
 	rsa_sess = rsa_new_eng_session(rsa);
 	if (!rsa_sess)

src/uadk_prov_rsa.h

@@ -155,7 +155,6 @@ int rsa_fill_prikey(RSA *rsa, struct uadk_rsa_sess *rsa_sess,
 int rsa_fill_pubkey(struct rsa_pubkey_param *pubkey_param,
 			   struct uadk_rsa_sess *rsa_sess,
 			   unsigned char *in_buf, unsigned char *to);
-int uadk_prov_rsa_init(void);
 void rsa_free_eng_session(struct uadk_rsa_sess *rsa_sess);
 struct uadk_rsa_sess *rsa_get_eng_session(RSA *rsa, unsigned int bits,
 						 int is_crt);

src/uadk_prov_rsa_enc.c

@@ -48,7 +48,6 @@ struct PROV_RSA_ASYM_CTX {
 	/* PKCS#1 v1.5 decryption mode */
 	unsigned int implicit_rejection;
 # endif
-	unsigned int soft : 1;
 };
 
 static UADK_PKEY_ASYM_CIPHER s_asym_cipher;
@@ -353,6 +352,9 @@ static int uadk_rsa_asym_init(void *vprsactx, void *vrsa,
 	RSA_free(priv->rsa);
 	priv->rsa = vrsa;
 	priv->operation = operation;
+# if OPENSSL_VERSION_NUMBER >= 0x30200000
+	priv->implicit_rejection = 1;
+#endif
 
 	switch (uadk_rsa_test_flags(priv->rsa, RSA_FLAG_TYPE_MASK)) {
 	case RSA_FLAG_TYPE_RSA:
@@ -363,9 +365,6 @@ static int uadk_rsa_asym_init(void *vprsactx, void *vrsa,
 		return UADK_P_FAIL;
 	}
 
-	if (uadk_prov_rsa_init())
-		priv->soft = 1;
-
 	return uadk_asym_cipher_rsa_set_ctx_params(vprsactx, params);
 }
 
@@ -475,21 +474,27 @@ static int uadk_asym_cipher_rsa_encrypt(void *vprsactx, unsigned char *out,
 	size_t len;
 	int ret;
 
-	if (!priv || priv->soft) {
-		ret = UADK_DO_SOFT;
-		goto exe_soft;
+	if (!priv) {
+		UADK_ERR("invalid: vprsactx is NULL for rsa encrypt\n");
+		return UADK_P_FAIL;
+	}
+
+	len = uadk_rsa_size(priv->rsa);
+	if (len == 0) {
+		UADK_ERR("invalid: rsa key size is 0.\n");
+		return UADK_P_FAIL;
 	}
 
 	if (!out) {
-		len = uadk_rsa_size(priv->rsa);
-		if (len == 0) {
-			UADK_ERR("invalid: rsa encrypt size.\n");
-			return UADK_P_FAIL;
-		}
 		*outlen = len;
 		return UADK_P_SUCCESS;
 	}
 
+	if (outsize < len) {
+		UADK_ERR("invalid: outsize %d is too small.\n", outsize);
+		return UADK_P_FAIL;
+	}
+
 	if (priv->pad_mode == RSA_PKCS1_OAEP_PADDING)
 		ret = uadk_asym_cipher_rsa_oaep_encrypt(priv, out, in, inlen);
 	else
@@ -604,9 +609,9 @@ static int uadk_asym_cipher_rsa_decrypt(void *vprsactx, unsigned char *out,
 	size_t len;
 	int ret;
 
-	if (!priv || priv->soft) {
-		ret = UADK_DO_SOFT;
-		goto exe_soft;
+	if (!priv) {
+		UADK_ERR("invalid: vprsactx is NULL for rsa decrypt\n");
+		return UADK_P_FAIL;
 	}
 
 	len = uadk_rsa_size(priv->rsa);

src/uadk_prov_rsa_kmgmt.c

@@ -893,12 +893,6 @@ static void *uadk_keymgmt_rsa_gen(void *genctx, OSSL_CALLBACK *osslcb, void *cba
 	if (!gctx)
 		return NULL;
 
-	ret = uadk_prov_rsa_init();
-	if (ret) {
-		ret = UADK_DO_SOFT;
-		goto exe_soft;
-	}
-
 	rsa = ossl_rsa_new_with_ctx(gctx->libctx);
 	if (!rsa)
 		return NULL;

src/uadk_prov_rsa_sign.c

@@ -35,7 +35,9 @@ struct PROV_RSA_SIG_CTX {
 	char *propq;
 	RSA *rsa;
 	int operation;
-
+#if OPENSSL_VERSION_NUMBER >= 0x30400000L
+	unsigned int flag_sigalg : 1;
+#endif
 	/*
 	 * Flag to determine if the hash function can be changed (1) or not (0)
 	 * Because it's dangerous to change during a DigestSign or DigestVerify
@@ -69,10 +71,19 @@ struct PROV_RSA_SIG_CTX {
 	size_t siglen;
 #endif
 
+#ifdef FIPS_MODULE
+#ifdef OPENSSL_VERSION_NUMBER >= 0x30400000L
+	/*
+	 * FIPS 140-3 IG 2.4.B mandates that verification based on a digest of a
+	 * message is not permitted.  However, signing based on a digest is still
+	 * permitted.
+	 */
+	int verify_message;
+#endif
+#endif
+
 	/* Temp buffer */
 	unsigned char *tbuf;
-
-	unsigned int soft : 1;
 };
 
 static int encode_pkcs1(unsigned char **out, size_t *out_len, int type,
@@ -637,6 +648,9 @@ static int uadk_rsa_signverify_init(void *vprsactx, void *vrsa,
 	/* Maximum for sign, auto for verify */
 	ctx->saltlen = RSA_PSS_SALTLEN_AUTO;
 	ctx->min_saltlen = -1;
+	ctx->flag_allow_oneshot = 1;
+	ctx->flag_allow_final = 1;
+	ctx->flag_allow_update = 1;
 
 	switch (uadk_rsa_test_flags(ctx->rsa, RSA_FLAG_TYPE_MASK)) {
 	case RSA_FLAG_TYPE_RSA:
@@ -652,9 +666,6 @@ static int uadk_rsa_signverify_init(void *vprsactx, void *vrsa,
 		return UADK_P_FAIL;
 	}
 
-	if (uadk_prov_rsa_init())
-		ctx->soft = 1;
-
 	if (!uadk_signature_rsa_set_ctx_params(ctx, params))
 		return UADK_P_FAIL;
 
@@ -723,9 +734,9 @@ static int uadk_signature_rsa_verify_recover(void *vprsactx, unsigned char *rout
 	struct PROV_RSA_SIG_CTX *priv = (struct PROV_RSA_SIG_CTX *)vprsactx;
 	int ret;
 
-	if (!priv || priv->soft) {
-		ret = UADK_DO_SOFT;
-		goto exe_soft;
+	if (!priv) {
+		UADK_ERR("invalid: vprsactx is NULL for rsa verify_recover\n");
+		return UADK_P_FAIL;
 	}
 
 	if (!rout) {
@@ -911,9 +922,9 @@ static int uadk_signature_rsa_verify(void *vprsactx, const unsigned char *sig,
 	size_t rslen = 0;
 	int ret;
 
-	if (!priv || priv->soft) {
-		ret = UADK_DO_SOFT;
-		goto exe_soft;
+	if (!priv) {
+		UADK_ERR("invalid: vprsactx is NULL for rsa verify\n");
+		return UADK_P_FAIL;
 	}
 
 	if (!priv->md) {
@@ -1140,9 +1151,9 @@ static int uadk_signature_rsa_sign(void *vprsactx, unsigned char *sig,
 	size_t mdsize;
 	int ret;
 
-	if (!priv || priv->soft) {
-		ret = UADK_DO_SOFT;
-		goto exe_soft;
+	if (!priv) {
+		UADK_ERR("invalid: vprsactx is NULL for rsa sign\n");
+		return UADK_P_FAIL;
 	}
 
 	rsasize = uadk_rsa_size(priv->rsa);
@@ -1567,6 +1578,7 @@ static int uadk_signature_rsa_digest_sign_final(void *vprsactx, unsigned char *s
 	struct PROV_RSA_SIG_CTX *priv = (struct PROV_RSA_SIG_CTX *)vprsactx;
 	unsigned char digest[EVP_MAX_MD_SIZE];
 	unsigned int dlen = 0;
+	int ret;
 
 	if (!priv)
 		return UADK_P_FAIL;
@@ -1589,7 +1601,14 @@ static int uadk_signature_rsa_digest_sign_final(void *vprsactx, unsigned char *s
 
 	priv->flag_allow_md = 1;
 
-	return uadk_signature_rsa_sign(vprsactx, sig, siglen, sigsize, digest, (size_t)dlen);
+	ret = uadk_signature_rsa_sign(vprsactx, sig, siglen, sigsize, digest, (size_t)dlen);
+	if (sig != NULL) {
+		priv->flag_allow_update = 0;
+		priv->flag_allow_oneshot = 0;
+		priv->flag_allow_final = 0;
+	}
+
+	return ret;
 }
 
 static int uadk_signature_rsa_digest_verify_init(void *vprsactx, const char *mdname,
@@ -1616,6 +1635,7 @@ static int uadk_signature_rsa_digest_verify_final(void *vprsactx, const unsigned
 	struct PROV_RSA_SIG_CTX *priv = (struct PROV_RSA_SIG_CTX *)vprsactx;
 	unsigned char digest[EVP_MAX_MD_SIZE];
 	unsigned int dlen = 0;
+	int ret;
 
 	if (!priv)
 		return UADK_P_FAIL;
@@ -1631,7 +1651,14 @@ static int uadk_signature_rsa_digest_verify_final(void *vprsactx, const unsigned
 		return UADK_P_FAIL;
 
 	priv->flag_allow_md = 1;
-	return uadk_signature_rsa_verify(vprsactx, sig, siglen, digest, (size_t)dlen);
+
+	ret = uadk_signature_rsa_verify(vprsactx, sig, siglen, digest, (size_t)dlen);
+
+	priv->flag_allow_update = 0;
+	priv->flag_allow_final = 0;
+	priv->flag_allow_oneshot = 0;
+
+	return ret;
 }
 
 static void *uadk_signature_rsa_dupctx(void *vprsactx)