EVP: Refactor the RSA-PSS key generation controls for providers

Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/11710)

4 files changed, 89 insertions, 51 deletions

diff --git a/crypto/evp/pmeth_lib.c b/crypto/evp/pmeth_lib.c
index 3476d83ea6..eca5178129 100644
--- a/crypto/evp/pmeth_lib.c
+++ b/crypto/evp/pmeth_lib.c
@@ -1017,6 +1017,12 @@ static int legacy_ctrl_str_to_param(EVP_PKEY_CTX *ctx, const char *name,
         name = OSSL_PKEY_PARAM_RSA_E;
     else if (strcmp(name, "rsa_keygen_primes") == 0)
         name = OSSL_PKEY_PARAM_RSA_PRIMES;
+    else if (strcmp(name, "rsa_pss_keygen_md") == 0)
+        name = OSSL_PKEY_PARAM_RSA_DIGEST;
+    else if (strcmp(name, "rsa_pss_keygen_mgf1_md") == 0)
+        name = OSSL_PKEY_PARAM_RSA_MGF1_DIGEST;
+    else if (strcmp(name, "rsa_pss_keygen_saltlen") == 0)
+        name = OSSL_PKEY_PARAM_RSA_PSS_SALTLEN;
 # ifndef OPENSSL_NO_DSA
     else if (strcmp(name, "dsa_paramgen_bits") == 0)
         name = OSSL_PKEY_PARAM_FFC_PBITS;
@@ -1066,7 +1072,8 @@ static int legacy_ctrl_str_to_param(EVP_PKEY_CTX *ctx, const char *name,
         if (!OSSL_PARAM_allocate_from_text(&params[0], settable, name, value,
                                            strlen(value), &exists)) {
             if (!exists) {
-                ERR_raise(ERR_LIB_EVP, EVP_R_COMMAND_NOT_SUPPORTED);
+                ERR_raise_data(ERR_LIB_EVP, EVP_R_COMMAND_NOT_SUPPORTED,
+                               "name=%s,value=%s", name, value);
                 return -2;
             }
             return 0;
diff --git a/crypto/rsa/rsa_lib.c b/crypto/rsa/rsa_lib.c
index 466115f7a0..e80416ed3f 100644
--- a/crypto/rsa/rsa_lib.c
+++ b/crypto/rsa/rsa_lib.c
@@ -1006,13 +1006,16 @@ int EVP_PKEY_CTX_get_rsa_oaep_md(EVP_PKEY_CTX *ctx, const EVP_MD **md)
     return 1;
 }
 
-int EVP_PKEY_CTX_set_rsa_mgf1_md(EVP_PKEY_CTX *ctx, const EVP_MD *md)
+static int int_set_rsa_mgf1_md(EVP_PKEY_CTX *ctx,
+                               /* For EVP_PKEY_CTX_ctrl() */
+                               int keytype, int optype, int cmd,
+                               const EVP_MD *md,
+                               /* For EVP_PKEY_CTX_set_params() */
+                               const char *mdname, const char *mdprops)
 {
-    const char *name;
+    OSSL_PARAM rsa_params[3], *p = rsa_params;
 
-    if (ctx == NULL
-            || (!EVP_PKEY_CTX_IS_ASYM_CIPHER_OP(ctx)
-                && !EVP_PKEY_CTX_IS_SIGNATURE_OP(ctx))) {
+    if (ctx == NULL || (ctx->operation & optype) == 0) {
         ERR_raise(ERR_LIB_EVP, EVP_R_COMMAND_NOT_SUPPORTED);
         /* Uses the same return values as EVP_PKEY_CTX_ctrl */
         return -2;
@@ -1020,43 +1023,25 @@ int EVP_PKEY_CTX_set_rsa_mgf1_md(EVP_PKEY_CTX *ctx, const EVP_MD *md)
 
     /* If key type not RSA return error */
     if (ctx->pmeth != NULL
-            && ctx->pmeth->pkey_id != EVP_PKEY_RSA
-            && ctx->pmeth->pkey_id != EVP_PKEY_RSA_PSS)
+        && (keytype == -1
+            ? (ctx->pmeth->pkey_id != EVP_PKEY_RSA
+               && ctx->pmeth->pkey_id != EVP_PKEY_RSA_PSS)
+            : ctx->pmeth->pkey_id != keytype))
         return -1;
 
     /* TODO(3.0): Remove this eventually when no more legacy */
-    if ((EVP_PKEY_CTX_IS_ASYM_CIPHER_OP(ctx)
-                && ctx->op.ciph.ciphprovctx == NULL)
+    if (cmd != -1) {
+        if ((EVP_PKEY_CTX_IS_ASYM_CIPHER_OP(ctx)
+             && ctx->op.ciph.ciphprovctx == NULL)
             || (EVP_PKEY_CTX_IS_SIGNATURE_OP(ctx)
-                && ctx->op.sig.sigprovctx == NULL))
-        return EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA,
-                                 EVP_PKEY_OP_TYPE_SIG | EVP_PKEY_OP_TYPE_CRYPT,
-                                 EVP_PKEY_CTRL_RSA_MGF1_MD, 0, (void *)md);
-
-    name = (md == NULL) ? "" : EVP_MD_name(md);
-
-    return EVP_PKEY_CTX_set_rsa_mgf1_md_name(ctx, name, NULL);
-}
-
-int EVP_PKEY_CTX_set_rsa_mgf1_md_name(EVP_PKEY_CTX *ctx, const char *mdname,
-                                      const char *mdprops)
-{
-    OSSL_PARAM rsa_params[3], *p = rsa_params;
+                && ctx->op.sig.sigprovctx == NULL)
+            || (EVP_PKEY_CTX_IS_GEN_OP(ctx)
+                && ctx->op.keymgmt.genctx == NULL))
+            return EVP_PKEY_CTX_ctrl(ctx, keytype, optype, cmd, 0, (void *)md);
 
-    if (ctx == NULL
-            || mdname == NULL
-            || (!EVP_PKEY_CTX_IS_ASYM_CIPHER_OP(ctx)
-                && !EVP_PKEY_CTX_IS_SIGNATURE_OP(ctx))) {
-        ERR_raise(ERR_LIB_EVP, EVP_R_COMMAND_NOT_SUPPORTED);
-        /* Uses the same return values as EVP_PKEY_CTX_ctrl */
-        return -2;
+        mdname = (md == NULL) ? "" : EVP_MD_name(md);
     }
 
-    /* If key type not RSA return error */
-    if (ctx->pmeth != NULL
-            && ctx->pmeth->pkey_id != EVP_PKEY_RSA
-            && ctx->pmeth->pkey_id != EVP_PKEY_RSA_PSS)
-        return -1;
 
     *p++ = OSSL_PARAM_construct_utf8_string(OSSL_PKEY_PARAM_MGF1_DIGEST,
                                             /*
@@ -1078,6 +1063,36 @@ int EVP_PKEY_CTX_set_rsa_mgf1_md_name(EVP_PKEY_CTX *ctx, const char *mdname,
     return EVP_PKEY_CTX_set_params(ctx, rsa_params);
 }
 
+int EVP_PKEY_CTX_set_rsa_mgf1_md(EVP_PKEY_CTX *ctx, const EVP_MD *md)
+{
+    return int_set_rsa_mgf1_md(ctx, -1,
+                               EVP_PKEY_OP_TYPE_CRYPT | EVP_PKEY_OP_TYPE_SIG,
+                               EVP_PKEY_CTRL_RSA_MGF1_MD, md, NULL, NULL);
+}
+
+int EVP_PKEY_CTX_set_rsa_mgf1_md_name(EVP_PKEY_CTX *ctx, const char *mdname,
+                                      const char *mdprops)
+{
+    return int_set_rsa_mgf1_md(ctx, -1,
+                               EVP_PKEY_OP_TYPE_CRYPT | EVP_PKEY_OP_TYPE_SIG,
+                               -1, NULL, mdname, mdprops);
+}
+
+int EVP_PKEY_CTX_set_rsa_pss_keygen_mgf1_md(EVP_PKEY_CTX *ctx, const EVP_MD *md)
+{
+    return int_set_rsa_mgf1_md(ctx, EVP_PKEY_RSA_PSS,
+                               EVP_PKEY_OP_KEYGEN, EVP_PKEY_CTRL_RSA_MGF1_MD,
+                               md, NULL, NULL);
+}
+
+int EVP_PKEY_CTX_set_rsa_pss_keygen_mgf1_md_name(EVP_PKEY_CTX *ctx,
+                                                 const char *mdname)
+{
+    return int_set_rsa_mgf1_md(ctx, EVP_PKEY_RSA_PSS,
+                               EVP_PKEY_OP_TYPE_CRYPT | EVP_PKEY_OP_TYPE_SIG,
+                               -1, NULL, mdname, NULL);
+}
+
 int EVP_PKEY_CTX_get_rsa_mgf1_md_name(EVP_PKEY_CTX *ctx, char *name,
                                       size_t namelen)
 {
@@ -1216,11 +1231,12 @@ int EVP_PKEY_CTX_get0_rsa_oaep_label(EVP_PKEY_CTX *ctx, unsigned char **label)
     return (int)labellen;
 }
 
-int EVP_PKEY_CTX_set_rsa_pss_saltlen(EVP_PKEY_CTX *ctx, int saltlen)
+static int int_set_rsa_pss_saltlen(EVP_PKEY_CTX *ctx, int saltlen,
+                                   int keytype, int optype)
 {
     OSSL_PARAM pad_params[2], *p = pad_params;
 
-    if (ctx == NULL) {
+    if (ctx == NULL || (ctx->operation & optype) == 0) {
         ERR_raise(ERR_LIB_EVP, EVP_R_COMMAND_NOT_SUPPORTED);
         /* Uses the same return values as EVP_PKEY_CTX_ctrl */
         return -2;
@@ -1228,14 +1244,19 @@ int EVP_PKEY_CTX_set_rsa_pss_saltlen(EVP_PKEY_CTX *ctx, int saltlen)
 
     /* If key type not RSA or RSA-PSS return error */
     if (ctx->pmeth != NULL
-            && ctx->pmeth->pkey_id != EVP_PKEY_RSA
-            && ctx->pmeth->pkey_id != EVP_PKEY_RSA_PSS)
+        && (keytype == -1
+            ? (ctx->pmeth->pkey_id != EVP_PKEY_RSA
+               && ctx->pmeth->pkey_id != EVP_PKEY_RSA_PSS)
+            : ctx->pmeth->pkey_id != keytype))
         return -1;
 
     /* TODO(3.0): Remove this eventually when no more legacy */
-    if (!EVP_PKEY_CTX_IS_SIGNATURE_OP(ctx)
-        || ctx->op.sig.sigprovctx == NULL)
-        return EVP_PKEY_CTX_ctrl(ctx, -1, -1, EVP_PKEY_CTRL_RSA_PSS_SALTLEN,
+    if ((EVP_PKEY_CTX_IS_SIGNATURE_OP(ctx)
+         && ctx->op.sig.sigprovctx == NULL)
+        || (EVP_PKEY_CTX_IS_GEN_OP(ctx)
+            && ctx->op.keymgmt.genctx == NULL))
+        return EVP_PKEY_CTX_ctrl(ctx, keytype, optype,
+                                 EVP_PKEY_CTRL_RSA_PSS_SALTLEN,
                                  saltlen, NULL);
 
     *p++ =
@@ -1245,6 +1266,17 @@ int EVP_PKEY_CTX_set_rsa_pss_saltlen(EVP_PKEY_CTX *ctx, int saltlen)
     return EVP_PKEY_CTX_set_params(ctx, pad_params);
 }
 
+int EVP_PKEY_CTX_set_rsa_pss_saltlen(EVP_PKEY_CTX *ctx, int saltlen)
+{
+    return int_set_rsa_pss_saltlen(ctx, saltlen, -1, EVP_PKEY_OP_TYPE_SIG);
+}
+
+int EVP_PKEY_CTX_set_rsa_pss_keygen_saltlen(EVP_PKEY_CTX *ctx, int saltlen)
+{
+    return int_set_rsa_pss_saltlen(ctx, saltlen, EVP_PKEY_RSA_PSS,
+                                   EVP_PKEY_OP_KEYGEN);
+}
+
 int EVP_PKEY_CTX_get_rsa_pss_saltlen(EVP_PKEY_CTX *ctx, int *saltlen)
 {
     OSSL_PARAM pad_params[2], *p = pad_params;
diff --git a/include/openssl/rsa.h b/include/openssl/rsa.h
index d942bc57ab..bf12b90088 100644
--- a/include/openssl/rsa.h
+++ b/include/openssl/rsa.h
@@ -134,6 +134,7 @@ int EVP_PKEY_CTX_get_rsa_pss_saltlen(EVP_PKEY_CTX *ctx, int *saltlen);
 int EVP_PKEY_CTX_set_rsa_keygen_bits(EVP_PKEY_CTX *ctx, int bits);
 int EVP_PKEY_CTX_set_rsa_keygen_pubexp(EVP_PKEY_CTX *ctx, BIGNUM *pubexp);
 int EVP_PKEY_CTX_set_rsa_keygen_primes(EVP_PKEY_CTX *ctx, int primes);
+int EVP_PKEY_CTX_set_rsa_pss_keygen_saltlen(EVP_PKEY_CTX *ctx, int saltlen);
 
 /* Salt length matches digest */
 #  define RSA_PSS_SALTLEN_DIGEST -1
@@ -144,20 +145,15 @@ int EVP_PKEY_CTX_set_rsa_keygen_primes(EVP_PKEY_CTX *ctx, int primes);
 /* Old compatible max salt length for sign only */
 #  define RSA_PSS_SALTLEN_MAX_SIGN    -2
 
-#  define EVP_PKEY_CTX_set_rsa_pss_keygen_saltlen(ctx, len) \
-        EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA_PSS, EVP_PKEY_OP_KEYGEN, \
-                          EVP_PKEY_CTRL_RSA_PSS_SALTLEN, len, NULL)
-
 int EVP_PKEY_CTX_set_rsa_mgf1_md(EVP_PKEY_CTX *ctx, const EVP_MD *md);
 int EVP_PKEY_CTX_set_rsa_mgf1_md_name(EVP_PKEY_CTX *ctx, const char *mdname,
                                       const char *mdprops);
 int EVP_PKEY_CTX_get_rsa_mgf1_md(EVP_PKEY_CTX *ctx, const EVP_MD **md);
 int EVP_PKEY_CTX_get_rsa_mgf1_md_name(EVP_PKEY_CTX *ctx, char *name,
                                       size_t namelen);
-
-#  define  EVP_PKEY_CTX_set_rsa_pss_keygen_mgf1_md(ctx, md) \
-        EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA_PSS, EVP_PKEY_OP_KEYGEN, \
-                          EVP_PKEY_CTRL_RSA_MGF1_MD, 0, (void *)(md))
+int EVP_PKEY_CTX_set_rsa_pss_keygen_mgf1_md(EVP_PKEY_CTX *ctx, const EVP_MD *md);
+int EVP_PKEY_CTX_set_rsa_pss_keygen_mgf1_md_name(EVP_PKEY_CTX *ctx,
+                                                 const char *mdname);
 
 int EVP_PKEY_CTX_set_rsa_oaep_md(EVP_PKEY_CTX *ctx, const EVP_MD *md);
 int EVP_PKEY_CTX_set_rsa_oaep_md_name(EVP_PKEY_CTX *ctx, const char *mdname,
diff --git a/util/libcrypto.num b/util/libcrypto.num
index ec6acaefd4..104e065bbd 100644
--- a/util/libcrypto.num
+++ b/util/libcrypto.num
@@ -5089,3 +5089,6 @@ EVP_PKEY_new_raw_private_key_with_libctx ?	3_0_0	EXIST::FUNCTION:
 EVP_PKEY_new_raw_public_key_with_libctx ?	3_0_0	EXIST::FUNCTION:
 OSSL_STORE_attach                       ?	3_0_0	EXIST::FUNCTION:
 OSSL_STORE_LOADER_set_attach            ?	3_0_0	EXIST::FUNCTION:
+EVP_PKEY_CTX_set_rsa_pss_keygen_saltlen ?	3_0_0	EXIST::FUNCTION:RSA
+EVP_PKEY_CTX_set_rsa_pss_keygen_mgf1_md ?	3_0_0	EXIST::FUNCTION:RSA
+EVP_PKEY_CTX_set_rsa_pss_keygen_mgf1_md_name ?	3_0_0	EXIST::FUNCTION:RSA