EVP: Add new domparams and key generation functionality

The following functions are added:

EVP_PKEY_gen_set_params(), replacing the older EVP_PKEY_CTX_ctrl()
EVP_PKEY_gen(), replacing both EVP_PKEY_keygen() and EVP_PKEY_paramgen()

These functions are made to work together with already existing domparams
and key generation functionality: EVP_PKEY_CTX_new_provided(),
EVP_PKEY_paramgen_init(), EVP_PKEY_keygen_init(), etc.

Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/10289)

4 files changed, 237 insertions, 66 deletions

diff --git a/crypto/evp/keymgmt_lib.c b/crypto/evp/keymgmt_lib.c
index 68ccdbb8ee..94be3c2a9c 100644
--- a/crypto/evp/keymgmt_lib.c
+++ b/crypto/evp/keymgmt_lib.c
@@ -350,3 +350,23 @@ int evp_keymgmt_util_copy(EVP_PKEY *to, EVP_PKEY *from, int selection)
 
     return 1;
 }
+
+void *evp_keymgmt_util_gen(EVP_PKEY *target, EVP_KEYMGMT *keymgmt,
+                           void *genctx, OSSL_CALLBACK *cb, void *cbarg)
+{
+    void *keydata = evp_keymgmt_gen(keymgmt, genctx, cb, cbarg);
+
+    if (keydata != NULL) {
+        if (!EVP_KEYMGMT_up_ref(keymgmt)) {
+            evp_keymgmt_freedata(keymgmt, keydata);
+            return NULL;
+        }
+
+        evp_keymgmt_util_clear_operation_cache(target);
+        target->keymgmt = keymgmt;
+        target->keydata = keydata;
+        evp_keymgmt_util_cache_keyinfo(target);
+    }
+
+    return keydata;
+}
diff --git a/crypto/evp/p_lib.c b/crypto/evp/p_lib.c
index 586ffaf041..3012790cee 100644
--- a/crypto/evp/p_lib.c
+++ b/crypto/evp/p_lib.c
@@ -1012,7 +1012,7 @@ int EVP_PKEY_up_ref(EVP_PKEY *pkey)
 }
 
 #ifndef FIPS_MODE
-static void evp_pkey_free_legacy(EVP_PKEY *x)
+void evp_pkey_free_legacy(EVP_PKEY *x)
 {
     if (x->ameth != NULL) {
         if (x->ameth->pkey_free != NULL)
diff --git a/crypto/evp/pmeth_gn.c b/crypto/evp/pmeth_gn.c
index 84149fabd7..03f1426d85 100644
--- a/crypto/evp/pmeth_gn.c
+++ b/crypto/evp/pmeth_gn.c
@@ -9,7 +9,10 @@
 
 #include <stdio.h>
 #include <stdlib.h>
+#include <openssl/core.h>
+#include <openssl/core_names.h>
 #include "internal/cryptlib.h"
+#include "internal/core.h"
 #include <openssl/objects.h>
 #include <openssl/evp.h>
 #include "crypto/bn.h"
@@ -17,102 +20,225 @@
 #include "crypto/evp.h"
 #include "evp_local.h"
 
-#ifndef FIPS_MODE
-int EVP_PKEY_paramgen_init(EVP_PKEY_CTX *ctx)
+static int gen_init(EVP_PKEY_CTX *ctx, int operation)
 {
-    int ret;
-    if (!ctx || !ctx->pmeth || !ctx->pmeth->paramgen) {
-        EVPerr(EVP_F_EVP_PKEY_PARAMGEN_INIT,
-               EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
-        return -2;
+    int ret = 0;
+
+    if (ctx == NULL)
+        goto not_supported;
+
+    evp_pkey_ctx_free_old_ops(ctx);
+    ctx->operation = operation;
+
+    if (ctx->engine != NULL || ctx->keytype == NULL)
+        goto legacy;
+
+    if (ctx->keymgmt == NULL) {
+        ctx->keymgmt =
+            EVP_KEYMGMT_fetch(ctx->libctx, ctx->keytype, ctx->propquery);
+        if (ctx->keymgmt == NULL
+            || ctx->keymgmt->gen_init == NULL) {
+            EVP_KEYMGMT_free(ctx->keymgmt);
+            ctx->keymgmt = NULL;
+            goto legacy;
+        }
+    }
+    if (ctx->keymgmt->gen_init == NULL)
+        goto not_supported;
+
+    switch (operation) {
+    case EVP_PKEY_OP_PARAMGEN:
+        ctx->op.keymgmt.genctx =
+            evp_keymgmt_gen_init(ctx->keymgmt,
+                                 OSSL_KEYMGMT_SELECT_ALL_PARAMETERS);
+        break;
+    case EVP_PKEY_OP_KEYGEN:
+        ctx->op.keymgmt.genctx =
+            evp_keymgmt_gen_init(ctx->keymgmt, OSSL_KEYMGMT_SELECT_KEYPAIR);
+        break;
+    }
+
+    if (ctx->op.keymgmt.genctx == NULL)
+        ERR_raise(ERR_LIB_EVP, EVP_R_INITIALIZATION_ERROR);
+    else
+        ret = 1;
+    goto end;
+
+ legacy:
+#ifdef FIPS_MODE
+    goto not_supported;
+#else
+    if (ctx->pmeth == NULL
+        || (operation == EVP_PKEY_OP_PARAMGEN
+            && ctx->pmeth->paramgen == NULL)
+        || (operation == EVP_PKEY_OP_KEYGEN
+            && ctx->pmeth->keygen == NULL))
+        goto not_supported;
+
+    ret = 1;
+    switch (operation) {
+    case EVP_PKEY_OP_PARAMGEN:
+        if (ctx->pmeth->paramgen_init != NULL)
+            ret = ctx->pmeth->paramgen_init(ctx);
+        break;
+    case EVP_PKEY_OP_KEYGEN:
+        if (ctx->pmeth->keygen_init != NULL)
+            ret = ctx->pmeth->keygen_init(ctx);
+        break;
     }
-    ctx->operation = EVP_PKEY_OP_PARAMGEN;
-    if (!ctx->pmeth->paramgen_init)
-        return 1;
-    ret = ctx->pmeth->paramgen_init(ctx);
+#endif
+
+ end:
     if (ret <= 0)
         ctx->operation = EVP_PKEY_OP_UNDEFINED;
     return ret;
+
+ not_supported:
+    ERR_raise(ERR_LIB_EVP, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
+    ret = -2;
+    goto end;
 }
 
-int EVP_PKEY_paramgen(EVP_PKEY_CTX *ctx, EVP_PKEY **ppkey)
+int EVP_PKEY_paramgen_init(EVP_PKEY_CTX *ctx)
 {
-    int ret;
-    if (!ctx || !ctx->pmeth || !ctx->pmeth->paramgen) {
-        EVPerr(EVP_F_EVP_PKEY_PARAMGEN,
-               EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
-        return -2;
-    }
+    return gen_init(ctx, EVP_PKEY_OP_PARAMGEN);
+}
 
-    if (ctx->operation != EVP_PKEY_OP_PARAMGEN) {
-        EVPerr(EVP_F_EVP_PKEY_PARAMGEN, EVP_R_OPERATON_NOT_INITIALIZED);
-        return -1;
-    }
+int EVP_PKEY_keygen_init(EVP_PKEY_CTX *ctx)
+{
+    return gen_init(ctx, EVP_PKEY_OP_KEYGEN);
+}
+
+static int ossl_callback_to_pkey_gencb(const OSSL_PARAM params[], void *arg)
+{
+    EVP_PKEY_CTX *ctx = arg;
+    const OSSL_PARAM *param = NULL;
+    int p = -1, n = -1;
+
+    if (ctx->pkey_gencb == NULL)
+        return 1;                /* No callback?  That's fine */
+
+    if ((param = OSSL_PARAM_locate_const(params, OSSL_GEN_PARAM_POTENTIAL))
+        == NULL
+        || !OSSL_PARAM_get_int(param, &p))
+        return 0;
+    if ((param = OSSL_PARAM_locate_const(params, OSSL_GEN_PARAM_ITERATION))
+        == NULL
+        || !OSSL_PARAM_get_int(param, &n))
+        return 0;
+
+    ctx->keygen_info[0] = p;
+    ctx->keygen_info[1] = n;
+
+    return ctx->pkey_gencb(ctx);
+}
+
+int EVP_PKEY_gen(EVP_PKEY_CTX *ctx, EVP_PKEY **ppkey)
+{
+    int ret = 0;
+    OSSL_CALLBACK cb;
+    EVP_PKEY *allocated_pkey = NULL;
 
     if (ppkey == NULL)
         return -1;
 
+    if (ctx == NULL)
+        goto not_supported;
+
+    if ((ctx->operation & EVP_PKEY_OP_TYPE_GEN) == 0)
+        goto not_initialized;
+
     if (*ppkey == NULL)
-        *ppkey = EVP_PKEY_new();
+        *ppkey = allocated_pkey = EVP_PKEY_new();
 
     if (*ppkey == NULL) {
-        EVPerr(EVP_F_EVP_PKEY_PARAMGEN, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE);
         return -1;
     }
 
-    ret = ctx->pmeth->paramgen(ctx, *ppkey);
+    if (ctx->keymgmt == NULL)
+        goto legacy;
+
+    ret = 1;
+    if (ctx->pkey != NULL) {
+        EVP_KEYMGMT *tmp_keymgmt = ctx->keymgmt;
+        void *keydata =
+            evp_pkey_export_to_provider(ctx->pkey, ctx->libctx,
+                                        &tmp_keymgmt, ctx->propquery);
+
+        if (keydata == NULL)
+            goto not_supported;
+        ret = evp_keymgmt_gen_set_template(ctx->keymgmt,
+                                           ctx->op.keymgmt.genctx, keydata);
+    }
+
+    /*
+     * the returned value from evp_keymgmt_util_gen() is cached in *ppkey,
+     * so we so not need to save it, just check it.
+     */
+    ret = ret
+        && (evp_keymgmt_util_gen(*ppkey, ctx->keymgmt, ctx->op.keymgmt.genctx,
+                                 ossl_callback_to_pkey_gencb, ctx)
+            != NULL);
+
+#ifndef FIPS_MODE
+    /* In case |*ppkey| was originally a legacy key */
+    if (ret)
+        evp_pkey_free_legacy(*ppkey);
+#endif
+
+    goto end;
+
+ legacy:
+#ifdef FIPS_MODE
+    goto not_supported;
+#else
+    switch (ctx->operation) {
+    case EVP_PKEY_OP_PARAMGEN:
+        ret = ctx->pmeth->paramgen(ctx, *ppkey);
+        break;
+    case EVP_PKEY_OP_KEYGEN:
+        ret = ctx->pmeth->keygen(ctx, *ppkey);
+        break;
+    default:
+        goto not_supported;
+    }
+#endif
+
+ end:
     if (ret <= 0) {
-        EVP_PKEY_free(*ppkey);
-        *ppkey = NULL;
+        if (allocated_pkey != NULL)
+            *ppkey = NULL;
+        EVP_PKEY_free(allocated_pkey);
     }
     return ret;
+
+ not_supported:
+    ERR_raise(ERR_LIB_EVP, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
+    ret = -2;
+    goto end;
+ not_initialized:
+    ERR_raise(ERR_LIB_EVP, EVP_R_OPERATON_NOT_INITIALIZED);
+    ret = -1;
+    goto end;
 }
 
-int EVP_PKEY_keygen_init(EVP_PKEY_CTX *ctx)
+int EVP_PKEY_paramgen(EVP_PKEY_CTX *ctx, EVP_PKEY **ppkey)
 {
-    int ret;
-    if (!ctx || !ctx->pmeth || !ctx->pmeth->keygen) {
-        EVPerr(EVP_F_EVP_PKEY_KEYGEN_INIT,
-               EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
-        return -2;
+    if (ctx->operation != EVP_PKEY_OP_PARAMGEN) {
+        ERR_raise(ERR_LIB_EVP, EVP_R_OPERATON_NOT_INITIALIZED);
+        return -1;
     }
-    ctx->operation = EVP_PKEY_OP_KEYGEN;
-    if (!ctx->pmeth->keygen_init)
-        return 1;
-    ret = ctx->pmeth->keygen_init(ctx);
-    if (ret <= 0)
-        ctx->operation = EVP_PKEY_OP_UNDEFINED;
-    return ret;
+    return EVP_PKEY_gen(ctx, ppkey);
 }
 
 int EVP_PKEY_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY **ppkey)
 {
-    int ret;
-
-    if (!ctx || !ctx->pmeth || !ctx->pmeth->keygen) {
-        EVPerr(EVP_F_EVP_PKEY_KEYGEN,
-               EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
-        return -2;
-    }
     if (ctx->operation != EVP_PKEY_OP_KEYGEN) {
-        EVPerr(EVP_F_EVP_PKEY_KEYGEN, EVP_R_OPERATON_NOT_INITIALIZED);
+        ERR_raise(ERR_LIB_EVP, EVP_R_OPERATON_NOT_INITIALIZED);
         return -1;
     }
-
-    if (ppkey == NULL)
-        return -1;
-
-    if (*ppkey == NULL)
-        *ppkey = EVP_PKEY_new();
-    if (*ppkey == NULL)
-        return -1;
-
-    ret = ctx->pmeth->keygen(ctx, *ppkey);
-    if (ret <= 0) {
-        EVP_PKEY_free(*ppkey);
-        *ppkey = NULL;
-    }
-    return ret;
+    return EVP_PKEY_gen(ctx, ppkey);
 }
 
 void EVP_PKEY_CTX_set_cb(EVP_PKEY_CTX *ctx, EVP_PKEY_gen_cb *cb)
@@ -152,6 +278,8 @@ int EVP_PKEY_CTX_get_keygen_info(EVP_PKEY_CTX *ctx, int idx)
     return ctx->keygen_info[idx];
 }
 
+#ifndef FIPS_MODE
+
 EVP_PKEY *EVP_PKEY_new_mac_key(int type, ENGINE *e,
                                const unsigned char *key, int keylen)
 {
diff --git a/crypto/evp/pmeth_lib.c b/crypto/evp/pmeth_lib.c
index 906b08156f..f7bdbebbc1 100644
--- a/crypto/evp/pmeth_lib.c
+++ b/crypto/evp/pmeth_lib.c
@@ -292,6 +292,9 @@ void evp_pkey_ctx_free_old_ops(EVP_PKEY_CTX *ctx)
         EVP_ASYM_CIPHER_free(ctx->op.ciph.cipher);
         ctx->op.ciph.ciphprovctx = NULL;
         ctx->op.ciph.cipher = NULL;
+    } else if (EVP_PKEY_CTX_IS_GEN_OP(ctx)) {
+        if (ctx->op.keymgmt.genctx != NULL && ctx->keymgmt != NULL)
+            evp_keymgmt_gen_cleanup(ctx->keymgmt, ctx->op.keymgmt.genctx);
     }
 #endif
 }
@@ -569,6 +572,12 @@ int EVP_PKEY_CTX_set_params(EVP_PKEY_CTX *ctx, OSSL_PARAM *params)
             && ctx->op.ciph.cipher->set_ctx_params != NULL)
         return ctx->op.ciph.cipher->set_ctx_params(ctx->op.ciph.ciphprovctx,
                                                      params);
+    if (EVP_PKEY_CTX_IS_GEN_OP(ctx)
+        && ctx->op.keymgmt.genctx != NULL
+        && ctx->keymgmt != NULL
+        && ctx->keymgmt->gen_set_params != NULL)
+        return evp_keymgmt_gen_set_params(ctx->keymgmt, ctx->op.keymgmt.genctx,
+                                          params);
     return 0;
 }
 
@@ -629,6 +638,10 @@ const OSSL_PARAM *EVP_PKEY_CTX_settable_params(EVP_PKEY_CTX *ctx)
             && ctx->op.ciph.cipher != NULL
             && ctx->op.ciph.cipher->settable_ctx_params != NULL)
         return ctx->op.ciph.cipher->settable_ctx_params();
+    if (EVP_PKEY_CTX_IS_GEN_OP(ctx)
+            && ctx->keymgmt != NULL
+            && ctx->keymgmt->gen_settable_params != NULL)
+        return evp_keymgmt_gen_settable_params(ctx->keymgmt);
 
     return NULL;
 }
@@ -859,6 +872,12 @@ static int legacy_ctrl_to_param(EVP_PKEY_CTX *ctx, int keytype, int optype,
             ERR_raise(ERR_LIB_EVP,
                       EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
             return -2;
+        case EVP_PKEY_CTRL_RSA_KEYGEN_BITS:
+            return EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, p1);
+        case EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP:
+            return EVP_PKEY_CTX_set_rsa_keygen_pubexp(ctx, p2);
+        case EVP_PKEY_CTRL_RSA_KEYGEN_PRIMES:
+            return EVP_PKEY_CTX_set_rsa_keygen_primes(ctx, p1);
         }
     }
     return 0;
@@ -878,7 +897,9 @@ int EVP_PKEY_CTX_ctrl(EVP_PKEY_CTX *ctx, int keytype, int optype,
             || (EVP_PKEY_CTX_IS_SIGNATURE_OP(ctx)
                 && ctx->op.sig.sigprovctx != NULL)
             || (EVP_PKEY_CTX_IS_ASYM_CIPHER_OP(ctx)
-                && ctx->op.ciph.ciphprovctx != NULL))
+                && ctx->op.ciph.ciphprovctx != NULL)
+            || (EVP_PKEY_CTX_IS_GEN_OP(ctx)
+                && ctx->op.keymgmt.genctx != NULL))
         return legacy_ctrl_to_param(ctx, keytype, optype, cmd, p1, p2);
 
     if (ctx->pmeth == NULL || ctx->pmeth->ctrl == NULL) {
@@ -979,7 +1000,9 @@ int EVP_PKEY_CTX_ctrl_str(EVP_PKEY_CTX *ctx,
             || (EVP_PKEY_CTX_IS_SIGNATURE_OP(ctx)
                 && ctx->op.sig.sigprovctx != NULL)
             || (EVP_PKEY_CTX_IS_ASYM_CIPHER_OP(ctx)
-                && ctx->op.ciph.ciphprovctx != NULL))
+                && ctx->op.ciph.ciphprovctx != NULL)
+            || (EVP_PKEY_CTX_IS_GEN_OP(ctx)
+                && ctx->op.keymgmt.genctx != NULL))
         return legacy_ctrl_str_to_param(ctx, name, value);
 
     if (!ctx || !ctx->pmeth || !ctx->pmeth->ctrl_str) {