rsa/rsa_gen.c: harmonize keygen's ability with RSA_security_bits.

Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/4791)

2 files changed, 9 insertions, 30 deletions

diff --git a/crypto/rsa/rsa_chk.c b/crypto/rsa/rsa_chk.c
index 4cf682258b..1b69be30ca 100644
--- a/crypto/rsa/rsa_chk.c
+++ b/crypto/rsa/rsa_chk.c
@@ -30,10 +30,13 @@ int RSA_check_key_ex(const RSA *key, BN_GENCB *cb)
     }
 
     /* multi-prime? */
-    if (key->version == RSA_ASN1_VERSION_MULTI
-        && (ex_primes = sk_RSA_PRIME_INFO_num(key->prime_infos)) <= 0) {
-        RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_INVALID_MULTI_PRIME_KEY);
-        return 0;
+    if (key->version == RSA_ASN1_VERSION_MULTI) {
+        ex_primes = sk_RSA_PRIME_INFO_num(key->prime_infos);
+        if (ex_primes <= 0
+                || (ex_primes + 2) > rsa_multip_cap(BN_num_bits(key->n))) {
+            RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_INVALID_MULTI_PRIME_KEY);
+            return 0;
+        }
     }
 
     i = BN_new();
diff --git a/crypto/rsa/rsa_gen.c b/crypto/rsa/rsa_gen.c
index b092bbab43..eda23b5481 100644
--- a/crypto/rsa/rsa_gen.c
+++ b/crypto/rsa/rsa_gen.c
@@ -73,16 +73,6 @@ static int rsa_builtin_keygen(RSA *rsa, int bits, int primes, BIGNUM *e_value,
     BN_ULONG bitst = 0;
 
     /*
-     * From Github pull request #4241:
-     *
-     * We are in disagreement on how to handle security trade-off, in other
-     * words:
-     *
-     * mechanical-check-for-maximum-of-16-prime-factors vs.
-     * limiting-number-depending-on-length-less-factors-for-shorter-keys.
-     */
-
-    /*
      * When generating ridiculously small keys, we can get stuck
      * continually regenerating the same prime values.
      */
@@ -92,8 +82,8 @@ static int rsa_builtin_keygen(RSA *rsa, int bits, int primes, BIGNUM *e_value,
         goto err;
     }
 
-    if (primes < RSA_DEFAULT_PRIME_NUM
-        || primes > RSA_MAX_PRIME_NUM || bits <= primes) {
+    if (primes < RSA_DEFAULT_PRIME_NUM || primes > rsa_multip_cap(bits)) {
+        ok = 0;             /* we set our own err */
         RSAerr(RSA_F_RSA_BUILTIN_KEYGEN, RSA_R_KEY_PRIME_NUM_INVALID);
         goto err;
     }
@@ -112,20 +102,6 @@ static int rsa_builtin_keygen(RSA *rsa, int bits, int primes, BIGNUM *e_value,
     quo = bits / primes;
     rmd = bits % primes;
 
-    if (primes > RSA_DEFAULT_PRIME_NUM && quo < RSA_MIN_PRIME_SIZE) {
-        /*
-         * this means primes are too many for the key bits.
-         *
-         * This only affects multi-prime keys. For normal RSA,
-         * it's limited above (bits >= 16, hence each prime >= 8).
-         *
-         * This is done in this way because the original normal
-         * RSA's behavior should not alter at least in OpenSSL 1.1.1.
-         */
-        RSAerr(RSA_F_RSA_BUILTIN_KEYGEN, RSA_R_KEY_PRIME_NUM_INVALID);
-        goto err;
-    }
-
     for (i = 0; i < primes; i++)
         bitsr[i] = (i < rmd) ? quo + 1 : quo;