Change DH_get_nid() to set the value of q if it is not already set

Fixes #11108.

It only sets q if a valid named group is found.
The function signature was recently changed to pass a non const DH pointer
in order to allow the nid to be cached internally. As an extension of this
the value of q can now also be set as q is always known for named groups.
The length field is also set if q is set.

Reviewed-by: Tomas Mraz <tmraz@fedoraproject.org>
(Merged from https://github.com/openssl/openssl/pull/11114)

3 files changed, 16 insertions, 5 deletions

diff --git a/crypto/rsa/rsa_lib.c b/crypto/rsa/rsa_lib.c
index 08ce8b4ef8..ada5388bb2 100644
--- a/crypto/rsa/rsa_lib.c
+++ b/crypto/rsa/rsa_lib.c
@@ -23,6 +23,7 @@
 #include "crypto/bn.h"
 #include "crypto/evp.h"
 #include "crypto/rsa.h"
+#include "crypto/security_bits.h"
 #include "rsa_local.h"
 
 static RSA *rsa_new_intern(ENGINE *engine, OPENSSL_CTX *libctx);
@@ -281,11 +282,20 @@ static uint32_t ilog_e(uint64_t v)
  * NIST SP 800-56B rev 2 Appendix D: Maximum Security Strength Estimates for IFC
  * Modulus Lengths.
  *
+ * Note that this formula is also referred to in SP800-56A rev3 Appendix D:
+ * for FFC safe prime groups for modp and ffdhe.
+ * After Table 25 and Table 26 it refers to
+ * "The maximum security strength estimates were calculated using the formula in
+ * Section 7.5 of the FIPS 140 IG and rounded to the nearest multiple of eight
+ * bits".
+ *
+ * The formula is:
+ *
  * E = \frac{1.923 \sqrt[3]{nBits \cdot log_e(2)}
  *           \cdot(log_e(nBits \cdot log_e(2))^{2/3} - 4.69}{log_e(2)}
  * The two cube roots are merged together here.
  */
-uint16_t rsa_compute_security_bits(int n)
+uint16_t ifc_ffc_compute_security_bits(int n)
 {
     uint64_t x;
     uint32_t lx;
@@ -322,6 +332,8 @@ uint16_t rsa_compute_security_bits(int n)
     return (y + 4) & ~7;
 }
 
+
+
 int RSA_security_bits(const RSA *rsa)
 {
     int bits = BN_num_bits(rsa->n);
@@ -335,7 +347,7 @@ int RSA_security_bits(const RSA *rsa)
             return 0;
     }
 #endif
-    return rsa_compute_security_bits(bits);
+    return ifc_ffc_compute_security_bits(bits);
 }
 
 int RSA_set0_key(RSA *r, BIGNUM *n, BIGNUM *e, BIGNUM *d)
diff --git a/crypto/rsa/rsa_local.h b/crypto/rsa/rsa_local.h
index 11d7635c35..ac8856207e 100644
--- a/crypto/rsa/rsa_local.h
+++ b/crypto/rsa/rsa_local.h
@@ -137,8 +137,6 @@ RSA_PRIME_INFO *rsa_multip_info_new(void);
 int rsa_multip_calc_product(RSA *rsa);
 int rsa_multip_cap(int bits);
 
-uint16_t rsa_compute_security_bits(int n);
-
 int rsa_sp800_56b_validate_strength(int nbits, int strength);
 int rsa_check_pminusq_diff(BIGNUM *diff, const BIGNUM *p, const BIGNUM *q,
                            int nbits);
diff --git a/crypto/rsa/rsa_sp800_56b_gen.c b/crypto/rsa/rsa_sp800_56b_gen.c
index 1f8d01d477..a60a428b14 100644
--- a/crypto/rsa/rsa_sp800_56b_gen.c
+++ b/crypto/rsa/rsa_sp800_56b_gen.c
@@ -11,6 +11,7 @@
 #include <openssl/err.h>
 #include <openssl/bn.h>
 #include "crypto/bn.h"
+#include "crypto/security_bits.h"
 #include "rsa_local.h"
 
 #define RSA_FIPS1864_MIN_KEYGEN_KEYSIZE 2048
@@ -144,7 +145,7 @@ err:
  */
 int rsa_sp800_56b_validate_strength(int nbits, int strength)
 {
-    int s = (int)rsa_compute_security_bits(nbits);
+    int s = (int)ifc_ffc_compute_security_bits(nbits);
 
     if (s < RSA_FIPS1864_MIN_KEYGEN_STRENGTH
             || s > RSA_FIPS1864_MAX_KEYGEN_STRENGTH) {