Stop raising ERR_R_MALLOC_FAILURE in most places

Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)

14 files changed, 40 insertions, 74 deletions

diff --git a/crypto/rsa/rsa_ameth.c b/crypto/rsa/rsa_ameth.c
index 9d5c32776d..03bbeecee0 100644
--- a/crypto/rsa/rsa_ameth.c
+++ b/crypto/rsa/rsa_ameth.c
@@ -151,14 +151,14 @@ static int rsa_priv_encode(PKCS8_PRIV_KEY_INFO *p8, const EVP_PKEY *pkey)
     rklen = i2d_RSAPrivateKey(pkey->pkey.rsa, &rk);
 
     if (rklen <= 0) {
-        ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_RSA, ERR_R_ASN1_LIB);
         ASN1_STRING_free(str);
         return 0;
     }
 
     if (!PKCS8_pkey_set0(p8, OBJ_nid2obj(pkey->ameth->pkey_id), 0,
                          strtype, str, rk, rklen)) {
-        ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_RSA, ERR_R_ASN1_LIB);
         ASN1_STRING_free(str);
         OPENSSL_clear_free(rk, rklen);
         return 0;
@@ -799,7 +799,7 @@ static int rsa_int_import_from(const OSSL_PARAM params[], void *vpctx,
     int ok = 0;
 
     if (rsa == NULL) {
-        ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_DH, ERR_R_RSA_LIB);
         return 0;
     }
 
diff --git a/crypto/rsa/rsa_backend.c b/crypto/rsa/rsa_backend.c
index bc658d9d30..c416d4bf61 100644
--- a/crypto/rsa/rsa_backend.c
+++ b/crypto/rsa/rsa_backend.c
@@ -401,10 +401,8 @@ RSA *ossl_rsa_dup(const RSA *rsa, int selection)
             const RSA_PRIME_INFO *pinfo = NULL;
             RSA_PRIME_INFO *duppinfo = NULL;
 
-            if ((duppinfo = OPENSSL_zalloc(sizeof(*duppinfo))) == NULL) {
-                ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+            if ((duppinfo = OPENSSL_zalloc(sizeof(*duppinfo))) == NULL)
                 goto err;
-            }
             /* push first so cleanup in error case works */
             (void)sk_RSA_PRIME_INFO_push(dupkey->prime_infos, duppinfo);
 
diff --git a/crypto/rsa/rsa_chk.c b/crypto/rsa/rsa_chk.c
index 8ab1166874..f2fc89285b 100644
--- a/crypto/rsa/rsa_chk.c
+++ b/crypto/rsa/rsa_chk.c
@@ -51,7 +51,7 @@ static int rsa_validate_keypair_multiprime(const RSA *key, BN_GENCB *cb)
     if (i == NULL || j == NULL || k == NULL || l == NULL
             || m == NULL || ctx == NULL) {
         ret = -1;
-        ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_RSA, ERR_R_BN_LIB);
         goto err;
     }
 
diff --git a/crypto/rsa/rsa_crpt.c b/crypto/rsa/rsa_crpt.c
index 6bc6aafcc8..21c922e609 100644
--- a/crypto/rsa/rsa_crpt.c
+++ b/crypto/rsa/rsa_crpt.c
@@ -129,7 +129,7 @@ BN_BLINDING *RSA_setup_blinding(RSA *rsa, BN_CTX *in_ctx)
     BN_CTX_start(ctx);
     e = BN_CTX_get(ctx);
     if (e == NULL) {
-        ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_RSA, ERR_R_BN_LIB);
         goto err;
     }
 
@@ -147,7 +147,7 @@ BN_BLINDING *RSA_setup_blinding(RSA *rsa, BN_CTX *in_ctx)
         BIGNUM *n = BN_new();
 
         if (n == NULL) {
-            ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_RSA, ERR_R_BN_LIB);
             goto err;
         }
         BN_with_flags(n, rsa->n, BN_FLG_CONSTTIME);
diff --git a/crypto/rsa/rsa_lib.c b/crypto/rsa/rsa_lib.c
index 9588a75964..fa3561c33f 100644
--- a/crypto/rsa/rsa_lib.c
+++ b/crypto/rsa/rsa_lib.c
@@ -76,15 +76,13 @@ static RSA *rsa_new_intern(ENGINE *engine, OSSL_LIB_CTX *libctx)
 {
     RSA *ret = OPENSSL_zalloc(sizeof(*ret));
 
-    if (ret == NULL) {
-        ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+    if (ret == NULL)
         return NULL;
-    }
 
     ret->references = 1;
     ret->lock = CRYPTO_THREAD_lock_new();
     if (ret->lock == NULL) {
-        ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_RSA, ERR_R_CRYPTO_LIB);
         OPENSSL_free(ret);
         return NULL;
     }
@@ -787,10 +785,8 @@ int ossl_rsa_set0_all_params(RSA *r, const STACK_OF(BIGNUM) *primes,
                 goto err;
 
             /* Using ossl_rsa_multip_info_new() is wasteful, so allocate directly */
-            if ((pinfo = OPENSSL_zalloc(sizeof(*pinfo))) == NULL) {
-                ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+            if ((pinfo = OPENSSL_zalloc(sizeof(*pinfo))) == NULL)
                 goto err;
-            }
 
             pinfo->r = prime;
             pinfo->d = exp;
diff --git a/crypto/rsa/rsa_meth.c b/crypto/rsa/rsa_meth.c
index 82f13bb359..f04098bd08 100644
--- a/crypto/rsa/rsa_meth.c
+++ b/crypto/rsa/rsa_meth.c
@@ -31,7 +31,6 @@ RSA_METHOD *RSA_meth_new(const char *name, int flags)
         OPENSSL_free(meth);
     }
 
-    ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
     return NULL;
 }
 
@@ -57,7 +56,6 @@ RSA_METHOD *RSA_meth_dup(const RSA_METHOD *meth)
         OPENSSL_free(ret);
     }
 
-    ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
     return NULL;
 }
 
@@ -70,10 +68,8 @@ int RSA_meth_set1_name(RSA_METHOD *meth, const char *name)
 {
     char *tmpname = OPENSSL_strdup(name);
 
-    if (tmpname == NULL) {
-        ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+    if (tmpname == NULL)
         return 0;
-    }
 
     OPENSSL_free(meth->name);
     meth->name = tmpname;
diff --git a/crypto/rsa/rsa_mp.c b/crypto/rsa/rsa_mp.c
index b785344cf0..45e2280944 100644
--- a/crypto/rsa/rsa_mp.c
+++ b/crypto/rsa/rsa_mp.c
@@ -33,10 +33,8 @@ RSA_PRIME_INFO *ossl_rsa_multip_info_new(void)
     RSA_PRIME_INFO *pinfo;
 
     /* create a RSA_PRIME_INFO structure */
-    if ((pinfo = OPENSSL_zalloc(sizeof(RSA_PRIME_INFO))) == NULL) {
-        ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+    if ((pinfo = OPENSSL_zalloc(sizeof(RSA_PRIME_INFO))) == NULL)
         return NULL;
-    }
     if ((pinfo->r = BN_secure_new()) == NULL)
         goto err;
     if ((pinfo->d = BN_secure_new()) == NULL)
diff --git a/crypto/rsa/rsa_oaep.c b/crypto/rsa/rsa_oaep.c
index d9be1a4f98..b9030440c4 100644
--- a/crypto/rsa/rsa_oaep.c
+++ b/crypto/rsa/rsa_oaep.c
@@ -112,10 +112,8 @@ int ossl_rsa_padding_add_PKCS1_OAEP_mgf1_ex(OSSL_LIB_CTX *libctx,
 
     dbmask_len = emlen - mdlen;
     dbmask = OPENSSL_malloc(dbmask_len);
-    if (dbmask == NULL) {
-        ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+    if (dbmask == NULL)
         goto err;
-    }
 
     /* step 3e: dbMask = MGF(mgfSeed, nLen - HLen - 1) */
     if (PKCS1_MGF1(dbmask, dbmask_len, seed, mdlen, mgf1md) < 0)
@@ -203,16 +201,12 @@ int RSA_padding_check_PKCS1_OAEP_mgf1(unsigned char *to, int tlen,
 
     dblen = num - mdlen - 1;
     db = OPENSSL_malloc(dblen);
-    if (db == NULL) {
-        ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+    if (db == NULL)
         goto cleanup;
-    }
 
     em = OPENSSL_malloc(num);
-    if (em == NULL) {
-        ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+    if (em == NULL)
         goto cleanup;
-    }
 
     /*
      * Caller is encouraged to pass zero-padded message created with
diff --git a/crypto/rsa/rsa_ossl.c b/crypto/rsa/rsa_ossl.c
index 381c659352..54e2a1c61c 100644
--- a/crypto/rsa/rsa_ossl.c
+++ b/crypto/rsa/rsa_ossl.c
@@ -104,10 +104,8 @@ static int rsa_ossl_public_encrypt(int flen, const unsigned char *from,
     ret = BN_CTX_get(ctx);
     num = BN_num_bytes(rsa->n);
     buf = OPENSSL_malloc(num);
-    if (ret == NULL || buf == NULL) {
-        ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+    if (ret == NULL || buf == NULL)
         goto err;
-    }
 
     switch (padding) {
     case RSA_PKCS1_PADDING:
@@ -261,10 +259,8 @@ static int rsa_ossl_private_encrypt(int flen, const unsigned char *from,
     ret = BN_CTX_get(ctx);
     num = BN_num_bytes(rsa->n);
     buf = OPENSSL_malloc(num);
-    if (ret == NULL || buf == NULL) {
-        ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+    if (ret == NULL || buf == NULL)
         goto err;
-    }
 
     switch (padding) {
     case RSA_PKCS1_PADDING:
@@ -307,7 +303,7 @@ static int rsa_ossl_private_encrypt(int flen, const unsigned char *from,
 
     if (blinding != NULL) {
         if (!local_blinding && ((unblind = BN_CTX_get(ctx)) == NULL)) {
-            ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_RSA, ERR_R_BN_LIB);
             goto err;
         }
         if (!rsa_blinding_convert(blinding, f, unblind, ctx))
@@ -324,7 +320,7 @@ static int rsa_ossl_private_encrypt(int flen, const unsigned char *from,
     } else {
         BIGNUM *d = BN_new();
         if (d == NULL) {
-            ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_RSA, ERR_R_BN_LIB);
             goto err;
         }
         if (rsa->d == NULL) {
@@ -391,12 +387,14 @@ static int rsa_ossl_private_decrypt(int flen, const unsigned char *from,
     BN_CTX_start(ctx);
     f = BN_CTX_get(ctx);
     ret = BN_CTX_get(ctx);
+    if (ret == NULL) {
+        ERR_raise(ERR_LIB_RSA, ERR_R_BN_LIB);
+        goto err;
+    }
     num = BN_num_bytes(rsa->n);
     buf = OPENSSL_malloc(num);
-    if (ret == NULL || buf == NULL) {
-        ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+    if (buf == NULL)
         goto err;
-    }
 
     /*
      * This check was for equality but PGP does evil things and chops off the
@@ -426,7 +424,7 @@ static int rsa_ossl_private_decrypt(int flen, const unsigned char *from,
 
     if (blinding != NULL) {
         if (!local_blinding && ((unblind = BN_CTX_get(ctx)) == NULL)) {
-            ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_RSA, ERR_R_BN_LIB);
             goto err;
         }
         if (!rsa_blinding_convert(blinding, f, unblind, ctx))
@@ -444,7 +442,7 @@ static int rsa_ossl_private_decrypt(int flen, const unsigned char *from,
     } else {
         BIGNUM *d = BN_new();
         if (d == NULL) {
-            ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_RSA, ERR_R_BN_LIB);
             goto err;
         }
         if (rsa->d == NULL) {
@@ -540,12 +538,14 @@ static int rsa_ossl_public_decrypt(int flen, const unsigned char *from,
     BN_CTX_start(ctx);
     f = BN_CTX_get(ctx);
     ret = BN_CTX_get(ctx);
+    if (ret == NULL) {
+        ERR_raise(ERR_LIB_RSA, ERR_R_BN_LIB);
+        goto err;
+    }
     num = BN_num_bytes(rsa->n);
     buf = OPENSSL_malloc(num);
-    if (ret == NULL || buf == NULL) {
-        ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+    if (buf == NULL)
         goto err;
-    }
 
     /*
      * This check was for equality but PGP does evil things and chops off the
diff --git a/crypto/rsa/rsa_pk1.c b/crypto/rsa/rsa_pk1.c
index 51507fc030..5f72fe1735 100644
--- a/crypto/rsa/rsa_pk1.c
+++ b/crypto/rsa/rsa_pk1.c
@@ -188,10 +188,8 @@ int RSA_padding_check_PKCS1_type_2(unsigned char *to, int tlen,
     }
 
     em = OPENSSL_malloc(num);
-    if (em == NULL) {
-        ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+    if (em == NULL)
         return -1;
-    }
     /*
      * Caller is encouraged to pass zero-padded message created with
      * BN_bn2binpad. Trouble is that since we can't read out of |from|'s
diff --git a/crypto/rsa/rsa_pmeth.c b/crypto/rsa/rsa_pmeth.c
index 1eebfea2cc..8b35e5c3c6 100644
--- a/crypto/rsa/rsa_pmeth.c
+++ b/crypto/rsa/rsa_pmeth.c
@@ -112,10 +112,8 @@ static int setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk)
     if (ctx->tbuf != NULL)
         return 1;
     if ((ctx->tbuf =
-            OPENSSL_malloc(RSA_size(EVP_PKEY_get0_RSA(pk->pkey)))) == NULL) {
-        ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+            OPENSSL_malloc(RSA_size(EVP_PKEY_get0_RSA(pk->pkey)))) == NULL)
         return 0;
-    }
     return 1;
 }
 
@@ -164,7 +162,7 @@ static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig,
                 return -1;
             }
             if (!setup_tbuf(rctx, ctx)) {
-                ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+                ERR_raise(ERR_LIB_RSA, ERR_R_RSA_LIB);
                 return -1;
             }
             memcpy(rctx->tbuf, tbs, tbslen);
diff --git a/crypto/rsa/rsa_pss.c b/crypto/rsa/rsa_pss.c
index 33874bfef8..0b408d9bfb 100644
--- a/crypto/rsa/rsa_pss.c
+++ b/crypto/rsa/rsa_pss.c
@@ -97,10 +97,8 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
     maskedDBLen = emLen - hLen - 1;
     H = EM + maskedDBLen;
     DB = OPENSSL_malloc(maskedDBLen);
-    if (DB == NULL) {
-        ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+    if (DB == NULL)
         goto err;
-    }
     if (PKCS1_MGF1(DB, maskedDBLen, H, hLen, mgf1Hash) < 0)
         goto err;
     for (i = 0; i < maskedDBLen; i++)
@@ -201,10 +199,8 @@ int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM,
     }
     if (sLen > 0) {
         salt = OPENSSL_malloc(sLen);
-        if (salt == NULL) {
-            ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+        if (salt == NULL)
             goto err;
-        }
         if (RAND_bytes_ex(rsa->libctx, salt, sLen, 0) <= 0)
             goto err;
     }
diff --git a/crypto/rsa/rsa_saos.c b/crypto/rsa/rsa_saos.c
index 58fa50785b..dc96b6dada 100644
--- a/crypto/rsa/rsa_saos.c
+++ b/crypto/rsa/rsa_saos.c
@@ -40,10 +40,8 @@ int RSA_sign_ASN1_OCTET_STRING(int type,
         return 0;
     }
     s = OPENSSL_malloc((unsigned int)j + 1);
-    if (s == NULL) {
-        ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+    if (s == NULL)
         return 0;
-    }
     p = s;
     i2d_ASN1_OCTET_STRING(&sig, &p);
     i = RSA_private_encrypt(i, s, sigret, rsa, RSA_PKCS1_PADDING);
@@ -72,10 +70,8 @@ int RSA_verify_ASN1_OCTET_STRING(int dtype,
     }
 
     s = OPENSSL_malloc((unsigned int)siglen);
-    if (s == NULL) {
-        ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+    if (s == NULL)
         goto err;
-    }
     i = RSA_public_decrypt((int)siglen, sigbuf, s, rsa, RSA_PKCS1_PADDING);
 
     if (i <= 0)
diff --git a/crypto/rsa/rsa_sign.c b/crypto/rsa/rsa_sign.c
index c5a664dc0b..cec4d639e3 100644
--- a/crypto/rsa/rsa_sign.c
+++ b/crypto/rsa/rsa_sign.c
@@ -258,10 +258,8 @@ static int encode_pkcs1(unsigned char **out, size_t *out_len, int type,
     }
     dig_info_len = di_prefix_len + m_len;
     dig_info = OPENSSL_malloc(dig_info_len);
-    if (dig_info == NULL) {
-        ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+    if (dig_info == NULL)
         return 0;
-    }
     memcpy(dig_info, di_prefix, di_prefix_len);
     memcpy(dig_info + di_prefix_len, m, m_len);
 
@@ -343,10 +341,8 @@ int ossl_rsa_verify(int type, const unsigned char *m, unsigned int m_len,
 
     /* Recover the encoded digest. */
     decrypt_buf = OPENSSL_malloc(siglen);
-    if (decrypt_buf == NULL) {
-        ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
+    if (decrypt_buf == NULL)
         goto err;
-    }
 
     len = RSA_public_decrypt((int)siglen, sigbuf, decrypt_buf, rsa,
                              RSA_PKCS1_PADDING);