Stop raising ERR_R_MALLOC_FAILURE in most places

Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)

6 files changed, 26 insertions, 35 deletions

diff --git a/crypto/encode_decode/decoder_lib.c b/crypto/encode_decode/decoder_lib.c
index 8863c316d6..8e19532b8e 100644
--- a/crypto/encode_decode/decoder_lib.c
+++ b/crypto/encode_decode/decoder_lib.c
@@ -224,10 +224,8 @@ OSSL_DECODER_INSTANCE *ossl_decoder_instance_new(OSSL_DECODER *decoder,
         return 0;
     }
 
-    if ((decoder_inst = OPENSSL_zalloc(sizeof(*decoder_inst))) == NULL) {
-        ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_MALLOC_FAILURE);
+    if ((decoder_inst = OPENSSL_zalloc(sizeof(*decoder_inst))) == NULL)
         return 0;
-    }
 
     prov = OSSL_DECODER_get0_provider(decoder);
     libctx = ossl_provider_libctx(prov);
@@ -291,7 +289,7 @@ int ossl_decoder_ctx_add_decoder_inst(OSSL_DECODER_CTX *ctx,
     if (ctx->decoder_insts == NULL
         && (ctx->decoder_insts =
             sk_OSSL_DECODER_INSTANCE_new_null()) == NULL) {
-        ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_CRYPTO_LIB);
         return 0;
     }
 
@@ -508,7 +506,7 @@ int OSSL_DECODER_CTX_add_extra(OSSL_DECODER_CTX *ctx,
 
     skdecoders = sk_OSSL_DECODER_new_null();
     if (skdecoders == NULL) {
-        ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_CRYPTO_LIB);
         return 0;
     }
     OSSL_DECODER_do_all_provided(libctx, collect_all_decoders, skdecoders);
@@ -824,7 +822,7 @@ static int decoder_process(const OSSL_PARAM params[], void *arg)
     }
 
     if ((cbio = ossl_core_bio_new_from_bio(bio)) == NULL) {
-        ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_BIO_LIB);
         goto end;
     }
 
diff --git a/crypto/encode_decode/decoder_meth.c b/crypto/encode_decode/decoder_meth.c
index 74c86a8fe8..b133402e09 100644
--- a/crypto/encode_decode/decoder_meth.c
+++ b/crypto/encode_decode/decoder_meth.c
@@ -29,10 +29,11 @@ static OSSL_DECODER *ossl_decoder_new(void)
 {
     OSSL_DECODER *decoder = NULL;
 
-    if ((decoder = OPENSSL_zalloc(sizeof(*decoder))) == NULL
-        || (decoder->base.lock = CRYPTO_THREAD_lock_new()) == NULL) {
+    if ((decoder = OPENSSL_zalloc(sizeof(*decoder))) == NULL)
+        return NULL;
+    if ((decoder->base.lock = CRYPTO_THREAD_lock_new()) == NULL) {
         OSSL_DECODER_free(decoder);
-        ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_CRYPTO_LIB);
         return NULL;
     }
 
@@ -625,9 +626,7 @@ OSSL_DECODER_CTX *OSSL_DECODER_CTX_new(void)
 {
     OSSL_DECODER_CTX *ctx;
 
-    if ((ctx = OPENSSL_zalloc(sizeof(*ctx))) == NULL)
-        ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_MALLOC_FAILURE);
-
+    ctx = OPENSSL_zalloc(sizeof(*ctx));
     return ctx;
 }
 
diff --git a/crypto/encode_decode/decoder_pkey.c b/crypto/encode_decode/decoder_pkey.c
index fa32f2b9fb..79b120b95c 100644
--- a/crypto/encode_decode/decoder_pkey.c
+++ b/crypto/encode_decode/decoder_pkey.c
@@ -395,17 +395,16 @@ int ossl_decoder_ctx_setup_for_pkey(OSSL_DECODER_CTX *ctx,
     } OSSL_TRACE_END(DECODER);
 
     /* Allocate data. */
-    if ((process_data = OPENSSL_zalloc(sizeof(*process_data))) == NULL
-        || (propquery != NULL
-            && (process_data->propq = OPENSSL_strdup(propquery)) == NULL)) {
-        ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_MALLOC_FAILURE);
+    if ((process_data = OPENSSL_zalloc(sizeof(*process_data))) == NULL)
+        goto err;
+    if ((propquery != NULL
+            && (process_data->propq = OPENSSL_strdup(propquery)) == NULL))
         goto err;
-    }
 
     /* Allocate our list of EVP_KEYMGMTs. */
     keymgmts = sk_EVP_KEYMGMT_new_null();
     if (keymgmts == NULL) {
-        ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_CRYPTO_LIB);
         goto err;
     }
 
@@ -481,7 +480,7 @@ OSSL_DECODER_CTX_new_for_pkey(EVP_PKEY **pkey,
     OSSL_DECODER_CTX *ctx = NULL;
 
     if ((ctx = OSSL_DECODER_CTX_new()) == NULL) {
-        ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_OSSL_DECODER_LIB);
         return NULL;
     }
 
diff --git a/crypto/encode_decode/encoder_lib.c b/crypto/encode_decode/encoder_lib.c
index 7868da79b7..28dae99dc8 100644
--- a/crypto/encode_decode/encoder_lib.c
+++ b/crypto/encode_decode/encoder_lib.c
@@ -193,10 +193,8 @@ static OSSL_ENCODER_INSTANCE *ossl_encoder_instance_new(OSSL_ENCODER *encoder,
         return 0;
     }
 
-    if ((encoder_inst = OPENSSL_zalloc(sizeof(*encoder_inst))) == NULL) {
-        ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_MALLOC_FAILURE);
+    if ((encoder_inst = OPENSSL_zalloc(sizeof(*encoder_inst))) == NULL)
         return 0;
-    }
 
     if (!OSSL_ENCODER_up_ref(encoder)) {
         ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_INTERNAL_ERROR);
@@ -259,7 +257,7 @@ static int ossl_encoder_ctx_add_encoder_inst(OSSL_ENCODER_CTX *ctx,
     if (ctx->encoder_insts == NULL
         && (ctx->encoder_insts =
             sk_OSSL_ENCODER_INSTANCE_new_null()) == NULL) {
-        ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_CRYPTO_LIB);
         return 0;
     }
 
diff --git a/crypto/encode_decode/encoder_meth.c b/crypto/encode_decode/encoder_meth.c
index 7092ba7ef8..9093f29abb 100644
--- a/crypto/encode_decode/encoder_meth.c
+++ b/crypto/encode_decode/encoder_meth.c
@@ -29,10 +29,11 @@ static OSSL_ENCODER *ossl_encoder_new(void)
 {
     OSSL_ENCODER *encoder = NULL;
 
-    if ((encoder = OPENSSL_zalloc(sizeof(*encoder))) == NULL
-        || (encoder->base.lock = CRYPTO_THREAD_lock_new()) == NULL) {
+    if ((encoder = OPENSSL_zalloc(sizeof(*encoder))) == NULL)
+        return NULL;
+    if ((encoder->base.lock = CRYPTO_THREAD_lock_new()) == NULL) {
         OSSL_ENCODER_free(encoder);
-        ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_CRYPTO_LIB);
         return NULL;
     }
 
@@ -609,9 +610,7 @@ OSSL_ENCODER_CTX *OSSL_ENCODER_CTX_new(void)
 {
     OSSL_ENCODER_CTX *ctx;
 
-    if ((ctx = OPENSSL_zalloc(sizeof(*ctx))) == NULL)
-        ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_MALLOC_FAILURE);
-
+    ctx = OPENSSL_zalloc(sizeof(*ctx));
     return ctx;
 }
 
diff --git a/crypto/encode_decode/encoder_pkey.c b/crypto/encode_decode/encoder_pkey.c
index 3a24317cf4..38239f810f 100644
--- a/crypto/encode_decode/encoder_pkey.c
+++ b/crypto/encode_decode/encoder_pkey.c
@@ -243,10 +243,8 @@ static int ossl_encoder_ctx_setup_for_pkey(OSSL_ENCODER_CTX *ctx,
         struct collected_encoder_st encoder_data;
         struct collected_names_st keymgmt_data;
 
-        if ((data = OPENSSL_zalloc(sizeof(*data))) == NULL) {
-            ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_MALLOC_FAILURE);
+        if ((data = OPENSSL_zalloc(sizeof(*data))) == NULL)
             goto err;
-        }
 
         /*
          * Select the first encoder implementations in two steps.
@@ -254,7 +252,7 @@ static int ossl_encoder_ctx_setup_for_pkey(OSSL_ENCODER_CTX *ctx,
          */
         keymgmt_data.names = sk_OPENSSL_CSTRING_new_null();
         if (keymgmt_data.names == NULL) {
-            ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_CRYPTO_LIB);
             goto err;
         }
 
@@ -288,7 +286,7 @@ static int ossl_encoder_ctx_setup_for_pkey(OSSL_ENCODER_CTX *ctx,
 
         sk_OPENSSL_CSTRING_free(keymgmt_data.names);
         if (encoder_data.error_occurred) {
-            ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_CRYPTO_LIB);
             goto err;
         }
     }
@@ -335,7 +333,7 @@ OSSL_ENCODER_CTX *OSSL_ENCODER_CTX_new_for_pkey(const EVP_PKEY *pkey,
     }
 
     if ((ctx = OSSL_ENCODER_CTX_new()) == NULL) {
-        ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_OSSL_ENCODER_LIB);
         return NULL;
     }