Memory allocator code cleanup

Typedefs of CRYPTO malloc, realloc and free.

MEM_CHECK "modes" are used only as a CRYPTO_mem_ctrl() parameter

The CRYPTO_mem_ctrl is defined only if OPENSSL_NO_CRYPTO_MDEBUG is
defined, thus define the MEM_CHECK modes under the same condition.

Maybe the macros can be removed at all since:
1. CRYPTO_mem_ctrl() just returns -1 and ignores the parameter
2. CRYPTO_mem_ctr() is declared as DEPRECATED by 3.0

Reviewed-by: Richard Levitte <levitte@openssl.org>
Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/11042)

5 files changed, 118 insertions, 114 deletions

diff --git a/crypto/mem.c b/crypto/mem.c
index 640107be40..ad64fda5e2 100644
--- a/crypto/mem.c
+++ b/crypto/mem.c
@@ -19,13 +19,9 @@
  * the following pointers may be changed as long as 'allow_customize' is set
  */
 static int allow_customize = 1;
-
-static void *(*malloc_impl)(size_t, const char *, int)
-    = CRYPTO_malloc;
-static void *(*realloc_impl)(void *, size_t, const char *, int)
-    = CRYPTO_realloc;
-static void (*free_impl)(void *, const char *, int)
-    = CRYPTO_free;
+static CRYPTO_malloc_fn malloc_impl = CRYPTO_malloc;
+static CRYPTO_realloc_fn realloc_impl = CRYPTO_realloc;
+static CRYPTO_free_fn free_impl = CRYPTO_free;
 
 #if !defined(OPENSSL_NO_CRYPTO_MDEBUG) && !defined(FIPS_MODE)
 # include "internal/tsan_assist.h"
@@ -52,33 +48,31 @@ static int shouldfail(void);
 # define FAILTEST() /* empty */
 #endif
 
-int CRYPTO_set_mem_functions(
-        void *(*m)(size_t, const char *, int),
-        void *(*r)(void *, size_t, const char *, int),
-        void (*f)(void *, const char *, int))
+int CRYPTO_set_mem_functions(CRYPTO_malloc_fn malloc_fn,
+                             CRYPTO_realloc_fn realloc_fn,
+                             CRYPTO_free_fn free_fn)
 {
     if (!allow_customize)
         return 0;
-    if (m)
-        malloc_impl = m;
-    if (r)
-        realloc_impl = r;
-    if (f)
-        free_impl = f;
+    if (malloc_fn != NULL)
+        malloc_impl = malloc_fn;
+    if (realloc_fn != NULL)
+        realloc_impl = realloc_fn;
+    if (free_fn != NULL)
+        free_impl = free_fn;
     return 1;
 }
 
-void CRYPTO_get_mem_functions(
-        void *(**m)(size_t, const char *, int),
-        void *(**r)(void *, size_t, const char *, int),
-        void (**f)(void *, const char *, int))
+void CRYPTO_get_mem_functions(CRYPTO_malloc_fn *malloc_fn,
+                              CRYPTO_realloc_fn *realloc_fn,
+                              CRYPTO_free_fn *free_fn)
 {
-    if (m != NULL)
-        *m = malloc_impl;
-    if (r != NULL)
-        *r = realloc_impl;
-    if (f != NULL)
-        *f = free_impl;
+    if (malloc_fn != NULL)
+        *malloc_fn = malloc_impl;
+    if (realloc_fn != NULL)
+        *realloc_fn = realloc_impl;
+    if (free_fn != NULL)
+        *free_fn = free_impl;
 }
 
 #if !defined(OPENSSL_NO_CRYPTO_MDEBUG) && !defined(FIPS_MODE)
@@ -170,10 +164,8 @@ void ossl_malloc_setup_failures(void)
 
 void *CRYPTO_malloc(size_t num, const char *file, int line)
 {
-    void *ret = NULL;
-
     INCREMENT(malloc_count);
-    if (malloc_impl != NULL && malloc_impl != CRYPTO_malloc)
+    if (malloc_impl != CRYPTO_malloc)
         return malloc_impl(num, file, line);
 
     if (num == 0)
@@ -188,26 +180,26 @@ void *CRYPTO_malloc(size_t num, const char *file, int line)
          */
         allow_customize = 0;
     }
-    (void)(file); (void)(line);
-    ret = malloc(num);
 
-    return ret;
+    return malloc(num);
 }
 
 void *CRYPTO_zalloc(size_t num, const char *file, int line)
 {
-    void *ret = CRYPTO_malloc(num, file, line);
+    void *ret;
 
+    ret = CRYPTO_malloc(num, file, line);
     FAILTEST();
     if (ret != NULL)
         memset(ret, 0, num);
+
     return ret;
 }
 
 void *CRYPTO_realloc(void *str, size_t num, const char *file, int line)
 {
     INCREMENT(realloc_count);
-    if (realloc_impl != NULL && realloc_impl != &CRYPTO_realloc)
+    if (realloc_impl != CRYPTO_realloc)
         return realloc_impl(str, num, file, line);
 
     FAILTEST();
@@ -219,9 +211,7 @@ void *CRYPTO_realloc(void *str, size_t num, const char *file, int line)
         return NULL;
     }
 
-    (void)(file); (void)(line);
     return realloc(str, num);
-
 }
 
 void *CRYPTO_clear_realloc(void *str, size_t old_len, size_t num,
@@ -254,7 +244,7 @@ void *CRYPTO_clear_realloc(void *str, size_t old_len, size_t num,
 void CRYPTO_free(void *str, const char *file, int line)
 {
     INCREMENT(free_count);
-    if (free_impl != NULL && free_impl != &CRYPTO_free) {
+    if (free_impl != CRYPTO_free) {
         free_impl(str, file, line);
         return;
     }
diff --git a/doc/man3/OPENSSL_malloc.pod b/doc/man3/OPENSSL_malloc.pod
index 8286b4aba8..d0688fdd09 100644
--- a/doc/man3/OPENSSL_malloc.pod
+++ b/doc/man3/OPENSSL_malloc.pod
@@ -12,6 +12,7 @@ CRYPTO_strdup, CRYPTO_strndup,
 OPENSSL_mem_debug_push, OPENSSL_mem_debug_pop,
 CRYPTO_mem_debug_push, CRYPTO_mem_debug_pop,
 CRYPTO_clear_realloc, CRYPTO_clear_free,
+CRYPTO_malloc_fn, CRYPTO_realloc_fn, CRYPTO_free_fn,
 CRYPTO_get_mem_functions, CRYPTO_set_mem_functions,
 CRYPTO_get_alloc_counts,
 CRYPTO_set_mem_debug, CRYPTO_mem_ctrl,
@@ -24,41 +25,43 @@ OPENSSL_MALLOC_FD
 
  #include <openssl/crypto.h>
 
- int OPENSSL_malloc_init(void)
+ int OPENSSL_malloc_init(void);
 
- void *OPENSSL_malloc(size_t num)
- void *OPENSSL_zalloc(size_t num)
- void *OPENSSL_realloc(void *addr, size_t num)
- void OPENSSL_free(void *addr)
- char *OPENSSL_strdup(const char *str)
- char *OPENSSL_strndup(const char *str, size_t s)
+ void *OPENSSL_malloc(size_t num);
+ void *OPENSSL_zalloc(size_t num);
+ void *OPENSSL_realloc(void *addr, size_t num);
+ void OPENSSL_free(void *addr);
+ char *OPENSSL_strdup(const char *str);
+ char *OPENSSL_strndup(const char *str, size_t s);
  size_t OPENSSL_strlcat(char *dst, const char *src, size_t size);
  size_t OPENSSL_strlcpy(char *dst, const char *src, size_t size);
- void *OPENSSL_memdup(void *data, size_t s)
- void *OPENSSL_clear_realloc(void *p, size_t old_len, size_t num)
- void OPENSSL_clear_free(void *str, size_t num)
+ void *OPENSSL_memdup(void *data, size_t s);
+ void *OPENSSL_clear_realloc(void *p, size_t old_len, size_t num);
+ void OPENSSL_clear_free(void *str, size_t num);
  void OPENSSL_cleanse(void *ptr, size_t len);
 
- void *CRYPTO_malloc(size_t num, const char *file, int line)
- void *CRYPTO_zalloc(size_t num, const char *file, int line)
- void *CRYPTO_realloc(void *p, size_t num, const char *file, int line)
- void CRYPTO_free(void *str, const char *, int)
- char *CRYPTO_strdup(const char *p, const char *file, int line)
- char *CRYPTO_strndup(const char *p, size_t num, const char *file, int line)
+ void *CRYPTO_malloc(size_t num, const char *file, int line);
+ void *CRYPTO_zalloc(size_t num, const char *file, int line);
+ void *CRYPTO_realloc(void *p, size_t num, const char *file, int line);
+ void CRYPTO_free(void *str, const char *, int);
+ char *CRYPTO_strdup(const char *p, const char *file, int line);
+ char *CRYPTO_strndup(const char *p, size_t num, const char *file, int line);
  void *CRYPTO_clear_realloc(void *p, size_t old_len, size_t num,
-                            const char *file, int line)
+                            const char *file, int line);
  void CRYPTO_clear_free(void *str, size_t num, const char *, int)
 
- void CRYPTO_get_mem_functions(
-         void *(**m)(size_t, const char *, int),
-         void *(**r)(void *, size_t, const char *, int),
-         void (**f)(void *, const char *, int))
- int CRYPTO_set_mem_functions(
-         void *(*m)(size_t, const char *, int),
-         void *(*r)(void *, size_t, const char *, int),
-         void (*f)(void *, const char *, int))
+ typedef void *(*CRYPTO_malloc_fn)(size_t num, const char *file, int line);
+ typedef void *(*CRYPTO_realloc_fn)(void *addr, size_t num, const char *file,
+                                    int line);
+ typedef void (*CRYPTO_free_fn)(void *addr, const char *file, int line);
+ void CRYPTO_get_mem_functions(CRYPTO_malloc_fn *malloc_fn,
+                               CRYPTO_realloc_fn *realloc_fn,
+                               CRYPTO_free_fn *free_fn);
+ int CRYPTO_set_mem_functions(CRYPTO_malloc_fn malloc_fn,
+                              CRYPTO_realloc_fn realloc_fn,
+                              CRYPTO_free_fn free_fn);
 
- void CRYPTO_get_alloc_counts(int *m, int *r, int *f)
+ void CRYPTO_get_alloc_counts(int *mcount, int *rcount, int *fcount);
 
  env OPENSSL_MALLOC_FAILURES=... <application>
  env OPENSSL_MALLOC_FD=... <application>
@@ -118,7 +121,8 @@ and replace them with alternate versions.
 CRYPTO_get_mem_functions() function fills in the given arguments with the
 function pointers for the current implementations.
 With CRYPTO_set_mem_functions(), you can specify a different set of functions.
-If any of B<m>, B<r>, or B<f> are NULL, then the function is not changed.
+If any of B<malloc_fn>, B<realloc_fn>, or B<free_fn> are NULL, then
+the function is not changed.
 While it's permitted to swap out only a few and not all the functions
 with CRYPTO_set_mem_functions(), it's recommended to swap them all out
 at once.
diff --git a/include/openssl/crypto.h b/include/openssl/crypto.h
index e44dc1b838..0cbed9e78e 100644
--- a/include/openssl/crypto.h
+++ b/include/openssl/crypto.h
@@ -80,44 +80,6 @@ void CRYPTO_THREAD_lock_free(CRYPTO_RWLOCK *lock);
 
 int CRYPTO_atomic_add(int *val, int amount, int *ret, CRYPTO_RWLOCK *lock);
 
-/*
- * The following can be used to detect memory leaks in the library. If
- * used, it turns on malloc checking
- */
-# define CRYPTO_MEM_CHECK_OFF     0x0   /* Control only */
-# define CRYPTO_MEM_CHECK_ON      0x1   /* Control and mode bit */
-# define CRYPTO_MEM_CHECK_ENABLE  0x2   /* Control and mode bit */
-# define CRYPTO_MEM_CHECK_DISABLE 0x3   /* Control only */
-
-struct crypto_ex_data_st {
-    OPENSSL_CTX *ctx;
-    STACK_OF(void) *sk;
-};
-DEFINE_STACK_OF(void)
-
-/*
- * Per class, we have a STACK of function pointers.
- */
-# define CRYPTO_EX_INDEX_SSL              0
-# define CRYPTO_EX_INDEX_SSL_CTX          1
-# define CRYPTO_EX_INDEX_SSL_SESSION      2
-# define CRYPTO_EX_INDEX_X509             3
-# define CRYPTO_EX_INDEX_X509_STORE       4
-# define CRYPTO_EX_INDEX_X509_STORE_CTX   5
-# define CRYPTO_EX_INDEX_DH               6
-# define CRYPTO_EX_INDEX_DSA              7
-# define CRYPTO_EX_INDEX_EC_KEY           8
-# define CRYPTO_EX_INDEX_RSA              9
-# define CRYPTO_EX_INDEX_ENGINE          10
-# define CRYPTO_EX_INDEX_UI              11
-# define CRYPTO_EX_INDEX_BIO             12
-# define CRYPTO_EX_INDEX_APP             13
-# define CRYPTO_EX_INDEX_UI_METHOD       14
-# define CRYPTO_EX_INDEX_RAND_DRBG       15
-# define CRYPTO_EX_INDEX_DRBG            CRYPTO_EX_INDEX_RAND_DRBG
-# define CRYPTO_EX_INDEX_OPENSSL_CTX     16
-# define CRYPTO_EX_INDEX__COUNT          17
-
 /* No longer needed, so this is a no-op */
 #define OPENSSL_malloc_init() while(0) continue
 
@@ -203,6 +165,36 @@ const char *OPENSSL_info(int type);
 
 int OPENSSL_issetugid(void);
 
+struct crypto_ex_data_st {
+    OPENSSL_CTX *ctx;
+    STACK_OF(void) *sk;
+};
+DEFINE_STACK_OF(void)
+
+/*
+ * Per class, we have a STACK of function pointers.
+ */
+# define CRYPTO_EX_INDEX_SSL              0
+# define CRYPTO_EX_INDEX_SSL_CTX          1
+# define CRYPTO_EX_INDEX_SSL_SESSION      2
+# define CRYPTO_EX_INDEX_X509             3
+# define CRYPTO_EX_INDEX_X509_STORE       4
+# define CRYPTO_EX_INDEX_X509_STORE_CTX   5
+# define CRYPTO_EX_INDEX_DH               6
+# define CRYPTO_EX_INDEX_DSA              7
+# define CRYPTO_EX_INDEX_EC_KEY           8
+# define CRYPTO_EX_INDEX_RSA              9
+# define CRYPTO_EX_INDEX_ENGINE          10
+# define CRYPTO_EX_INDEX_UI              11
+# define CRYPTO_EX_INDEX_BIO             12
+# define CRYPTO_EX_INDEX_APP             13
+# define CRYPTO_EX_INDEX_UI_METHOD       14
+# define CRYPTO_EX_INDEX_RAND_DRBG       15
+# define CRYPTO_EX_INDEX_DRBG            CRYPTO_EX_INDEX_RAND_DRBG
+# define CRYPTO_EX_INDEX_OPENSSL_CTX     16
+# define CRYPTO_EX_INDEX_COUNT           17
+# define CRYPTO_EX_INDEX__COUNT          CRYPTO_EX_INDEX_COUNT
+
 typedef void CRYPTO_EX_new (void *parent, void *ptr, CRYPTO_EX_DATA *ad,
                            int idx, long argl, void *argp);
 typedef void CRYPTO_EX_free (void *parent, void *ptr, CRYPTO_EX_DATA *ad,
@@ -210,8 +202,9 @@ typedef void CRYPTO_EX_free (void *parent, void *ptr, CRYPTO_EX_DATA *ad,
 typedef int CRYPTO_EX_dup (CRYPTO_EX_DATA *to, const CRYPTO_EX_DATA *from,
                            void *from_d, int idx, long argl, void *argp);
 __owur int CRYPTO_get_ex_new_index(int class_index, long argl, void *argp,
-                            CRYPTO_EX_new *new_func, CRYPTO_EX_dup *dup_func,
-                            CRYPTO_EX_free *free_func);
+                                   CRYPTO_EX_new *new_func,
+                                   CRYPTO_EX_dup *dup_func,
+                                   CRYPTO_EX_free *free_func);
 /* No longer use an index. */
 int CRYPTO_free_ex_index(int class_index, int idx);
 
@@ -297,14 +290,16 @@ typedef struct crypto_threadid_st {
 #  define CRYPTO_get_dynlock_destroy_callback()         (NULL)
 # endif /* OPENSSL_NO_DEPRECATED_1_1_0 */
 
-int CRYPTO_set_mem_functions(
-        void *(*m) (size_t, const char *, int),
-        void *(*r) (void *, size_t, const char *, int),
-        void (*f) (void *, const char *, int));
-void CRYPTO_get_mem_functions(
-        void *(**m) (size_t, const char *, int),
-        void *(**r) (void *, size_t, const char *, int),
-        void (**f) (void *, const char *, int));
+typedef void *(*CRYPTO_malloc_fn)(size_t num, const char *file, int line);
+typedef void *(*CRYPTO_realloc_fn)(void *addr, size_t num, const char *file,
+                                   int line);
+typedef void (*CRYPTO_free_fn)(void *addr, const char *file, int line);
+int CRYPTO_set_mem_functions(CRYPTO_malloc_fn malloc_fn,
+                             CRYPTO_realloc_fn realloc_fn,
+                             CRYPTO_free_fn free_fn);
+void CRYPTO_get_mem_functions(CRYPTO_malloc_fn *malloc_fn,
+                              CRYPTO_realloc_fn *realloc_fn,
+                              CRYPTO_free_fn *free_fn);
 
 void *CRYPTO_malloc(size_t num, const char *file, int line);
 void *CRYPTO_zalloc(size_t num, const char *file, int line);
@@ -332,6 +327,15 @@ size_t CRYPTO_secure_used(void);
 void OPENSSL_cleanse(void *ptr, size_t len);
 
 # ifndef OPENSSL_NO_CRYPTO_MDEBUG
+/*
+ * The following can be used to detect memory leaks in the library. If
+ * used, it turns on malloc checking
+ */
+# define CRYPTO_MEM_CHECK_OFF     0x0   /* Control only */
+# define CRYPTO_MEM_CHECK_ON      0x1   /* Control and mode bit */
+# define CRYPTO_MEM_CHECK_ENABLE  0x2   /* Control and mode bit */
+# define CRYPTO_MEM_CHECK_DISABLE 0x3   /* Control only */
+
 void CRYPTO_get_alloc_counts(int *mcount, int *rcount, int *fcount);
 #  ifndef OPENSSL_NO_DEPRECATED_3_0
 #    define OPENSSL_mem_debug_push(info) \
@@ -360,7 +364,7 @@ DEPRECATEDIN_3_0(int CRYPTO_mem_leaks_cb(
 DEPRECATEDIN_3_0(int CRYPTO_mem_leaks_fp(FILE *))
 #  endif
 DEPRECATEDIN_3_0(int CRYPTO_mem_leaks(BIO *bio))
-# endif
+# endif /* OPENSSL_NO_CRYPTO_MDEBUG */
 
 /* die if we have to */
 ossl_noreturn void OPENSSL_die(const char *assertion, const char *file, int line);
diff --git a/util/indent.pro b/util/indent.pro
index de260ae6b7..509377d204 100644
--- a/util/indent.pro
+++ b/util/indent.pro
@@ -162,6 +162,9 @@
 -T CONF_METHOD
 -T CONF_MODULE
 -T CONF_VALUE
+-T CRYPTO_malloc_fn
+-T CRYPTO_realloc_fn
+-T CRYPTO_free_fn
 -T CRYPTO_EX_DATA
 -T CRYPTO_EX_dup
 -T CRYPTO_EX_free
diff --git a/util/other.syms b/util/other.syms
index 62a5a7c843..bdcc283718 100644
--- a/util/other.syms
+++ b/util/other.syms
@@ -18,6 +18,9 @@ BIO_callback_fn                         datatype
 BIO_callback_fn_ex                      datatype
 BIO_hostserv_priorities                 datatype
 BIO_lookup_type                         datatype
+CRYPTO_malloc_fn                        datatype
+CRYPTO_realloc_fn                       datatype
+CRYPTO_free_fn                          datatype
 CRYPTO_EX_dup                           datatype
 CRYPTO_EX_free                          datatype
 CRYPTO_EX_new                           datatype