dsa/dsa_pmeth.c: Add the checks for the EVP_MD_CTX_get_size()

Add the checks for the return value of EVP_MD_CTX_get_size() before explicitly cast them to size_t to avoid the integer overflow.

Fixes: 9d04f83410 ("Add DSA digest length checks.")
Signed-off-by: Jiasheng Jiang <jiasheng@purdue.edu>

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Todd Short <todd.short@me.com>
Reviewed-by: Neil Horman <nhorman@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/23954)

1 files changed, 16 insertions, 6 deletions

diff --git a/crypto/dsa/dsa_pmeth.c b/crypto/dsa/dsa_pmeth.c
index ba6be720a2..f7e3f03dca 100644
--- a/crypto/dsa/dsa_pmeth.c
+++ b/crypto/dsa/dsa_pmeth.c
@@ -78,7 +78,7 @@ static int pkey_dsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig,
                          size_t *siglen, const unsigned char *tbs,
                          size_t tbslen)
 {
-    int ret;
+    int ret, md_size;
     unsigned int sltmp;
     DSA_PKEY_CTX *dctx = ctx->data;
     /*
@@ -88,8 +88,13 @@ static int pkey_dsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig,
      */
     DSA *dsa = (DSA *)EVP_PKEY_get0_DSA(ctx->pkey);
 
-    if (dctx->md != NULL && tbslen != (size_t)EVP_MD_get_size(dctx->md))
-        return 0;
+    if (dctx->md != NULL) {
+        md_size = EVP_MD_get_size(dctx->md);
+        if (md_size <= 0)
+            return 0;
+        if (tbslen != (size_t)md_size)
+            return 0;
+    }
 
     ret = DSA_sign(0, tbs, tbslen, sig, &sltmp, dsa);
 
@@ -103,7 +108,7 @@ static int pkey_dsa_verify(EVP_PKEY_CTX *ctx,
                            const unsigned char *sig, size_t siglen,
                            const unsigned char *tbs, size_t tbslen)
 {
-    int ret;
+    int ret, md_size;
     DSA_PKEY_CTX *dctx = ctx->data;
     /*
      * Discard const. Its marked as const because this may be a cached copy of
@@ -112,8 +117,13 @@ static int pkey_dsa_verify(EVP_PKEY_CTX *ctx,
      */
     DSA *dsa = (DSA *)EVP_PKEY_get0_DSA(ctx->pkey);
 
-    if (dctx->md != NULL && tbslen != (size_t)EVP_MD_get_size(dctx->md))
-        return 0;
+    if (dctx->md != NULL) {
+        md_size = EVP_MD_get_size(dctx->md);
+        if (md_size <= 0)
+            return 0;
+        if (tbslen != (size_t)md_size)
+            return 0;
+    }
 
     ret = DSA_verify(0, tbs, tbslen, sig, siglen, dsa);