EVP_MD_size() updates

For SHAKE algorithms we now return 0 from EVP_MD_size().
So all the places that check for < 0 needed to change to <= 0
(Otherwise the behaviour will be to digest nothing in most cases).

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/25285)

11 files changed, 20 insertions, 13 deletions

diff --git a/crypto/evp/bio_ok.c b/crypto/evp/bio_ok.c
index 2aa1ed7558..52709c2bde 100644
--- a/crypto/evp/bio_ok.c
+++ b/crypto/evp/bio_ok.c
@@ -443,6 +443,8 @@ static int sig_out(BIO *b)
     md_size = EVP_MD_get_size(digest);
     md_data = EVP_MD_CTX_get0_md_data(md);
 
+    if (md_size <= 0)
+        goto berr;
     if (ctx->buf_len + 2 * md_size > OK_BLOCK_SIZE)
         return 1;
 
@@ -485,7 +487,7 @@ static int sig_in(BIO *b)
     if ((md = ctx->md) == NULL)
         goto berr;
     digest = EVP_MD_CTX_get0_md(md);
-    if ((md_size = EVP_MD_get_size(digest)) < 0)
+    if ((md_size = EVP_MD_get_size(digest)) <= 0)
         goto berr;
     md_data = EVP_MD_CTX_get0_md_data(md);
 
@@ -533,6 +535,8 @@ static int block_out(BIO *b)
     md = ctx->md;
     digest = EVP_MD_CTX_get0_md(md);
     md_size = EVP_MD_get_size(digest);
+    if (md_size <= 0)
+        goto berr;
 
     tl = ctx->buf_len - OK_BLOCK_BLOCK;
     ctx->buf[0] = (unsigned char)(tl >> 24);
@@ -563,7 +567,7 @@ static int block_in(BIO *b)
     ctx = BIO_get_data(b);
     md = ctx->md;
     md_size = EVP_MD_get_size(EVP_MD_CTX_get0_md(md));
-    if (md_size < 0)
+    if (md_size <= 0)
         goto berr;
 
     assert(sizeof(tl) >= OK_BLOCK_BLOCK); /* always true */
diff --git a/crypto/evp/m_sigver.c b/crypto/evp/m_sigver.c
index ca8f6b9953..10027717bf 100644
--- a/crypto/evp/m_sigver.c
+++ b/crypto/evp/m_sigver.c
@@ -601,7 +601,7 @@ int EVP_DigestSignFinal(EVP_MD_CTX *ctx, unsigned char *sigret,
         } else {
             int s = EVP_MD_get_size(ctx->digest);
 
-            if (s < 0 || EVP_PKEY_sign(pctx, sigret, siglen, NULL, s) <= 0)
+            if (s <= 0 || EVP_PKEY_sign(pctx, sigret, siglen, NULL, s) <= 0)
                 return 0;
         }
     }
diff --git a/crypto/evp/p5_crpt.c b/crypto/evp/p5_crpt.c
index f3ac675ff2..91816bf1fd 100644
--- a/crypto/evp/p5_crpt.c
+++ b/crypto/evp/p5_crpt.c
@@ -78,7 +78,7 @@ int PKCS5_PBE_keyivgen_ex(EVP_CIPHER_CTX *cctx, const char *pass, int passlen,
         passlen = strlen(pass);
 
     mdsize = EVP_MD_get_size(md);
-    if (mdsize < 0)
+    if (mdsize <= 0)
         goto err;
 
     kdf = EVP_KDF_fetch(libctx, OSSL_KDF_NAME_PBKDF1, propq);
diff --git a/crypto/ffc/ffc_params_generate.c b/crypto/ffc/ffc_params_generate.c
index 14834e5f7e..8c5b25e23a 100644
--- a/crypto/ffc/ffc_params_generate.c
+++ b/crypto/ffc/ffc_params_generate.c
@@ -322,6 +322,9 @@ static int generate_q_fips186_4(BN_CTX *ctx, BIGNUM *q, const EVP_MD *evpmd,
     unsigned char *pmd;
     OSSL_LIB_CTX *libctx = ossl_bn_get_libctx(ctx);
 
+    if (mdsize <= 0)
+        goto err;
+
     /* find q */
     for (;;) {
         if (!BN_GENCB_call(cb, 0, m++))
diff --git a/crypto/hmac/hmac.c b/crypto/hmac/hmac.c
index 4ea18dfabd..19fc7d3b4f 100644
--- a/crypto/hmac/hmac.c
+++ b/crypto/hmac/hmac.c
@@ -46,7 +46,7 @@ int HMAC_Init_ex(HMAC_CTX *ctx, const void *key, int len,
      * The HMAC construction is not allowed to be used with the
      * extendable-output functions (XOF) shake128 and shake256.
      */
-    if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0)
+    if (EVP_MD_xof(md))
         return 0;
 
 #ifdef OPENSSL_HMAC_S390X
@@ -254,7 +254,7 @@ unsigned char *HMAC(const EVP_MD *evp_md, const void *key, int key_len,
     size_t temp_md_len = 0;
     unsigned char *ret = NULL;
 
-    if (size >= 0) {
+    if (size > 0) {
         ret = EVP_Q_mac(NULL, "HMAC", NULL, EVP_MD_get0_name(evp_md), NULL,
                         key, key_len, data, data_len,
                         md == NULL ? static_md : md, size, &temp_md_len);
diff --git a/crypto/ocsp/ocsp_vfy.c b/crypto/ocsp/ocsp_vfy.c
index b0827e9a22..61be41ae2f 100644
--- a/crypto/ocsp/ocsp_vfy.c
+++ b/crypto/ocsp/ocsp_vfy.c
@@ -328,7 +328,7 @@ static int ocsp_match_issuerid(X509 *cert, OCSP_CERTID *cid,
         (void)ERR_pop_to_mark();
 
         mdlen = EVP_MD_get_size(dgst);
-        if (mdlen < 0) {
+        if (mdlen <= 0) {
             ERR_raise(ERR_LIB_OCSP, OCSP_R_DIGEST_SIZE_ERR);
             goto end;
         }
diff --git a/crypto/pkcs12/p12_mutl.c b/crypto/pkcs12/p12_mutl.c
index d410978a49..62a06357c6 100644
--- a/crypto/pkcs12/p12_mutl.c
+++ b/crypto/pkcs12/p12_mutl.c
@@ -207,7 +207,7 @@ static int pkcs12_gen_mac(PKCS12 *p12, const char *pass, int passlen,
 
     keylen = EVP_MD_get_size(md);
     md_nid = EVP_MD_get_type(md);
-    if (keylen < 0)
+    if (keylen <= 0)
         goto err;
 
     /* For PBMAC1 we use a special keygen callback if not provided (e.g. on verification) */
diff --git a/crypto/rsa/rsa_pss.c b/crypto/rsa/rsa_pss.c
index a8572523a2..6131097292 100644
--- a/crypto/rsa/rsa_pss.c
+++ b/crypto/rsa/rsa_pss.c
@@ -62,7 +62,7 @@ int ossl_rsa_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
         mgf1Hash = Hash;
 
     hLen = EVP_MD_get_size(Hash);
-    if (hLen < 0)
+    if (hLen <= 0)
         goto err;
     /*-
      * Negative sLen has special meanings:
@@ -187,7 +187,7 @@ int ossl_rsa_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM,
         mgf1Hash = Hash;
 
     hLen = EVP_MD_get_size(Hash);
-    if (hLen < 0)
+    if (hLen <= 0)
         goto err;
     /*-
      * Negative sLen has special meanings:
diff --git a/crypto/sm2/sm2_crypt.c b/crypto/sm2/sm2_crypt.c
index b7303af522..0e5017cff6 100644
--- a/crypto/sm2/sm2_crypt.c
+++ b/crypto/sm2/sm2_crypt.c
@@ -91,7 +91,7 @@ int ossl_sm2_ciphertext_size(const EC_KEY *key, const EVP_MD *digest,
     const int md_size = EVP_MD_get_size(digest);
     size_t sz;
 
-    if (field_size == 0 || md_size < 0)
+    if (field_size == 0 || md_size <= 0)
         return 0;
 
     /* Integer and string are simple type; set constructed = 0, means primitive and definite length encoding. */
diff --git a/crypto/sm2/sm2_sign.c b/crypto/sm2/sm2_sign.c
index 1ffbb171fa..248f53f1a6 100644
--- a/crypto/sm2/sm2_sign.c
+++ b/crypto/sm2/sm2_sign.c
@@ -160,7 +160,7 @@ static BIGNUM *sm2_compute_msg_hash(const EVP_MD *digest,
     OSSL_LIB_CTX *libctx = ossl_ec_key_get_libctx(key);
     const char *propq = ossl_ec_key_get0_propq(key);
 
-    if (md_size < 0) {
+    if (md_size <= 0) {
         ERR_raise(ERR_LIB_SM2, SM2_R_INVALID_DIGEST);
         goto done;
     }
diff --git a/crypto/ts/ts_rsp_verify.c b/crypto/ts/ts_rsp_verify.c
index 2dae352d0f..739ff8012f 100644
--- a/crypto/ts/ts_rsp_verify.c
+++ b/crypto/ts/ts_rsp_verify.c
@@ -448,7 +448,7 @@ static int ts_compute_imprint(BIO *data, TS_TST_INFO *tst_info,
     (void)ERR_pop_to_mark();
 
     length = EVP_MD_get_size(md);
-    if (length < 0)
+    if (length <= 0)
         goto err;
     *imprint_len = length;
     if ((*imprint = OPENSSL_malloc(*imprint_len)) == NULL)