EVP_MD_size() updates

For SHAKE algorithms we now return 0 from EVP_MD_size().
So all the places that check for < 0 needed to change to <= 0
(Otherwise the behaviour will be to digest nothing in most cases).

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/25285)

23 files changed, 41 insertions, 31 deletions

diff --git a/apps/ts.c b/apps/ts.c
index 71b8df1997..0bde4fdf51 100644
--- a/apps/ts.c
+++ b/apps/ts.c
@@ -513,7 +513,7 @@ static int create_digest(BIO *input, const char *digest, const EVP_MD *md,
     EVP_MD_CTX *md_ctx = NULL;
 
     md_value_len = EVP_MD_get_size(md);
-    if (md_value_len < 0)
+    if (md_value_len <= 0)
         return 0;
 
     if (input != NULL) {
diff --git a/crypto/evp/bio_ok.c b/crypto/evp/bio_ok.c
index 2aa1ed7558..52709c2bde 100644
--- a/crypto/evp/bio_ok.c
+++ b/crypto/evp/bio_ok.c
@@ -443,6 +443,8 @@ static int sig_out(BIO *b)
     md_size = EVP_MD_get_size(digest);
     md_data = EVP_MD_CTX_get0_md_data(md);
 
+    if (md_size <= 0)
+        goto berr;
     if (ctx->buf_len + 2 * md_size > OK_BLOCK_SIZE)
         return 1;
 
@@ -485,7 +487,7 @@ static int sig_in(BIO *b)
     if ((md = ctx->md) == NULL)
         goto berr;
     digest = EVP_MD_CTX_get0_md(md);
-    if ((md_size = EVP_MD_get_size(digest)) < 0)
+    if ((md_size = EVP_MD_get_size(digest)) <= 0)
         goto berr;
     md_data = EVP_MD_CTX_get0_md_data(md);
 
@@ -533,6 +535,8 @@ static int block_out(BIO *b)
     md = ctx->md;
     digest = EVP_MD_CTX_get0_md(md);
     md_size = EVP_MD_get_size(digest);
+    if (md_size <= 0)
+        goto berr;
 
     tl = ctx->buf_len - OK_BLOCK_BLOCK;
     ctx->buf[0] = (unsigned char)(tl >> 24);
@@ -563,7 +567,7 @@ static int block_in(BIO *b)
     ctx = BIO_get_data(b);
     md = ctx->md;
     md_size = EVP_MD_get_size(EVP_MD_CTX_get0_md(md));
-    if (md_size < 0)
+    if (md_size <= 0)
         goto berr;
 
     assert(sizeof(tl) >= OK_BLOCK_BLOCK); /* always true */
diff --git a/crypto/evp/m_sigver.c b/crypto/evp/m_sigver.c
index ca8f6b9953..10027717bf 100644
--- a/crypto/evp/m_sigver.c
+++ b/crypto/evp/m_sigver.c
@@ -601,7 +601,7 @@ int EVP_DigestSignFinal(EVP_MD_CTX *ctx, unsigned char *sigret,
         } else {
             int s = EVP_MD_get_size(ctx->digest);
 
-            if (s < 0 || EVP_PKEY_sign(pctx, sigret, siglen, NULL, s) <= 0)
+            if (s <= 0 || EVP_PKEY_sign(pctx, sigret, siglen, NULL, s) <= 0)
                 return 0;
         }
     }
diff --git a/crypto/evp/p5_crpt.c b/crypto/evp/p5_crpt.c
index f3ac675ff2..91816bf1fd 100644
--- a/crypto/evp/p5_crpt.c
+++ b/crypto/evp/p5_crpt.c
@@ -78,7 +78,7 @@ int PKCS5_PBE_keyivgen_ex(EVP_CIPHER_CTX *cctx, const char *pass, int passlen,
         passlen = strlen(pass);
 
     mdsize = EVP_MD_get_size(md);
-    if (mdsize < 0)
+    if (mdsize <= 0)
         goto err;
 
     kdf = EVP_KDF_fetch(libctx, OSSL_KDF_NAME_PBKDF1, propq);
diff --git a/crypto/ffc/ffc_params_generate.c b/crypto/ffc/ffc_params_generate.c
index 14834e5f7e..8c5b25e23a 100644
--- a/crypto/ffc/ffc_params_generate.c
+++ b/crypto/ffc/ffc_params_generate.c
@@ -322,6 +322,9 @@ static int generate_q_fips186_4(BN_CTX *ctx, BIGNUM *q, const EVP_MD *evpmd,
     unsigned char *pmd;
     OSSL_LIB_CTX *libctx = ossl_bn_get_libctx(ctx);
 
+    if (mdsize <= 0)
+        goto err;
+
     /* find q */
     for (;;) {
         if (!BN_GENCB_call(cb, 0, m++))
diff --git a/crypto/hmac/hmac.c b/crypto/hmac/hmac.c
index 4ea18dfabd..19fc7d3b4f 100644
--- a/crypto/hmac/hmac.c
+++ b/crypto/hmac/hmac.c
@@ -46,7 +46,7 @@ int HMAC_Init_ex(HMAC_CTX *ctx, const void *key, int len,
      * The HMAC construction is not allowed to be used with the
      * extendable-output functions (XOF) shake128 and shake256.
      */
-    if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0)
+    if (EVP_MD_xof(md))
         return 0;
 
 #ifdef OPENSSL_HMAC_S390X
@@ -254,7 +254,7 @@ unsigned char *HMAC(const EVP_MD *evp_md, const void *key, int key_len,
     size_t temp_md_len = 0;
     unsigned char *ret = NULL;
 
-    if (size >= 0) {
+    if (size > 0) {
         ret = EVP_Q_mac(NULL, "HMAC", NULL, EVP_MD_get0_name(evp_md), NULL,
                         key, key_len, data, data_len,
                         md == NULL ? static_md : md, size, &temp_md_len);
diff --git a/crypto/ocsp/ocsp_vfy.c b/crypto/ocsp/ocsp_vfy.c
index b0827e9a22..61be41ae2f 100644
--- a/crypto/ocsp/ocsp_vfy.c
+++ b/crypto/ocsp/ocsp_vfy.c
@@ -328,7 +328,7 @@ static int ocsp_match_issuerid(X509 *cert, OCSP_CERTID *cid,
         (void)ERR_pop_to_mark();
 
         mdlen = EVP_MD_get_size(dgst);
-        if (mdlen < 0) {
+        if (mdlen <= 0) {
             ERR_raise(ERR_LIB_OCSP, OCSP_R_DIGEST_SIZE_ERR);
             goto end;
         }
diff --git a/crypto/pkcs12/p12_mutl.c b/crypto/pkcs12/p12_mutl.c
index d410978a49..62a06357c6 100644
--- a/crypto/pkcs12/p12_mutl.c
+++ b/crypto/pkcs12/p12_mutl.c
@@ -207,7 +207,7 @@ static int pkcs12_gen_mac(PKCS12 *p12, const char *pass, int passlen,
 
     keylen = EVP_MD_get_size(md);
     md_nid = EVP_MD_get_type(md);
-    if (keylen < 0)
+    if (keylen <= 0)
         goto err;
 
     /* For PBMAC1 we use a special keygen callback if not provided (e.g. on verification) */
diff --git a/crypto/rsa/rsa_pss.c b/crypto/rsa/rsa_pss.c
index a8572523a2..6131097292 100644
--- a/crypto/rsa/rsa_pss.c
+++ b/crypto/rsa/rsa_pss.c
@@ -62,7 +62,7 @@ int ossl_rsa_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
         mgf1Hash = Hash;
 
     hLen = EVP_MD_get_size(Hash);
-    if (hLen < 0)
+    if (hLen <= 0)
         goto err;
     /*-
      * Negative sLen has special meanings:
@@ -187,7 +187,7 @@ int ossl_rsa_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM,
         mgf1Hash = Hash;
 
     hLen = EVP_MD_get_size(Hash);
-    if (hLen < 0)
+    if (hLen <= 0)
         goto err;
     /*-
      * Negative sLen has special meanings:
diff --git a/crypto/sm2/sm2_crypt.c b/crypto/sm2/sm2_crypt.c
index b7303af522..0e5017cff6 100644
--- a/crypto/sm2/sm2_crypt.c
+++ b/crypto/sm2/sm2_crypt.c
@@ -91,7 +91,7 @@ int ossl_sm2_ciphertext_size(const EC_KEY *key, const EVP_MD *digest,
     const int md_size = EVP_MD_get_size(digest);
     size_t sz;
 
-    if (field_size == 0 || md_size < 0)
+    if (field_size == 0 || md_size <= 0)
         return 0;
 
     /* Integer and string are simple type; set constructed = 0, means primitive and definite length encoding. */
diff --git a/crypto/sm2/sm2_sign.c b/crypto/sm2/sm2_sign.c
index 1ffbb171fa..248f53f1a6 100644
--- a/crypto/sm2/sm2_sign.c
+++ b/crypto/sm2/sm2_sign.c
@@ -160,7 +160,7 @@ static BIGNUM *sm2_compute_msg_hash(const EVP_MD *digest,
     OSSL_LIB_CTX *libctx = ossl_ec_key_get_libctx(key);
     const char *propq = ossl_ec_key_get0_propq(key);
 
-    if (md_size < 0) {
+    if (md_size <= 0) {
         ERR_raise(ERR_LIB_SM2, SM2_R_INVALID_DIGEST);
         goto done;
     }
diff --git a/crypto/ts/ts_rsp_verify.c b/crypto/ts/ts_rsp_verify.c
index 2dae352d0f..739ff8012f 100644
--- a/crypto/ts/ts_rsp_verify.c
+++ b/crypto/ts/ts_rsp_verify.c
@@ -448,7 +448,7 @@ static int ts_compute_imprint(BIO *data, TS_TST_INFO *tst_info,
     (void)ERR_pop_to_mark();
 
     length = EVP_MD_get_size(md);
-    if (length < 0)
+    if (length <= 0)
         goto err;
     *imprint_len = length;
     if ((*imprint = OPENSSL_malloc(*imprint_len)) == NULL)
diff --git a/providers/implementations/kdfs/hkdf.c b/providers/implementations/kdfs/hkdf.c
index 587a9f43a8..624f85a9e4 100644
--- a/providers/implementations/kdfs/hkdf.c
+++ b/providers/implementations/kdfs/hkdf.c
@@ -186,7 +186,7 @@ static size_t kdf_hkdf_size(KDF_HKDF *ctx)
         return 0;
     }
     sz = EVP_MD_get_size(md);
-    if (sz < 0)
+    if (sz <= 0)
         return 0;
 
     return sz;
@@ -266,7 +266,7 @@ static int hkdf_common_set_ctx_params(KDF_HKDF *ctx, const OSSL_PARAM params[])
             return 0;
 
         md = ossl_prov_digest_md(&ctx->digest);
-        if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0) {
+        if (EVP_MD_xof(md)) {
             ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED);
             return 0;
         }
@@ -463,7 +463,7 @@ static int HKDF(OSSL_LIB_CTX *libctx, const EVP_MD *evp_md,
     size_t prk_len;
 
     sz = EVP_MD_get_size(evp_md);
-    if (sz < 0)
+    if (sz <= 0)
         return 0;
     prk_len = (size_t)sz;
 
@@ -510,7 +510,7 @@ static int HKDF_Extract(OSSL_LIB_CTX *libctx, const EVP_MD *evp_md,
 {
     int sz = EVP_MD_get_size(evp_md);
 
-    if (sz < 0)
+    if (sz <= 0)
         return 0;
     if (prk_len != (size_t)sz) {
         ERR_raise(ERR_LIB_PROV, PROV_R_WRONG_OUTPUT_BUFFER_SIZE);
diff --git a/providers/implementations/kdfs/pbkdf1.c b/providers/implementations/kdfs/pbkdf1.c
index 69d3f7cb29..1b7e4d8a2e 100644
--- a/providers/implementations/kdfs/pbkdf1.c
+++ b/providers/implementations/kdfs/pbkdf1.c
@@ -70,7 +70,7 @@ static int kdf_pbkdf1_do_derive(const unsigned char *pass, size_t passlen,
         || !EVP_DigestFinal_ex(ctx, md_tmp, NULL))
         goto err;
     mdsize = EVP_MD_size(md_type);
-    if (mdsize < 0)
+    if (mdsize <= 0)
         goto err;
     if (n > (size_t)mdsize) {
         ERR_raise(ERR_LIB_PROV, PROV_R_LENGTH_TOO_LARGE);
diff --git a/ssl/record/methods/dtls_meth.c b/ssl/record/methods/dtls_meth.c
index a5e6c82341..a69629b07b 100644
--- a/ssl/record/methods/dtls_meth.c
+++ b/ssl/record/methods/dtls_meth.c
@@ -151,7 +151,7 @@ static int dtls_process_record(OSSL_RECORD_LAYER *rl, DTLS_BITMAP *bitmap)
 
         if (tmpmd != NULL) {
             imac_size = EVP_MD_get_size(tmpmd);
-            if (!ossl_assert(imac_size >= 0 && imac_size <= EVP_MAX_MD_SIZE)) {
+            if (!ossl_assert(imac_size > 0 && imac_size <= EVP_MAX_MD_SIZE)) {
                 RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB);
                 return 0;
             }
diff --git a/ssl/record/methods/tls_common.c b/ssl/record/methods/tls_common.c
index 6f98518048..175086ee17 100644
--- a/ssl/record/methods/tls_common.c
+++ b/ssl/record/methods/tls_common.c
@@ -73,7 +73,7 @@ int ossl_set_tls_provider_parameters(OSSL_RECORD_LAYER *rl,
     if ((EVP_CIPHER_get_flags(ciph) & EVP_CIPH_FLAG_AEAD_CIPHER) == 0
             && !rl->use_etm)
         imacsize = EVP_MD_get_size(md);
-    if (imacsize >= 0)
+    if (imacsize > 0)
         macsize = (size_t)imacsize;
 
     *pprm++ = OSSL_PARAM_construct_int(OSSL_CIPHER_PARAM_TLS_VERSION,
@@ -773,7 +773,7 @@ int tls_get_more_records(OSSL_RECORD_LAYER *rl)
 
         if (tmpmd != NULL) {
             imac_size = EVP_MD_get_size(tmpmd);
-            if (!ossl_assert(imac_size >= 0 && imac_size <= EVP_MAX_MD_SIZE)) {
+            if (!ossl_assert(imac_size > 0 && imac_size <= EVP_MAX_MD_SIZE)) {
                 RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB);
                 return OSSL_RECORD_RETURN_FATAL;
             }
diff --git a/ssl/s3_enc.c b/ssl/s3_enc.c
index 878556b069..cda1f7f83b 100644
--- a/ssl/s3_enc.c
+++ b/ssl/s3_enc.c
@@ -113,7 +113,7 @@ int ssl3_change_cipher_state(SSL_CONNECTION *s, int which)
 
     p = s->s3.tmp.key_block;
     mdi = EVP_MD_get_size(md);
-    if (mdi < 0) {
+    if (mdi <= 0) {
         SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR);
         goto err;
     }
@@ -188,7 +188,7 @@ int ssl3_setup_key_block(SSL_CONNECTION *s)
 #endif
 
     num = EVP_MD_get_size(hash);
-    if (num < 0)
+    if (num <= 0)
         return 0;
 
     num = EVP_CIPHER_get_key_length(c) + num + EVP_CIPHER_get_iv_length(c);
diff --git a/ssl/ssl_ciph.c b/ssl/ssl_ciph.c
index ce6d0d99a2..e5d6237176 100644
--- a/ssl/ssl_ciph.c
+++ b/ssl/ssl_ciph.c
@@ -338,7 +338,8 @@ int ssl_load_ciphers(SSL_CTX *ctx)
             ctx->disabled_mac_mask |= t->mask;
         } else {
             int tmpsize = EVP_MD_get_size(md);
-            if (!ossl_assert(tmpsize >= 0))
+
+            if (!ossl_assert(tmpsize > 0))
                 return 0;
             ctx->ssl_mac_secret_size[i] = tmpsize;
         }
diff --git a/ssl/statem/extensions.c b/ssl/statem/extensions.c
index e0e25afcb6..d963425562 100644
--- a/ssl/statem/extensions.c
+++ b/ssl/statem/extensions.c
@@ -1556,7 +1556,7 @@ int tls_psk_do_binder(SSL_CONNECTION *s, const EVP_MD *md,
     SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s);
 
     /* Ensure cast to size_t is safe */
-    if (!ossl_assert(hashsizei >= 0)) {
+    if (!ossl_assert(hashsizei > 0)) {
         SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR);
         goto err;
     }
diff --git a/ssl/statem/statem_clnt.c b/ssl/statem/statem_clnt.c
index df5a099235..80a997a73c 100644
--- a/ssl/statem/statem_clnt.c
+++ b/ssl/statem/statem_clnt.c
@@ -2829,7 +2829,7 @@ MSG_PROCESS_RETURN tls_process_new_session_ticket(SSL_CONNECTION *s,
         static const unsigned char nonce_label[] = "resumption";
 
         /* Ensure cast to size_t is safe */
-        if (!ossl_assert(hashleni >= 0)) {
+        if (!ossl_assert(hashleni > 0)) {
             SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR);
             goto err;
         }
diff --git a/ssl/statem/statem_srvr.c b/ssl/statem/statem_srvr.c
index 08544ed0bf..b0a6bc42ee 100644
--- a/ssl/statem/statem_srvr.c
+++ b/ssl/statem/statem_srvr.c
@@ -4175,7 +4175,7 @@ CON_FUNC_RETURN tls_construct_new_session_ticket(SSL_CONNECTION *s, WPACKET *pkt
         int hashleni = EVP_MD_get_size(md);
 
         /* Ensure cast to size_t is safe */
-        if (!ossl_assert(hashleni >= 0)) {
+        if (!ossl_assert(hashleni > 0)) {
             SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR);
             goto err;
         }
diff --git a/ssl/tls13_enc.c b/ssl/tls13_enc.c
index f6b4b9f4c2..b89099bef8 100644
--- a/ssl/tls13_enc.c
+++ b/ssl/tls13_enc.c
@@ -188,7 +188,7 @@ int tls13_generate_secret(SSL_CONNECTION *s, const EVP_MD *md,
 
     mdleni = EVP_MD_get_size(md);
     /* Ensure cast to size_t is safe */
-    if (!ossl_assert(mdleni >= 0)) {
+    if (!ossl_assert(mdleni > 0)) {
         SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR);
         EVP_KDF_CTX_free(kctx);
         return 0;
@@ -361,7 +361,7 @@ static int derive_secret_key_and_iv(SSL_CONNECTION *s, const EVP_MD *md,
     int mode, mac_mdleni;
 
     /* Ensure cast to size_t is safe */
-    if (!ossl_assert(hashleni >= 0)) {
+    if (!ossl_assert(hashleni > 0)) {
         SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB);
         return 0;
     }
@@ -379,7 +379,7 @@ static int derive_secret_key_and_iv(SSL_CONNECTION *s, const EVP_MD *md,
         && mac_type == NID_hmac) {
         mac_mdleni = EVP_MD_get_size(mac_md);
 
-        if (mac_mdleni < 0) {
+        if (mac_mdleni <= 0) {
             SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR);
             return 0;
         }
diff --git a/util/libcrypto.num b/util/libcrypto.num
index 2c485fb153..f2f925d09a 100644
--- a/util/libcrypto.num
+++ b/util/libcrypto.num
@@ -5726,6 +5726,8 @@ EVP_PKEY_verify_message_init            ?	3_4_0	EXIST::FUNCTION:
 EVP_PKEY_verify_message_update          ?	3_4_0	EXIST::FUNCTION:
 EVP_PKEY_verify_message_final           ?	3_4_0	EXIST::FUNCTION:
 EVP_PKEY_verify_recover_init_ex2        ?	3_4_0	EXIST::FUNCTION:
+EVP_MD_xof                              ?	3_4_0	EXIST::FUNCTION:
+EVP_MD_CTX_get_size_ex                  ?	3_4_0	EXIST::FUNCTION:
 EVP_CIPHER_CTX_set_algor_params         ?	3_4_0	EXIST::FUNCTION:
 EVP_CIPHER_CTX_get_algor_params         ?	3_4_0	EXIST::FUNCTION:
 EVP_CIPHER_CTX_get_algor                ?	3_4_0	EXIST::FUNCTION: