Update code to use EVP_MD_xof()

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/25285)

17 files changed, 21 insertions, 21 deletions

diff --git a/apps/dgst.c b/apps/dgst.c
index 118754c4db..818139f4e1 100644
--- a/apps/dgst.c
+++ b/apps/dgst.c
@@ -419,7 +419,7 @@ int dgst_main(int argc, char **argv)
         md_name = EVP_MD_get0_name(md);
 
     if (xoflen > 0) {
-        if (!(EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF)) {
+        if (!EVP_MD_xof(md)) {
             BIO_printf(bio_err, "Length can only be specified for XOF\n");
             goto end;
         }
diff --git a/apps/speed.c b/apps/speed.c
index 48f91b2213..0079fd7c30 100644
--- a/apps/speed.c
+++ b/apps/speed.c
@@ -631,7 +631,7 @@ static int EVP_Digest_loop(const char *mdname, ossl_unused int algindex, void *a
 
     if (!opt_md_silent(mdname, &md))
         return -1;
-    if (EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) {
+    if (EVP_MD_xof(md)) {
         ctx = EVP_MD_CTX_new();
         if (ctx == NULL) {
             count = -1;
diff --git a/crypto/rsa/rsa_oaep.c b/crypto/rsa/rsa_oaep.c
index 86a81499c2..25fdfa53ee 100644
--- a/crypto/rsa/rsa_oaep.c
+++ b/crypto/rsa/rsa_oaep.c
@@ -78,11 +78,11 @@ int ossl_rsa_padding_add_PKCS1_OAEP_mgf1_ex(OSSL_LIB_CTX *libctx,
 
 #ifdef FIPS_MODULE
     /* XOF are approved as standalone; Shake256 in Ed448; MGF */
-    if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0) {
+    if (EVP_MD_xof(md)) {
         ERR_raise(ERR_LIB_RSA, RSA_R_DIGEST_NOT_ALLOWED);
         return 0;
     }
-    if ((EVP_MD_get_flags(mgf1md) & EVP_MD_FLAG_XOF) != 0) {
+    if (EVP_MD_xof(mgf1md)) {
         ERR_raise(ERR_LIB_RSA, RSA_R_MGF1_DIGEST_NOT_ALLOWED);
         return 0;
     }
@@ -196,11 +196,11 @@ int RSA_padding_check_PKCS1_OAEP_mgf1(unsigned char *to, int tlen,
 
 #ifdef FIPS_MODULE
     /* XOF are approved as standalone; Shake256 in Ed448; MGF */
-    if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0) {
+    if (EVP_MD_xof(md)) {
         ERR_raise(ERR_LIB_RSA, RSA_R_DIGEST_NOT_ALLOWED);
         return -1;
     }
-    if ((EVP_MD_get_flags(mgf1md) & EVP_MD_FLAG_XOF) != 0) {
+    if (EVP_MD_xof(mgf1md)) {
         ERR_raise(ERR_LIB_RSA, RSA_R_MGF1_DIGEST_NOT_ALLOWED);
         return -1;
     }
@@ -360,7 +360,7 @@ int PKCS1_MGF1(unsigned char *mask, long len,
     if (c == NULL)
         goto err;
     mdlen = EVP_MD_get_size(dgst);
-    if (mdlen < 0)
+    if (mdlen <= 0)
         goto err;
     /* step 4 */
     for (i = 0; outlen < len; i++) {
diff --git a/providers/implementations/exchange/dh_exch.c b/providers/implementations/exchange/dh_exch.c
index 2c054bb992..b105826438 100644
--- a/providers/implementations/exchange/dh_exch.c
+++ b/providers/implementations/exchange/dh_exch.c
@@ -392,7 +392,7 @@ static int dh_set_ctx_params(void *vpdhctx, const OSSL_PARAM params[])
         if (pdhctx->kdf_md == NULL)
             return 0;
         /* XOF digests are not allowed */
-        if ((EVP_MD_get_flags(pdhctx->kdf_md) & EVP_MD_FLAG_XOF) != 0) {
+        if (EVP_MD_xof(pdhctx->kdf_md)) {
             ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED);
             return 0;
         }
diff --git a/providers/implementations/exchange/ecdh_exch.c b/providers/implementations/exchange/ecdh_exch.c
index 39307e4e1e..29ef20330d 100644
--- a/providers/implementations/exchange/ecdh_exch.c
+++ b/providers/implementations/exchange/ecdh_exch.c
@@ -315,7 +315,7 @@ int ecdh_set_ctx_params(void *vpecdhctx, const OSSL_PARAM params[])
         if (pectx->kdf_md == NULL)
             return 0;
         /* XOF digests are not allowed */
-        if ((EVP_MD_get_flags(pectx->kdf_md) & EVP_MD_FLAG_XOF) != 0) {
+        if (EVP_MD_xof(pectx->kdf_md)) {
             ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED);
             return 0;
         }
diff --git a/providers/implementations/kdfs/hmacdrbg_kdf.c b/providers/implementations/kdfs/hmacdrbg_kdf.c
index 3df5221580..bdaea6b4a2 100644
--- a/providers/implementations/kdfs/hmacdrbg_kdf.c
+++ b/providers/implementations/kdfs/hmacdrbg_kdf.c
@@ -217,7 +217,7 @@ static int hmac_drbg_kdf_set_ctx_params(void *vctx,
         /* Confirm digest is allowed. Allow all digests that are not XOF */
         md = ossl_prov_digest_md(&drbg->digest);
         if (md != NULL) {
-            if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0) {
+            if (EVP_MD_xof(md)) {
                 ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED);
                 return 0;
             }
diff --git a/providers/implementations/kdfs/pbkdf2.c b/providers/implementations/kdfs/pbkdf2.c
index 1c94ece494..0615aecfa9 100644
--- a/providers/implementations/kdfs/pbkdf2.c
+++ b/providers/implementations/kdfs/pbkdf2.c
@@ -266,7 +266,7 @@ static int kdf_pbkdf2_set_ctx_params(void *vctx, const OSSL_PARAM params[])
         if (!ossl_prov_digest_load_from_params(&ctx->digest, params, provctx))
             return 0;
         md = ossl_prov_digest_md(&ctx->digest);
-        if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0) {
+        if (EVP_MD_xof(md)) {
             ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED);
             return 0;
         }
diff --git a/providers/implementations/kdfs/sshkdf.c b/providers/implementations/kdfs/sshkdf.c
index c1d3066951..191b7be0c5 100644
--- a/providers/implementations/kdfs/sshkdf.c
+++ b/providers/implementations/kdfs/sshkdf.c
@@ -229,7 +229,7 @@ static int kdf_sshkdf_set_ctx_params(void *vctx, const OSSL_PARAM params[])
             return 0;
 
         md = ossl_prov_digest_md(&ctx->digest);
-        if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0) {
+        if (EVP_MD_xof(md)) {
             ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED);
             return 0;
         }
diff --git a/providers/implementations/kdfs/sskdf.c b/providers/implementations/kdfs/sskdf.c
index ce9d0a7150..ff5ec50138 100644
--- a/providers/implementations/kdfs/sskdf.c
+++ b/providers/implementations/kdfs/sskdf.c
@@ -577,7 +577,7 @@ static int sskdf_common_set_ctx_params(KDF_SSKDF *ctx, const OSSL_PARAM params[]
             return 0;
 
         md = ossl_prov_digest_md(&ctx->digest);
-        if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0) {
+        if (EVP_MD_xof(md)) {
             ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED);
             return 0;
         }
diff --git a/providers/implementations/kdfs/tls1_prf.c b/providers/implementations/kdfs/tls1_prf.c
index 451d8a818f..31316401bc 100644
--- a/providers/implementations/kdfs/tls1_prf.c
+++ b/providers/implementations/kdfs/tls1_prf.c
@@ -324,7 +324,7 @@ static int kdf_tls1_prf_set_ctx_params(void *vctx, const OSSL_PARAM params[])
             return 0;
 
         md = ossl_prov_digest_md(&digest);
-        if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0) {
+        if (EVP_MD_xof(md)) {
             ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED);
             ossl_prov_digest_reset(&digest);
             return 0;
diff --git a/providers/implementations/kdfs/x942kdf.c b/providers/implementations/kdfs/x942kdf.c
index 41eaf52404..205738cef1 100644
--- a/providers/implementations/kdfs/x942kdf.c
+++ b/providers/implementations/kdfs/x942kdf.c
@@ -517,7 +517,7 @@ static int x942kdf_set_ctx_params(void *vctx, const OSSL_PARAM params[])
         if (!ossl_prov_digest_load_from_params(&ctx->digest, params, provctx))
             return 0;
         md = ossl_prov_digest_md(&ctx->digest);
-        if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0) {
+        if (EVP_MD_xof(md)) {
             ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED);
             return 0;
         }
diff --git a/providers/implementations/rands/drbg.c b/providers/implementations/rands/drbg.c
index 825fe30214..255bf9b507 100644
--- a/providers/implementations/rands/drbg.c
+++ b/providers/implementations/rands/drbg.c
@@ -1030,7 +1030,7 @@ int ossl_drbg_verify_digest(PROV_DRBG *drbg, OSSL_LIB_CTX *libctx,
     }
 #else   /* FIPS_MODULE */
     /* Outside of FIPS, any digests that are not XOF are allowed */
-    if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0) {
+    if (EVP_MD_xof(md)) {
         ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED);
         return 0;
     }
diff --git a/providers/implementations/signature/dsa_sig.c b/providers/implementations/signature/dsa_sig.c
index a4b8c21f76..ec2205aa5a 100644
--- a/providers/implementations/signature/dsa_sig.c
+++ b/providers/implementations/signature/dsa_sig.c
@@ -156,7 +156,7 @@ static int dsa_setup_md(PROV_DSA_CTX *ctx,
             goto err;
         }
         /* XOF digests don't work */
-        if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0) {
+        if (EVP_MD_xof(md)) {
             ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED);
             goto err;
         }
diff --git a/providers/implementations/signature/ecdsa_sig.c b/providers/implementations/signature/ecdsa_sig.c
index 1a58850b23..46d100995c 100644
--- a/providers/implementations/signature/ecdsa_sig.c
+++ b/providers/implementations/signature/ecdsa_sig.c
@@ -289,7 +289,7 @@ static int ecdsa_setup_md(PROV_ECDSA_CTX *ctx, const char *mdname,
         goto err;
     }
     /* XOF digests don't work */
-    if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0) {
+    if (EVP_MD_xof(md)) {
         ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED);
         goto err;
     }
diff --git a/providers/implementations/signature/rsa_sig.c b/providers/implementations/signature/rsa_sig.c
index 2ca9c65898..45c36899e4 100644
--- a/providers/implementations/signature/rsa_sig.c
+++ b/providers/implementations/signature/rsa_sig.c
@@ -397,7 +397,7 @@ static int rsa_setup_md(PROV_RSA_CTX *ctx, const char *mdname,
          * We don't support XOF digests with RSA PSS (yet), so just fail.
          * When we do support them, uncomment the second clause.
          */
-        if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0
+        if (EVP_MD_xof(md)
                 /* && ctx->pad_mode != RSA_PKCS1_PSS_PADDING */) {
             ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED);
             goto err;
diff --git a/providers/implementations/signature/sm2_sig.c b/providers/implementations/signature/sm2_sig.c
index 5172d6622b..3decca33c4 100644
--- a/providers/implementations/signature/sm2_sig.c
+++ b/providers/implementations/signature/sm2_sig.c
@@ -99,7 +99,7 @@ static int sm2sig_set_mdname(PROV_SM2_CTX *psm2ctx, const char *mdname)
         return 0;
 
     /* XOF digests don't work */
-    if ((EVP_MD_get_flags(psm2ctx->md) & EVP_MD_FLAG_XOF) != 0) {
+    if (EVP_MD_xof(psm2ctx->md)) {
         ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED);
         return 0;
     }
diff --git a/test/evp_test.c b/test/evp_test.c
index 6a47765745..69ecafe7eb 100644
--- a/test/evp_test.c
+++ b/test/evp_test.c
@@ -788,7 +788,7 @@ static int digest_test_run(EVP_TEST *t)
         goto err;
     }
 
-    xof |= (EVP_MD_get_flags(expected->digest) & EVP_MD_FLAG_XOF) != 0;
+    xof |= EVP_MD_xof(expected->digest);
     if (xof) {
         EVP_MD_CTX *mctx_cpy;