Replace provider digest flags with separate param fields

Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/13830)

4 files changed, 31 insertions, 19 deletions

diff --git a/providers/implementations/digests/digestcommon.c b/providers/implementations/digests/digestcommon.c
index 6d926713c8..b8e7efde60 100644
--- a/providers/implementations/digests/digestcommon.c
+++ b/providers/implementations/digests/digestcommon.c
@@ -26,8 +26,15 @@ int digest_default_get_params(OSSL_PARAM params[], size_t blksz, size_t paramsz,
         ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER);
         return 0;
     }
-    p = OSSL_PARAM_locate(params, OSSL_DIGEST_PARAM_FLAGS);
-    if (p != NULL && !OSSL_PARAM_set_ulong(p, flags)) {
+    p = OSSL_PARAM_locate(params, OSSL_DIGEST_PARAM_XOF);
+    if (p != NULL
+        && !OSSL_PARAM_set_int(p, (flags & PROV_DIGEST_FLAG_XOF) != 0)) {
+        ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER);
+        return 0;
+    }
+    p = OSSL_PARAM_locate(params, OSSL_DIGEST_PARAM_ALGID_ABSENT);
+    if (p != NULL
+        && !OSSL_PARAM_set_int(p, (flags & PROV_DIGEST_FLAG_ALGID_ABSENT) != 0)) {
         ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER);
         return 0;
     }
@@ -37,7 +44,8 @@ int digest_default_get_params(OSSL_PARAM params[], size_t blksz, size_t paramsz,
 static const OSSL_PARAM digest_default_known_gettable_params[] = {
     OSSL_PARAM_size_t(OSSL_DIGEST_PARAM_BLOCK_SIZE, NULL),
     OSSL_PARAM_size_t(OSSL_DIGEST_PARAM_SIZE, NULL),
-    OSSL_PARAM_ulong(OSSL_DIGEST_PARAM_FLAGS, NULL),
+    OSSL_PARAM_int(OSSL_DIGEST_PARAM_XOF, NULL),
+    OSSL_PARAM_int(OSSL_DIGEST_PARAM_ALGID_ABSENT, NULL),
     OSSL_PARAM_END
 };
 const OSSL_PARAM *digest_default_gettable_params(void *provctx)
diff --git a/providers/implementations/digests/sha2_prov.c b/providers/implementations/digests/sha2_prov.c
index 2f01149ad9..4cff62131c 100644
--- a/providers/implementations/digests/sha2_prov.c
+++ b/providers/implementations/digests/sha2_prov.c
@@ -24,6 +24,8 @@
 #include "prov/implementations.h"
 #include "crypto/sha.h"
 
+#define SHA2_FLAGS PROV_DIGEST_FLAG_ALGID_ABSENT
+
 static OSSL_FUNC_digest_set_ctx_params_fn sha1_set_ctx_params;
 static OSSL_FUNC_digest_settable_ctx_params_fn sha1_settable_ctx_params;
 
@@ -53,43 +55,37 @@ static int sha1_set_ctx_params(void *vctx, const OSSL_PARAM params[])
 
 /* ossl_sha1_functions */
 IMPLEMENT_digest_functions_with_settable_ctx(
-    sha1, SHA_CTX, SHA_CBLOCK, SHA_DIGEST_LENGTH, EVP_MD_FLAG_DIGALGID_ABSENT,
+    sha1, SHA_CTX, SHA_CBLOCK, SHA_DIGEST_LENGTH, SHA2_FLAGS,
     SHA1_Init, SHA1_Update, SHA1_Final,
     sha1_settable_ctx_params, sha1_set_ctx_params)
 
 /* ossl_sha224_functions */
 IMPLEMENT_digest_functions(sha224, SHA256_CTX,
-                           SHA256_CBLOCK, SHA224_DIGEST_LENGTH,
-                           EVP_MD_FLAG_DIGALGID_ABSENT,
+                           SHA256_CBLOCK, SHA224_DIGEST_LENGTH, SHA2_FLAGS,
                            SHA224_Init, SHA224_Update, SHA224_Final)
 
 /* ossl_sha256_functions */
 IMPLEMENT_digest_functions(sha256, SHA256_CTX,
-                           SHA256_CBLOCK, SHA256_DIGEST_LENGTH,
-                           EVP_MD_FLAG_DIGALGID_ABSENT,
+                           SHA256_CBLOCK, SHA256_DIGEST_LENGTH, SHA2_FLAGS,
                            SHA256_Init, SHA256_Update, SHA256_Final)
 
 /* ossl_sha384_functions */
 IMPLEMENT_digest_functions(sha384, SHA512_CTX,
-                           SHA512_CBLOCK, SHA384_DIGEST_LENGTH,
-                           EVP_MD_FLAG_DIGALGID_ABSENT,
+                           SHA512_CBLOCK, SHA384_DIGEST_LENGTH, SHA2_FLAGS,
                            SHA384_Init, SHA384_Update, SHA384_Final)
 
 /* ossl_sha512_functions */
 IMPLEMENT_digest_functions(sha512, SHA512_CTX,
-                           SHA512_CBLOCK, SHA512_DIGEST_LENGTH,
-                           EVP_MD_FLAG_DIGALGID_ABSENT,
+                           SHA512_CBLOCK, SHA512_DIGEST_LENGTH, SHA2_FLAGS,
                            SHA512_Init, SHA512_Update, SHA512_Final)
 
 /* ossl_sha512_224_functions */
 IMPLEMENT_digest_functions(sha512_224, SHA512_CTX,
-                           SHA512_CBLOCK, SHA224_DIGEST_LENGTH,
-                           EVP_MD_FLAG_DIGALGID_ABSENT,
+                           SHA512_CBLOCK, SHA224_DIGEST_LENGTH, SHA2_FLAGS,
                            sha512_224_init, SHA512_Update, SHA512_Final)
 
 /* ossl_sha512_256_functions */
 IMPLEMENT_digest_functions(sha512_256, SHA512_CTX,
-                           SHA512_CBLOCK, SHA256_DIGEST_LENGTH,
-                           EVP_MD_FLAG_DIGALGID_ABSENT,
+                           SHA512_CBLOCK, SHA256_DIGEST_LENGTH, SHA2_FLAGS,
                            sha512_256_init, SHA512_Update, SHA512_Final)
 
diff --git a/providers/implementations/digests/sha3_prov.c b/providers/implementations/digests/sha3_prov.c
index 6b44792529..6e731fd842 100644
--- a/providers/implementations/digests/sha3_prov.c
+++ b/providers/implementations/digests/sha3_prov.c
@@ -18,6 +18,10 @@
 #include "prov/implementations.h"
 #include "prov/providercommonerr.h"
 
+#define SHA3_FLAGS PROV_DIGEST_FLAG_ALGID_ABSENT
+#define SHAKE_FLAGS PROV_DIGEST_FLAG_XOF
+#define KMAC_FLAGS PROV_DIGEST_FLAG_XOF
+
 /*
  * Forward declaration of any unique methods implemented here. This is not strictly
  * necessary for the compiler, but provides an assurance that the signatures
@@ -286,18 +290,18 @@ static int shake_set_ctx_params(void *vctx, const OSSL_PARAM params[])
     SHA3_newctx(sha3, SHA3_##bitlen, sha3_##bitlen, bitlen, '\x06')            \
     PROV_FUNC_SHA3_DIGEST(sha3_##bitlen, bitlen,                               \
                           SHA3_BLOCKSIZE(bitlen), SHA3_MDSIZE(bitlen),         \
-                          EVP_MD_FLAG_DIGALGID_ABSENT)
+                          SHA3_FLAGS)
 
 #define IMPLEMENT_SHAKE_functions(bitlen)                                      \
     SHA3_newctx(shake, SHAKE_##bitlen, shake_##bitlen, bitlen, '\x1f')         \
     PROV_FUNC_SHAKE_DIGEST(shake_##bitlen, bitlen,                             \
                           SHA3_BLOCKSIZE(bitlen), SHA3_MDSIZE(bitlen),         \
-                          EVP_MD_FLAG_XOF)
+                          SHAKE_FLAGS)
 #define IMPLEMENT_KMAC_functions(bitlen)                                       \
     KMAC_newctx(keccak_kmac_##bitlen, bitlen, '\x04')                          \
     PROV_FUNC_SHAKE_DIGEST(keccak_kmac_##bitlen, bitlen,                       \
                            SHA3_BLOCKSIZE(bitlen), KMAC_MDSIZE(bitlen),        \
-                           EVP_MD_FLAG_XOF)
+                           KMAC_FLAGS)
 
 /* ossl_sha3_224_functions */
 IMPLEMENT_SHA3_functions(224)
diff --git a/providers/implementations/include/prov/digestcommon.h b/providers/implementations/include/prov/digestcommon.h
index 99004731fa..f1164c5a1a 100644
--- a/providers/implementations/include/prov/digestcommon.h
+++ b/providers/implementations/include/prov/digestcommon.h
@@ -15,6 +15,10 @@
 # include <openssl/params.h>
 # include "prov/providercommon.h"
 
+/* Internal flags that can be queried */
+#define PROV_DIGEST_FLAG_XOF             0x0001
+#define PROV_DIGEST_FLAG_ALGID_ABSENT    0x0002
+
 # ifdef __cplusplus
 extern "C" {
 # endif