fix: refactor the EVP_PKEY_OP checks

On the one hand, we have public macros that are collections of EVP_PKEY_OP
bits, like EVP_PKEY_OP_TYPE_SIG, obviously meant to be used like this:

    if ((ctx->operation & EVP_PKEY_OP_TYPE_SIG) == 0) ...

On the other hand, we also have internal test macros, like
EVP_PKEY_CTX_IS_SIGNATURE_OP(), obviously meant to be used like this:

    if (EVP_PKEY_CTX_IS_SIGNATURE_OP(ctx)) ...

Unfortunately, these two sets of macros were completely separate, forcing
developers to keep them both sync, manually.

This refactor makes the internal macros use the corresponding public macros,
and adds the missing public macros, for consistency.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
Reviewed-by: Todd Short <todd.short@me.com>
(Merged from https://github.com/openssl/openssl/pull/24854)

2 files changed, 21 insertions, 15 deletions

diff --git a/include/crypto/evp.h b/include/crypto/evp.h
index 50ad737fc8..36fee9f1c0 100644
--- a/include/crypto/evp.h
+++ b/include/crypto/evp.h
@@ -728,30 +728,25 @@ struct evp_pkey_st {
     } cache;
 } /* EVP_PKEY */ ;
 
+/* The EVP_PKEY_OP_TYPE_ macros are found in include/openssl/evp.h */
+
 #define EVP_PKEY_CTX_IS_SIGNATURE_OP(ctx) \
-    ((ctx)->operation == EVP_PKEY_OP_SIGN \
-     || (ctx)->operation == EVP_PKEY_OP_SIGNCTX \
-     || (ctx)->operation == EVP_PKEY_OP_VERIFY \
-     || (ctx)->operation == EVP_PKEY_OP_VERIFYCTX \
-     || (ctx)->operation == EVP_PKEY_OP_VERIFYRECOVER)
+    (((ctx)->operation & EVP_PKEY_OP_TYPE_SIG) != 0)
 
 #define EVP_PKEY_CTX_IS_DERIVE_OP(ctx) \
-    ((ctx)->operation == EVP_PKEY_OP_DERIVE)
+    (((ctx)->operation & EVP_PKEY_OP_TYPE_DERIVE) != 0)
 
 #define EVP_PKEY_CTX_IS_ASYM_CIPHER_OP(ctx) \
-    ((ctx)->operation == EVP_PKEY_OP_ENCRYPT \
-     || (ctx)->operation == EVP_PKEY_OP_DECRYPT)
+    (((ctx)->operation & EVP_PKEY_OP_TYPE_CRYPT) != 0)
 
 #define EVP_PKEY_CTX_IS_GEN_OP(ctx) \
-    ((ctx)->operation == EVP_PKEY_OP_PARAMGEN \
-     || (ctx)->operation == EVP_PKEY_OP_KEYGEN)
+    (((ctx)->operation & EVP_PKEY_OP_TYPE_GEN) != 0)
 
 #define EVP_PKEY_CTX_IS_FROMDATA_OP(ctx) \
-    ((ctx)->operation == EVP_PKEY_OP_FROMDATA)
+    (((ctx)->operation & EVP_PKEY_OP_TYPE_DATA) != 0)
 
 #define EVP_PKEY_CTX_IS_KEM_OP(ctx) \
-    ((ctx)->operation == EVP_PKEY_OP_ENCAPSULATE \
-     || (ctx)->operation == EVP_PKEY_OP_DECAPSULATE)
+    (((ctx)->operation & EVP_PKEY_OP_TYPE_KEM) != 0)
 
 void openssl_add_all_ciphers_int(void);
 void openssl_add_all_digests_int(void);
diff --git a/include/openssl/evp.h b/include/openssl/evp.h
index f70b9d744d..af62cd30e8 100644
--- a/include/openssl/evp.h
+++ b/include/openssl/evp.h
@@ -1703,6 +1703,8 @@ const char *EVP_PKEY_get0_type_name(const EVP_PKEY *key);
 # define EVP_PKEY_OP_DERIVE              (1<<11)
 # define EVP_PKEY_OP_ENCAPSULATE         (1<<12)
 # define EVP_PKEY_OP_DECAPSULATE         (1<<13)
+/* Update the following when adding new EVP_PKEY_OPs */
+# define EVP_PKEY_OP_ALL                ((1<<14) - 1)
 
 # define EVP_PKEY_OP_TYPE_SIG    \
         (EVP_PKEY_OP_SIGN | EVP_PKEY_OP_VERIFY | EVP_PKEY_OP_VERIFYRECOVER \
@@ -1711,12 +1713,21 @@ const char *EVP_PKEY_get0_type_name(const EVP_PKEY *key);
 # define EVP_PKEY_OP_TYPE_CRYPT \
         (EVP_PKEY_OP_ENCRYPT | EVP_PKEY_OP_DECRYPT)
 
-# define EVP_PKEY_OP_TYPE_NOGEN \
-        (EVP_PKEY_OP_TYPE_SIG | EVP_PKEY_OP_TYPE_CRYPT | EVP_PKEY_OP_DERIVE)
+# define EVP_PKEY_OP_TYPE_DERIVE \
+        (EVP_PKEY_OP_DERIVE)
+
+# define EVP_PKEY_OP_TYPE_DATA \
+        (EVP_PKEY_OP_FROMDATA)
+
+# define EVP_PKEY_OP_TYPE_KEM \
+        (EVP_PKEY_OP_ENCAPSULATE | EVP_PKEY_OP_DECAPSULATE)
+
 
 # define EVP_PKEY_OP_TYPE_GEN \
         (EVP_PKEY_OP_PARAMGEN | EVP_PKEY_OP_KEYGEN)
 
+# define EVP_PKEY_OP_TYPE_NOGEN \
+        (EVP_PKEY_OP_ALL & ~EVP_PKEY_OP_TYPE_GEN)
 
 int EVP_PKEY_CTX_set_mac_key(EVP_PKEY_CTX *ctx, const unsigned char *key,
                              int keylen);