RSA: Add RSA key types

The support of restricted RSA key types (OAEP and PSS) was lacking, or
dependent on the presence of restriction parameters.  For example,
this means that even though an RSA-PSS key may have been generated, it
may appear as a plain unrestricted RSA key if parameters weren't
present (which is the case when default restriction parameters are
used)

To make it clearer what an RSA key is intended for, and avoid
depending in an EVP_PKEY, we introduce RSA key types.  This is done by
reserving a section of the RSA flags (4 bits, which allows a total of
16 different types).

This isn't terribly important for EVP_PKEY_ASN1_METHOD code, as that
has access to the wrapping EVP_PKEY.  This is very important for
provider code, which has no access to the wrapping EVP_PKEY.

Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/11710)

2 files changed, 33 insertions, 1 deletions

diff --git a/crypto/rsa/rsa_ameth.c b/crypto/rsa/rsa_ameth.c
index 27aa9f422d..b530754528 100644
--- a/crypto/rsa/rsa_ameth.c
+++ b/crypto/rsa/rsa_ameth.c
@@ -43,7 +43,7 @@ static int rsa_param_encode(const EVP_PKEY *pkey,
 
     *pstr = NULL;
     /* If RSA it's just NULL type */
-    if (pkey->ameth->pkey_id != EVP_PKEY_RSA_PSS) {
+    if (RSA_test_flags(rsa, RSA_FLAG_TYPE_MASK) != RSA_FLAG_TYPE_RSASSAPSS) {
         *pstrtype = V_ASN1_NULL;
         return 1;
     }
@@ -196,6 +196,20 @@ static int rsa_priv_decode(EVP_PKEY *pkey, const PKCS8_PRIV_KEY_INFO *p8)
         RSA_free(rsa);
         return 0;
     }
+
+    RSA_clear_flags(rsa, RSA_FLAG_TYPE_MASK);
+    switch (pkey->ameth->pkey_id) {
+    case EVP_PKEY_RSA:
+        RSA_set_flags(rsa, RSA_FLAG_TYPE_RSA);
+        break;
+    case EVP_PKEY_RSA_PSS:
+        RSA_set_flags(rsa, RSA_FLAG_TYPE_RSASSAPSS);
+        break;
+    default:
+        /* Leave the type bits zero */
+        break;
+    }
+
     EVP_PKEY_assign(pkey, pkey->ameth->pkey_id, rsa);
     return 1;
 }
diff --git a/include/openssl/rsa.h b/include/openssl/rsa.h
index 2508abd81c..d942bc57ab 100644
--- a/include/openssl/rsa.h
+++ b/include/openssl/rsa.h
@@ -107,6 +107,24 @@ extern "C" {
 #   define RSA_FLAG_NO_EXP_CONSTTIME RSA_FLAG_NO_CONSTTIME
 #  endif
 
+/*-
+ * New with 3.0: use part of the flags to denote exact type of RSA key,
+ * some of which are limited to specific signature and encryption schemes.
+ * These different types share the same RSA structure, but indicate the
+ * use of certain fields in that structure.
+ * Currently known are:
+ * RSA          - this is the "normal" unlimited RSA structure (typenum 0)
+ * RSASSA-PSS   - indicates that the PSS parameters are used.
+ * RSAES-OAEP   - no specific field used for the moment, but OAEP padding
+ *                is expected.  (currently unused)
+ *
+ * 4 bits allow for 16 types
+ */
+#  define RSA_FLAG_TYPE_MASK            0xF000
+#  define RSA_FLAG_TYPE_RSA             0x0000
+#  define RSA_FLAG_TYPE_RSASSAPSS       0x1000
+#  define RSA_FLAG_TYPE_RSAESOAEP       0x2000
+
 int EVP_PKEY_CTX_set_rsa_padding(EVP_PKEY_CTX *ctx, int pad_mode);
 int EVP_PKEY_CTX_get_rsa_padding(EVP_PKEY_CTX *ctx, int *pad_mode);