Add EVP_PKEY_supports_digest_nid()

Rather than relying only on mandatory default digests, add a way for
the EVP_PKEY to individually report whether each digest algorithm is
supported.

Reviewed-by: Nicola Tuveri <nic.tuv@gmail.com>
Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/7408)

1 files changed, 30 insertions, 25 deletions

diff --git a/ssl/t1_lib.c b/ssl/t1_lib.c
index 2e785a909b..91353e738a 100644
--- a/ssl/t1_lib.c
+++ b/ssl/t1_lib.c
@@ -2496,8 +2496,7 @@ static int tls12_get_cert_sigalg_idx(const SSL *s, const SIGALG_LOOKUP *lu)
 static int has_usable_cert(SSL *s, const SIGALG_LOOKUP *sig, int idx)
 {
     const SIGALG_LOOKUP *lu;
-    int mdnid, pknid, default_mdnid;
-    int mandatory_md = 0;
+    int mdnid, pknid, supported;
     size_t i;
 
     /* TLS 1.2 callers can override lu->sig_idx, but not TLS 1.3 callers. */
@@ -2505,39 +2504,45 @@ static int has_usable_cert(SSL *s, const SIGALG_LOOKUP *sig, int idx)
         idx = sig->sig_idx;
     if (!ssl_has_cert(s, idx))
         return 0;
-    /* If the EVP_PKEY reports a mandatory digest, allow nothing else. */
-    ERR_set_mark();
-    switch (EVP_PKEY_get_default_digest_nid(s->cert->pkeys[idx].privatekey,
-                                            &default_mdnid)) {
-    case 2:
-        mandatory_md = 1;
-        break;
-    case 1:
-        break;
-    default: /* If it didn't report a mandatory NID, for whatever reasons,
-              * just clear the error and allow all hashes to be used. */
-        ERR_pop_to_mark();
-    }
     if (s->s3->tmp.peer_cert_sigalgs != NULL) {
         for (i = 0; i < s->s3->tmp.peer_cert_sigalgslen; i++) {
             lu = tls1_lookup_sigalg(s->s3->tmp.peer_cert_sigalgs[i]);
             if (lu == NULL
                 || !X509_get_signature_info(s->cert->pkeys[idx].x509, &mdnid,
                                             &pknid, NULL, NULL)
-                || (mandatory_md && mdnid != default_mdnid))
+                /*
+                 * TODO this does not differentiate between the
+                 * rsa_pss_pss_* and rsa_pss_rsae_* schemes since we do not
+                 * have a chain here that lets us look at the key OID in the
+                 * signing certificate.
+                 */
+                || mdnid != lu->hash
+                || pknid != lu->sig)
                 continue;
-            /*
-             * TODO this does not differentiate between the
-             * rsa_pss_pss_* and rsa_pss_rsae_* schemes since we do not
-             * have a chain here that lets us look at the key OID in the
-             * signing certificate.
-             */
-            if (mdnid == lu->hash && pknid == lu->sig)
-                return 1;
+
+            ERR_set_mark();
+            supported = EVP_PKEY_supports_digest_nid(s->cert->pkeys[idx].privatekey,
+                                                     mdnid);
+            if (supported == 0)
+                continue;
+            else if (supported < 0)
+            {
+                /* If it didn't report a mandatory NID, for whatever reasons,
+                 * just clear the error and allow all hashes to be used. */
+                ERR_pop_to_mark();
+            }
+            return 1;
         }
         return 0;
     }
-    return !mandatory_md || sig->hash == default_mdnid;
+    supported = EVP_PKEY_supports_digest_nid(s->cert->pkeys[idx].privatekey,
+                                             sig->hash);
+    if (supported == 0)
+        return 0;
+    else if (supported < 0)
+        ERR_clear_error();
+
+    return 1;
 }
 
 /*