EVP: Don't call digest_custom() quite so early

A huge problem with calling digest_custom() already in the
initialization of DigestSign, DigestVerify etc, is that it force all
callers to know that certain controls must be performed before Init
and the rest after.  This has lead to quite interesting hacks in our
own openssl app, where the SM2 ID had to get special treatment instead
of just being another sign option or verification option among others.

This change moves the call of digest_custom() to the Update and Final
functions, to be done exactly once, subject to a flag that's set in
the Init function.  Seeing to the process of data, through these
operations, this makes no difference at all.  Seeing to making it
possible to perform all controls after the Init call, this makes a
huge difference.

Fixes #11293

Reviewed-by: Paul Yang <kaishen.yy@antfin.com>
(Merged from https://github.com/openssl/openssl/pull/11302)

1 files changed, 26 insertions, 1 deletions

diff --git a/crypto/evp/m_sigver.c b/crypto/evp/m_sigver.c
index 4b2cb4eb35..1948f234ca 100644
--- a/crypto/evp/m_sigver.c
+++ b/crypto/evp/m_sigver.c
@@ -250,8 +250,9 @@ static int do_sigver_init(EVP_MD_CTX *ctx, EVP_PKEY_CTX **pctx,
      * This indicates the current algorithm requires
      * special treatment before hashing the tbs-message.
      */
+    ctx->pctx->flag_call_digest_custom = 0;
     if (ctx->pctx->pmeth->digest_custom != NULL)
-        return ctx->pctx->pmeth->digest_custom(ctx->pctx, ctx);
+        ctx->pctx->flag_call_digest_custom = 1;
 
     return 1;
 }
@@ -301,6 +302,12 @@ int EVP_DigestSignUpdate(EVP_MD_CTX *ctx, const void *data, size_t dsize)
                                                       data, dsize);
 
  legacy:
+    /* do_sigver_init() checked that |digest_custom| is non-NULL */
+    if (pctx->flag_call_digest_custom
+        && !ctx->pctx->pmeth->digest_custom(ctx->pctx, ctx))
+        return 0;
+    pctx->flag_call_digest_custom = 0;
+
     return EVP_DigestUpdate(ctx, data, dsize);
 }
 
@@ -323,6 +330,12 @@ int EVP_DigestVerifyUpdate(EVP_MD_CTX *ctx, const void *data, size_t dsize)
                                                         data, dsize);
 
  legacy:
+    /* do_sigver_init() checked that |digest_custom| is non-NULL */
+    if (pctx->flag_call_digest_custom
+        && !ctx->pctx->pmeth->digest_custom(ctx->pctx, ctx))
+        return 0;
+    pctx->flag_call_digest_custom = 0;
+
     return EVP_DigestUpdate(ctx, data, dsize);
 }
 
@@ -348,6 +361,12 @@ int EVP_DigestSignFinal(EVP_MD_CTX *ctx, unsigned char *sigret,
         return 0;
     }
 
+    /* do_sigver_init() checked that |digest_custom| is non-NULL */
+    if (pctx->flag_call_digest_custom
+        && !ctx->pctx->pmeth->digest_custom(ctx->pctx, ctx))
+        return 0;
+    pctx->flag_call_digest_custom = 0;
+
     if (pctx->pmeth->flags & EVP_PKEY_FLAG_SIGCTX_CUSTOM) {
         if (sigret == NULL)
             return pctx->pmeth->signctx(pctx, sigret, siglen, ctx);
@@ -458,6 +477,12 @@ int EVP_DigestVerifyFinal(EVP_MD_CTX *ctx, const unsigned char *sig,
         return 0;
     }
 
+    /* do_sigver_init() checked that |digest_custom| is non-NULL */
+    if (pctx->flag_call_digest_custom
+        && !ctx->pctx->pmeth->digest_custom(ctx->pctx, ctx))
+        return 0;
+    pctx->flag_call_digest_custom = 0;
+
     if (pctx->pmeth->verifyctx != NULL)
         vctx = 1;
     else