Process digest option after loading providers

Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/14135)

11 files changed, 68 insertions, 36 deletions

diff --git a/apps/cms.c b/apps/cms.c
index 451f0862c2..4f1e614e4c 100644
--- a/apps/cms.c
+++ b/apps/cms.c
@@ -286,7 +286,7 @@ int cms_main(int argc, char **argv)
     X509_VERIFY_PARAM *vpm = NULL;
     char *certfile = NULL, *keyfile = NULL, *contfile = NULL;
     const char *CAfile = NULL, *CApath = NULL, *CAstore = NULL;
-    char *certsoutfile = NULL;
+    char *certsoutfile = NULL, *digestname = NULL;
     int noCAfile = 0, noCApath = 0, noCAstore = 0;
     char *infile = NULL, *outfile = NULL, *rctfile = NULL;
     char *passinarg = NULL, *passin = NULL, *signerfile = NULL, *originatorfile = NULL, *recipfile = NULL;
@@ -565,8 +565,7 @@ int cms_main(int argc, char **argv)
             certsoutfile = opt_arg();
             break;
         case OPT_MD:
-            if (!opt_md(opt_arg(), &sign_md))
-                goto end;
+            digestname = opt_arg();
             break;
         case OPT_SIGNER:
             /* If previous -signer argument add signer to list */
@@ -699,6 +698,8 @@ int cms_main(int argc, char **argv)
         }
     }
     app_RAND_load();
+    if (digestname != NULL && !opt_md(digestname, &sign_md))
+        goto end;
 
     /* Remaining args are files to process. */
     argc = opt_num_rest();
diff --git a/apps/crl.c b/apps/crl.c
index ddbf96bfca..34551cc3a4 100644
--- a/apps/crl.c
+++ b/apps/crl.c
@@ -84,6 +84,7 @@ int crl_main(int argc, char **argv)
     EVP_PKEY *pkey;
     const EVP_MD *digest = EVP_sha1();
     char *infile = NULL, *outfile = NULL, *crldiff = NULL, *keyfile = NULL;
+    char *digestname = NULL;
     const char *CAfile = NULL, *CApath = NULL, *CAstore = NULL, *prog;
     OPTION_CHOICE o;
     int hash = 0, issuer = 0, lastupdate = 0, nextupdate = 0, noout = 0;
@@ -192,8 +193,7 @@ int crl_main(int argc, char **argv)
                 goto opthelp;
             break;
         case OPT_MD:
-            if (!opt_md(opt_unknown(), &digest))
-                goto opthelp;
+            digestname = opt_unknown();
             break;
         case OPT_PROV_CASES:
             if (!opt_provider(o))
@@ -207,6 +207,8 @@ int crl_main(int argc, char **argv)
     if (argc != 0)
         goto opthelp;
 
+    if (digestname != NULL && !opt_md(digestname, &digest))
+        goto opthelp;
     x = load_crl(infile, "CRL");
     if (x == NULL)
         goto end;
diff --git a/apps/dgst.c b/apps/dgst.c
index 8f22334b37..891cf79279 100644
--- a/apps/dgst.c
+++ b/apps/dgst.c
@@ -97,9 +97,9 @@ int dgst_main(int argc, char **argv)
     EVP_PKEY *sigkey = NULL;
     STACK_OF(OPENSSL_STRING) *sigopts = NULL, *macopts = NULL;
     char *hmac_key = NULL;
-    char *mac_name = NULL;
+    char *mac_name = NULL, *digestname = NULL;
     char *passinarg = NULL, *passin = NULL;
-    const EVP_MD *md = NULL, *m;
+    const EVP_MD *md = NULL;
     const char *outfile = NULL, *keyfile = NULL, *prog = NULL;
     const char *sigfile = NULL;
     const char *md_name = NULL;
@@ -209,9 +209,7 @@ int dgst_main(int argc, char **argv)
                 goto opthelp;
             break;
         case OPT_DIGEST:
-            if (!opt_md(opt_unknown(), &m))
-                goto opthelp;
-            md = m;
+            digestname = opt_unknown();
             break;
         case OPT_PROV_CASES:
             if (!opt_provider(o))
@@ -228,6 +226,10 @@ int dgst_main(int argc, char **argv)
         goto end;
     }
     app_RAND_load();
+    if (digestname != NULL) {
+        if (!opt_md(digestname, &md))
+            goto opthelp;
+    }
 
     if (do_verify && sigfile == NULL) {
         BIO_printf(bio_err,
diff --git a/apps/enc.c b/apps/enc.c
index 4549895b9e..9982337c01 100644
--- a/apps/enc.c
+++ b/apps/enc.c
@@ -111,7 +111,7 @@ int enc_main(int argc, char **argv)
     EVP_CIPHER_CTX *ctx = NULL;
     const EVP_CIPHER *cipher = NULL;
     const EVP_MD *dgst = NULL;
-    const char *dgstname = NULL;
+    const char *digestname = NULL;
     char *hkey = NULL, *hiv = NULL, *hsalt = NULL, *p;
     char *infile = NULL, *outfile = NULL, *prog;
     char *str = NULL, *passarg = NULL, *pass = NULL, *strbuf = NULL;
@@ -260,7 +260,7 @@ int enc_main(int argc, char **argv)
             hiv = opt_arg();
             break;
         case OPT_MD:
-            dgstname = opt_arg();
+            digestname = opt_arg();
             break;
         case OPT_CIPHER:
             ciphername = opt_unknown();
@@ -308,8 +308,8 @@ int enc_main(int argc, char **argv)
         BIO_printf(bio_err, "%s XTS ciphers not supported\n", prog);
         goto end;
     }
-    if (dgstname != NULL) {
-        if (!!opt_md(opt_arg(), &dgst))
+    if (digestname != NULL) {
+        if (!opt_md(digestname, &dgst))
             goto opthelp;
     }
     if (dgst == NULL)
diff --git a/apps/ocsp.c b/apps/ocsp.c
index 982423d1ef..dd1677b1c1 100644
--- a/apps/ocsp.c
+++ b/apps/ocsp.c
@@ -223,7 +223,7 @@ int ocsp_main(int argc, char **argv)
     X509_STORE *store = NULL;
     X509_VERIFY_PARAM *vpm = NULL;
     const char *CAfile = NULL, *CApath = NULL, *CAstore = NULL;
-    char *header, *value;
+    char *header, *value, *respdigname = NULL;
     char *host = NULL, *port = NULL, *path = "/", *outfile = NULL;
     char *rca_filename = NULL, *reqin = NULL, *respin = NULL;
     char *reqout = NULL, *respout = NULL, *ridx_filename = NULL;
@@ -467,8 +467,7 @@ int ocsp_main(int argc, char **argv)
             rcertfile = opt_arg();
             break;
         case OPT_RMD:   /* Response MessageDigest */
-            if (!opt_md(opt_arg(), &rsign_md))
-                goto end;
+            respdigname = opt_arg();
             break;
         case OPT_RSIGOPT:
             if (rsign_sigopts == NULL)
@@ -526,6 +525,11 @@ int ocsp_main(int argc, char **argv)
         goto opthelp;
     }
 
+    if (respdigname != NULL) {
+        if (!opt_md(respdigname, &rsign_md))
+            goto end;
+    }
+
     /* Have we anything to do? */
     if (req == NULL && reqin == NULL
         && respin == NULL && !(port != NULL && ridx_filename != NULL))
diff --git a/apps/pkeyutl.c b/apps/pkeyutl.c
index 28c4401745..b70f9935b6 100644
--- a/apps/pkeyutl.c
+++ b/apps/pkeyutl.c
@@ -117,7 +117,7 @@ int pkeyutl_main(int argc, char **argv)
     size_t buf_outlen;
     const char *inkey = NULL;
     const char *peerkey = NULL;
-    const char *kdfalg = NULL;
+    const char *kdfalg = NULL, *digestname = NULL;
     int kdflen = 0;
     STACK_OF(OPENSSL_STRING) *pkeyopts = NULL;
     STACK_OF(OPENSSL_STRING) *pkeyopts_passin = NULL;
@@ -244,8 +244,7 @@ int pkeyutl_main(int argc, char **argv)
             rawin = 1;
             break;
         case OPT_DIGEST:
-            if (!opt_md(opt_arg(), &md))
-                goto end;
+            digestname = opt_arg();
             break;
         }
     }
@@ -256,6 +255,10 @@ int pkeyutl_main(int argc, char **argv)
         goto opthelp;
 
     app_RAND_load();
+    if (digestname != NULL) {
+        if (!opt_md(digestname, &md))
+            goto end;
+    }
 
     if (rawin && pkey_op != EVP_PKEY_OP_SIGN && pkey_op != EVP_PKEY_OP_VERIFY) {
         BIO_printf(bio_err,
diff --git a/apps/req.c b/apps/req.c
index 4cd31cf4ba..881cbb45c7 100644
--- a/apps/req.c
+++ b/apps/req.c
@@ -245,7 +245,7 @@ int req_main(int argc, char **argv)
     BIO *addext_bio = NULL;
     char *extensions = NULL;
     const char *infile = NULL, *CAfile = NULL, *CAkeyfile = NULL;
-    char *outfile = NULL, *keyfile = NULL;
+    char *outfile = NULL, *keyfile = NULL, *digestname = NULL;
     char *keyalgstr = NULL, *p, *prog, *passargin = NULL, *passargout = NULL;
     char *passin = NULL, *passout = NULL;
     char *nofree_passin = NULL, *nofree_passout = NULL;
@@ -468,9 +468,7 @@ int req_main(int argc, char **argv)
             newreq = precert = 1;
             break;
         case OPT_MD:
-            if (!opt_md(opt_unknown(), &md_alg))
-                goto opthelp;
-            digest = md_alg;
+            digestname = opt_unknown();
             break;
         }
     }
@@ -481,6 +479,12 @@ int req_main(int argc, char **argv)
         goto opthelp;
 
     app_RAND_load();
+    if (digestname != NULL) {
+        if (!opt_md(digestname, &md_alg))
+            goto opthelp;
+        digest = md_alg;
+    }
+
     if (!gen_x509) {
         if (days != UNSET_DAYS)
             BIO_printf(bio_err, "Ignoring -days without -x509; not generating a certificate\n");
diff --git a/apps/smime.c b/apps/smime.c
index db9cf8f1f1..fd1e2e6810 100644
--- a/apps/smime.c
+++ b/apps/smime.c
@@ -145,7 +145,8 @@ int smime_main(int argc, char **argv)
     const char *CAfile = NULL, *CApath = NULL, *CAstore = NULL, *prog = NULL;
     char *certfile = NULL, *keyfile = NULL, *contfile = NULL;
     char *infile = NULL, *outfile = NULL, *signerfile = NULL, *recipfile = NULL;
-    char *passinarg = NULL, *passin = NULL, *to = NULL, *from = NULL, *subject = NULL;
+    char *passinarg = NULL, *passin = NULL, *to = NULL, *from = NULL;
+    char *subject = NULL, *digestname = NULL;
     OPTION_CHOICE o;
     int noCApath = 0, noCAfile = 0, noCAstore = 0;
     int flags = PKCS7_DETACHED, operation = 0, ret = 0, indef = 0;
@@ -293,8 +294,7 @@ int smime_main(int argc, char **argv)
             recipfile = opt_arg();
             break;
         case OPT_MD:
-            if (!opt_md(opt_arg(), &sign_md))
-                goto opthelp;
+            digestname = opt_arg();
             break;
         case OPT_CIPHER:
             if (!opt_cipher(opt_unknown(), &cipher))
@@ -361,6 +361,10 @@ int smime_main(int argc, char **argv)
     argv = opt_rest();
 
     app_RAND_load();
+    if (digestname != NULL) {
+        if (!opt_md(digestname, &sign_md))
+            goto opthelp;
+    }
     if (!(operation & SMIME_SIGNERS) && (skkeys != NULL || sksigners != NULL)) {
         BIO_puts(bio_err, "Multiple signers or keys not allowed\n");
         goto opthelp;
diff --git a/apps/storeutl.c b/apps/storeutl.c
index 9333c478f2..7c13092fe5 100644
--- a/apps/storeutl.c
+++ b/apps/storeutl.c
@@ -81,7 +81,7 @@ int storeutl_main(int argc, char *argv[])
     ASN1_INTEGER *serial = NULL;
     unsigned char *fingerprint = NULL;
     size_t fingerprintlen = 0;
-    char *alias = NULL;
+    char *alias = NULL, *digestname = NULL;
     OSSL_STORE_SEARCH *search = NULL;
     const EVP_MD *digest = NULL;
     OSSL_LIB_CTX *libctx = app_get0_libctx();
@@ -247,8 +247,8 @@ int storeutl_main(int argc, char *argv[])
             e = setup_engine(opt_arg(), 0);
             break;
         case OPT_MD:
-            if (!opt_md(opt_unknown(), &digest))
-                goto opthelp;
+            digestname = opt_unknown();
+            break;
         case OPT_PROV_CASES:
             if (!opt_provider(o))
                 goto end;
@@ -262,6 +262,11 @@ int storeutl_main(int argc, char *argv[])
     if (argc != 1)
         goto opthelp;
 
+    if (digestname != NULL) {
+        if (!opt_md(digestname, &digest))
+            goto opthelp;
+    }
+
     if (criterion != 0) {
         switch (criterion) {
         case OSSL_STORE_SEARCH_BY_NAME:
diff --git a/apps/ts.c b/apps/ts.c
index 12958fbbd0..8500968a0c 100644
--- a/apps/ts.c
+++ b/apps/ts.c
@@ -160,7 +160,7 @@ int ts_main(int argc, char **argv)
     CONF *conf = NULL;
     const char *CAfile = NULL, *untrusted = NULL, *prog;
     const char *configfile = default_config_file, *engine = NULL;
-    const char *section = NULL;
+    const char *section = NULL, *digestname = NULL;
     char **helpp;
     char *password = NULL;
     char *data = NULL, *digest = NULL, *policy = NULL;
@@ -276,8 +276,7 @@ int ts_main(int argc, char **argv)
             engine = opt_arg();
             break;
         case OPT_MD:
-            if (!opt_md(opt_unknown(), &md))
-                goto opthelp;
+            digestname = opt_unknown();
             break;
         case OPT_V_CASES:
             if (!opt_verify(o, vpm))
@@ -293,6 +292,10 @@ int ts_main(int argc, char **argv)
         goto opthelp;
 
     app_RAND_load();
+    if (digestname != NULL) {
+        if (!opt_md(digestname, &md))
+            goto opthelp;
+    }
     if (mode == OPT_REPLY && passin &&
         !app_passwd(passin, NULL, &password, NULL)) {
         BIO_printf(bio_err, "Error getting password.\n");
diff --git a/apps/x509.c b/apps/x509.c
index 2ddb549939..67895c8169 100644
--- a/apps/x509.c
+++ b/apps/x509.c
@@ -246,7 +246,7 @@ int x509_main(int argc, char **argv)
     X509V3_CTX ext_ctx;
     EVP_PKEY *signkey = NULL, *CAkey = NULL, *pubkey = NULL;
     int newcert = 0;
-    char *subj = NULL;
+    char *subj = NULL, *digestname = NULL;
     X509_NAME *fsubj = NULL;
     const unsigned long chtype = MBSTRING_ASC;
     const int multirdn = 1;
@@ -569,8 +569,8 @@ int x509_main(int argc, char **argv)
             preserve_dates = 1;
             break;
         case OPT_MD:
-            if (!opt_md(opt_unknown(), &digest))
-                goto opthelp;
+            digestname = opt_unknown();
+            break;
         }
     }
 
@@ -580,6 +580,10 @@ int x509_main(int argc, char **argv)
         goto opthelp;
 
     app_RAND_load();
+    if (digestname != NULL) {
+        if (!opt_md(digestname, &digest))
+            goto opthelp;
+    }
     if (preserve_dates && days != UNSET_DAYS) {
         BIO_printf(bio_err, "Cannot use -preserve_dates with -days option\n");
         goto end;