Generalize the HTTP client so far implemented mostly in crypto/ocsp/ocsp_ht.c

The new client has become an independent libcrpyto module in crypto/http/ and
* can handle any types of requests and responses (ASN.1-encoded and plain)
* does not include potentially busy loops when waiting for responses but
* makes use of a new timeout mechanism integrated with socket-based BIO
* supports the use of HTTP proxies and TLS, including HTTPS over proxies
* supports HTTP redirection via codes 301 and 302 for GET requests
* returns more useful diagnostics in various error situations
Also adapts - and strongly simplifies - hitherto uses of HTTP in crypto/ocsp/,
crypto/x509/x_all.c, apps/lib/apps.c, and apps/{ocsp,s_client,s_server}.c

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: David von Oheimb <david.von.oheimb@siemens.com>
(Merged from https://github.com/openssl/openssl/pull/10667)

4 files changed, 17 insertions, 8 deletions

diff --git a/util/err-to-raise b/util/err-to-raise
index a62ee3b790..7ff7a8aa85 100755
--- a/util/err-to-raise
+++ b/util/err-to-raise
@@ -42,6 +42,7 @@ s/ENGINEerr\(\w+, *(\w+)\)/ERR_raise(ERR_LIB_ENGINE, $1)/;
 s/ESSerr\(\w+, *(\w+)\)/ERR_raise(ERR_LIB_ESS, $1)/;
 s/EVPerr\(\w+, *(\w+)\)/ERR_raise(ERR_LIB_EVP, $1)/;
 s/FIPSerr\(\w+, *(\w+)\)/ERR_raise(ERR_LIB_FIPS, $1)/;
+s/HTTPerr\(\w+, *(\w+)\)/ERR_raise(ERR_LIB_HTTP, $1)/;
 s/KDFerr\(\w+, *(\w+)\)/ERR_raise(ERR_LIB_KDF, $1)/;
 s/OBJerr\(\w+, *(\w+)\)/ERR_raise(ERR_LIB_OBJ, $1)/;
 s/OCSPerr\(\w+, *(\w+)\)/ERR_raise(ERR_LIB_OCSP, $1)/;
diff --git a/util/libcrypto.num b/util/libcrypto.num
index e648370dd8..aa6ce17e7d 100644
--- a/util/libcrypto.num
+++ b/util/libcrypto.num
@@ -245,7 +245,7 @@ RSA_get_ex_data                         249	3_0_0	EXIST::FUNCTION:RSA
 EVP_PKEY_meth_get_decrypt               250	3_0_0	EXIST::FUNCTION:
 DES_cfb_encrypt                         251	3_0_0	EXIST::FUNCTION:DEPRECATEDIN_3_0,DES
 CMS_SignerInfo_set1_signer_cert         252	3_0_0	EXIST::FUNCTION:CMS
-X509_CRL_http_nbio                      253	3_0_0	EXIST::FUNCTION:OCSP
+X509_CRL_load_http                      253	3_0_0	EXIST::FUNCTION:SOCK
 ENGINE_register_all_ciphers             254	3_0_0	EXIST::FUNCTION:ENGINE
 SXNET_new                               255	3_0_0	EXIST::FUNCTION:
 EVP_camellia_256_ctr                    256	3_0_0	EXIST::FUNCTION:CAMELLIA
@@ -266,7 +266,7 @@ WHIRLPOOL_Init                          271	3_0_0	EXIST::FUNCTION:DEPRECATEDIN_3
 EVP_OpenInit                            272	3_0_0	EXIST::FUNCTION:RSA
 OCSP_response_get1_basic                273	3_0_0	EXIST::FUNCTION:OCSP
 CRYPTO_gcm128_tag                       274	3_0_0	EXIST::FUNCTION:
-OCSP_parse_url                          275	3_0_0	EXIST::FUNCTION:OCSP
+OSSL_HTTP_parse_url                     275	3_0_0	EXIST::FUNCTION:
 UI_get0_test_string                     276	3_0_0	EXIST::FUNCTION:
 CRYPTO_secure_free                      277	3_0_0	EXIST::FUNCTION:
 DSA_print_fp                            278	3_0_0	EXIST::FUNCTION:DSA,STDIO
@@ -615,7 +615,7 @@ UI_get0_result_string                   629	3_0_0	EXIST::FUNCTION:
 TS_RESP_CTX_add_policy                  630	3_0_0	EXIST::FUNCTION:TS
 X509_REQ_dup                            631	3_0_0	EXIST::FUNCTION:
 d2i_DSA_PUBKEY_fp                       633	3_0_0	EXIST::FUNCTION:DSA,STDIO
-OCSP_REQ_CTX_nbio_d2i                   634	3_0_0	EXIST::FUNCTION:OCSP
+OCSP_REQ_CTX_nbio_d2i                   634	3_0_0	EXIST::FUNCTION:OCSP,SOCK
 d2i_X509_REQ_fp                         635	3_0_0	EXIST::FUNCTION:STDIO
 DH_OpenSSL                              636	3_0_0	EXIST::FUNCTION:DH
 BN_get_rfc3526_prime_8192               637	3_0_0	EXIST::FUNCTION:DH
@@ -1243,7 +1243,7 @@ TS_REQ_set_cert_req                     1271	3_0_0	EXIST::FUNCTION:TS
 TXT_DB_get_by_index                     1272	3_0_0	EXIST::FUNCTION:
 X509_check_ca                           1273	3_0_0	EXIST::FUNCTION:
 DH_get_2048_224                         1274	3_0_0	EXIST::FUNCTION:DH
-X509_http_nbio                          1275	3_0_0	EXIST::FUNCTION:OCSP
+X509_load_http                          1275	3_0_0	EXIST::FUNCTION:SOCK
 i2d_AUTHORITY_INFO_ACCESS               1276	3_0_0	EXIST::FUNCTION:
 EVP_get_cipherbyname                    1277	3_0_0	EXIST::FUNCTION:
 CONF_dump_fp                            1278	3_0_0	EXIST::FUNCTION:STDIO
@@ -3615,7 +3615,7 @@ EVP_CIPHER_CTX_encrypting               3694	3_0_0	EXIST::FUNCTION:
 EC_KEY_can_sign                         3695	3_0_0	EXIST::FUNCTION:EC
 PEM_write_bio_RSAPublicKey              3696	3_0_0	EXIST::FUNCTION:RSA
 X509_CRL_set1_lastUpdate                3697	3_0_0	EXIST::FUNCTION:
-OCSP_sendreq_nbio                       3698	3_0_0	EXIST::FUNCTION:OCSP
+OCSP_sendreq_nbio                       3698	3_0_0	EXIST::FUNCTION:OCSP,SOCK
 PKCS8_encrypt                           3699	3_0_0	EXIST::FUNCTION:
 i2d_PKCS7_fp                            3700	3_0_0	EXIST::FUNCTION:STDIO
 i2d_X509_REQ                            3701	3_0_0	EXIST::FUNCTION:
@@ -4923,3 +4923,9 @@ RAND_DRBG_get_callback_data             ?	3_0_0	EXIST::FUNCTION:
 BIO_wait                                ?	3_0_0	EXIST::FUNCTION:SOCK
 BIO_socket_wait                         ?	3_0_0	EXIST::FUNCTION:SOCK
 BIO_connect_retry                       ?	3_0_0	EXIST::FUNCTION:SOCK
+ERR_load_HTTP_strings                   ?	3_0_0	EXIST::FUNCTION:
+OSSL_HTTP_get                           ?	3_0_0	EXIST::FUNCTION:SOCK
+OSSL_HTTP_get_asn1                      ?	3_0_0	EXIST::FUNCTION:SOCK
+OSSL_HTTP_post_asn1                     ?	3_0_0	EXIST::FUNCTION:SOCK
+OSSL_HTTP_transfer                      ?	3_0_0	EXIST::FUNCTION:SOCK
+OSSL_HTTP_proxy_connect                 ?	3_0_0	EXIST::FUNCTION:SOCK
diff --git a/util/missingcrypto.txt b/util/missingcrypto.txt
index 5e8f6dea0d..a1f0c3487e 100644
--- a/util/missingcrypto.txt
+++ b/util/missingcrypto.txt
@@ -597,6 +597,7 @@ ERR_load_ENGINE_strings(3)
 ERR_load_ERR_strings(3)
 ERR_load_ESS_strings(3)
 ERR_load_EVP_strings(3)
+ERR_load_HTTP_strings(3)
 ERR_load_KDF_strings(3)
 ERR_load_OBJ_strings(3)
 ERR_load_OCSP_strings(3)
@@ -1314,7 +1315,6 @@ X509_CRL_diff(3)
 X509_CRL_get_lastUpdate(3)
 X509_CRL_get_meth_data(3)
 X509_CRL_get_nextUpdate(3)
-X509_CRL_http_nbio(3)
 X509_CRL_it(3)
 X509_CRL_print(3)
 X509_CRL_print_ex(3)
@@ -1460,7 +1460,6 @@ X509_get_default_private_dir(3)
 X509_get_pubkey_parameters(3)
 X509_get_signature_type(3)
 X509_gmtime_adj(3)
-X509_http_nbio(3)
 X509_issuer_and_serial_hash(3)
 X509_issuer_name_hash(3)
 X509_issuer_name_hash_old(3)
diff --git a/util/other.syms b/util/other.syms
index bdcc283718..78d436f73a 100644
--- a/util/other.syms
+++ b/util/other.syms
@@ -37,6 +37,8 @@ GEN_SESSION_CB                          datatype
 OPENSSL_Applink                         external
 OPENSSL_CTX                             datatype
 NAMING_AUTHORITY                        datatype
+OCSP_parse_url                          define
+OSSL_HTTP_bio_cb_t                      datatype
 OSSL_PARAM                              datatype
 OSSL_PROVIDER                           datatype
 OSSL_SERIALIZER                         datatype
@@ -369,7 +371,6 @@ OSSL_CMP_log4                           define
 OSSL_CMP_severity                       datatype
 OSSL_CMP_warn                           define
 OSSL_cmp_certConf_cb_t                  datatype
-OSSL_cmp_http_cb_t                      datatype
 OSSL_cmp_log_cb_t                       datatype
 OSSL_cmp_transfer_cb_t                  datatype
 OSSL_PARAM_TYPE                         define
@@ -548,6 +549,8 @@ SSLv23_client_method                    define
 SSLv23_method                           define
 SSLv23_server_method                    define
 TLS_DEFAULT_CIPHERSUITES                define deprecated 3.0.0
+X509_CRL_http_nbio                      define
+X509_http_nbio                          define
 X509_STORE_set_lookup_crls_cb           define
 X509_STORE_set_verify_func              define
 EVP_PKEY_CTX_set1_id                    define