Change the return type of EVP_EncodeUpdate

Previously EVP_EncodeUpdate returned a void. However there are a couple
of error conditions that can occur. Therefore the return type has been
changed to an int, with 0 indicating error and 1 indicating success.

Reviewed-by: Rich Salz <rsalz@openssl.org>

3 files changed, 12 insertions, 8 deletions

diff --git a/crypto/evp/bio_b64.c b/crypto/evp/bio_b64.c
index 9067848578..32a884a711 100644
--- a/crypto/evp/bio_b64.c
+++ b/crypto/evp/bio_b64.c
@@ -403,9 +403,10 @@ static int b64_write(BIO *b, const char *in, int inl)
                 ret += n;
             }
         } else {
-            EVP_EncodeUpdate(ctx->base64,
-                             (unsigned char *)ctx->buf, &ctx->buf_len,
-                             (unsigned char *)in, n);
+            if (!EVP_EncodeUpdate(ctx->base64,
+                                 (unsigned char *)ctx->buf, &ctx->buf_len,
+                                 (unsigned char *)in, n))
+                return ((ret == 0) ? -1 : ret);
             OPENSSL_assert(ctx->buf_len <= (int)sizeof(ctx->buf));
             OPENSSL_assert(ctx->buf_len >= ctx->buf_off);
             ret += n;
diff --git a/crypto/evp/encode.c b/crypto/evp/encode.c
index bd2bbc0b09..e026a8f0ca 100644
--- a/crypto/evp/encode.c
+++ b/crypto/evp/encode.c
@@ -114,7 +114,7 @@ void EVP_EncodeInit(EVP_ENCODE_CTX *ctx)
     ctx->line_num = 0;
 }
 
-void EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
+int EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
                       const unsigned char *in, int inl)
 {
     int i, j;
@@ -122,12 +122,12 @@ void EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
 
     *outl = 0;
     if (inl <= 0)
-        return;
+        return 0;
     OPENSSL_assert(ctx->length <= (int)sizeof(ctx->enc_data));
     if (ctx->length - ctx->num > inl) {
         memcpy(&(ctx->enc_data[ctx->num]), in, inl);
         ctx->num += inl;
-        return;
+        return 1;
     }
     if (ctx->num != 0) {
         i = ctx->length - ctx->num;
@@ -153,12 +153,14 @@ void EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
     if (total > INT_MAX) {
         /* Too much output data! */
         *outl = 0;
-        return;
+        return 0;
     }
     if (inl != 0)
         memcpy(&(ctx->enc_data[0]), in, inl);
     ctx->num = inl;
     *outl = total;
+
+    return 1;
 }
 
 void EVP_EncodeFinal(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl)
diff --git a/crypto/pem/pem_lib.c b/crypto/pem/pem_lib.c
index 90893f1954..8965fda8d0 100644
--- a/crypto/pem/pem_lib.c
+++ b/crypto/pem/pem_lib.c
@@ -618,7 +618,8 @@ int PEM_write_bio(BIO *bp, const char *name, const char *header,
     i = j = 0;
     while (len > 0) {
         n = (int)((len > (PEM_BUFSIZE * 5)) ? (PEM_BUFSIZE * 5) : len);
-        EVP_EncodeUpdate(ctx, buf, &outl, &(data[j]), n);
+        if (!EVP_EncodeUpdate(ctx, buf, &outl, &(data[j]), n))
+            goto err;
         if ((outl) && (BIO_write(bp, (char *)buf, outl) != outl))
             goto err;
         i += outl;