DESERIALIZER: Make OSSL_DESERIALIZER_from_{bio,fp} use BIO_tell() / BIO_seek()

Depending on the BIO used, using BIO_reset() may lead to "interesting"
results.  For example, a BIO_f_buffer() on top of another BIO that
handles BIO_reset() as a BIO_seek(bio, 0), the deserialization process
may find itself with a file that's rewound more than expected.

Therefore, OSSL_DESERIALIZER_from_{bio,fp}'s behaviour is changed to
rely purely on BIO_tell() / BIO_seek(), and since BIO_s_mem() is used
internally, it's changed to handle BIO_tell() and BIO_seek() better.

This does currently mean that OSSL_DESERIALIZER can't be easily used
with streams that don't support BIO_tell() / BIO_seek().

Fixes #12541

Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/12544)

1 files changed, 22 insertions, 3 deletions

diff --git a/crypto/bio/bss_mem.c b/crypto/bio/bss_mem.c
index 4043043626..d9580e6d37 100644
--- a/crypto/bio/bss_mem.c
+++ b/crypto/bio/bss_mem.c
@@ -176,6 +176,7 @@ static int mem_buf_free(BIO *a)
 
 /*
  * Reallocate memory buffer if read pointer differs
+ * NOT FOR RDONLY
  */
 static int mem_buf_sync(BIO *b)
 {
@@ -247,12 +248,18 @@ static long mem_ctrl(BIO *b, int cmd, long num, void *ptr)
     long ret = 1;
     char **pptr;
     BIO_BUF_MEM *bbm = (BIO_BUF_MEM *)b->ptr;
-    BUF_MEM *bm;
+    BUF_MEM *bm, *bo;            /* bio_mem, bio_other */
+    long off, remain;
 
-    if (b->flags & BIO_FLAGS_MEM_RDONLY)
+    if (b->flags & BIO_FLAGS_MEM_RDONLY) {
         bm = bbm->buf;
-    else
+        bo = bbm->readp;
+    } else {
         bm = bbm->readp;
+        bo = bbm->buf;
+    }
+    off = bm->data - bo->data;
+    remain = bm->length;
 
     switch (cmd) {
     case BIO_CTRL_RESET:
@@ -270,6 +277,18 @@ static long mem_ctrl(BIO *b, int cmd, long num, void *ptr)
             }
         }
         break;
+    case BIO_C_FILE_SEEK:
+        if (num < 0 || num > off + remain)
+            return -1;   /* Can't see outside of the current buffer */
+
+        bm->data = bo->data + num;
+        bm->length = bo->length - num;
+        bm->max = bo->max - num;
+        off = num;
+        /* FALLTHRU */
+    case BIO_C_FILE_TELL:
+        ret = off;
+        break;
     case BIO_CTRL_EOF:
         ret = (long)(bm->length == 0);
         break;