asn1/tasn_utl.c: fix logical error in and overhaul asn1_do_lock.

CRYPTO_atomic_add was assumed to return negative value on error, while
it returns 0.

Reviewed-by: Rich Salz <rsalz@openssl.org>

1 files changed, 23 insertions, 12 deletions

diff --git a/crypto/asn1/tasn_utl.c b/crypto/asn1/tasn_utl.c
index a93fd871dc..7ceecffce7 100644
--- a/crypto/asn1/tasn_utl.c
+++ b/crypto/asn1/tasn_utl.c
@@ -58,8 +58,10 @@ int asn1_set_choice_selector(ASN1_VALUE **pval, int value,
 int asn1_do_lock(ASN1_VALUE **pval, int op, const ASN1_ITEM *it)
 {
     const ASN1_AUX *aux;
-    int *lck, ret;
+    CRYPTO_REF_COUNT *lck;
     CRYPTO_RWLOCK **lock;
+    int ret = -1;
+
     if ((it->itype != ASN1_ITYPE_SEQUENCE)
         && (it->itype != ASN1_ITYPE_NDEF_SEQUENCE))
         return 0;
@@ -68,25 +70,34 @@ int asn1_do_lock(ASN1_VALUE **pval, int op, const ASN1_ITEM *it)
         return 0;
     lck = offset2ptr(*pval, aux->ref_offset);
     lock = offset2ptr(*pval, aux->ref_lock);
-    if (op == 0) {
-        *lck = 1;
+
+    switch (op) {
+    case 0:
+        *lck = ret = 1;
         *lock = CRYPTO_THREAD_lock_new();
         if (*lock == NULL) {
             ASN1err(ASN1_F_ASN1_DO_LOCK, ERR_R_MALLOC_FAILURE);
             return -1;
         }
-        return 1;
-    }
-    if (CRYPTO_atomic_add(lck, op, &ret, *lock) < 0)
-        return -1;  /* failed */
+        break;
+    case 1:
+        if (!CRYPTO_UP_REF(lck, &ret, *lock))
+            return -1;
+        break;
+    case -1:
+        if (!CRYPTO_DOWN_REF(lck, &ret, *lock))
+            return -1;  /* failed */
 #ifdef REF_PRINT
-    fprintf(stderr, "%p:%4d:%s\n", it, *lck, it->sname);
+        fprintf(stderr, "%p:%4d:%s\n", it, ret, it->sname);
 #endif
-    REF_ASSERT_ISNT(ret < 0);
-    if (ret == 0) {
-        CRYPTO_THREAD_lock_free(*lock);
-        *lock = NULL;
+        REF_ASSERT_ISNT(ret < 0);
+        if (ret == 0) {
+            CRYPTO_THREAD_lock_free(*lock);
+            *lock = NULL;
+        }
+        break;
     }
+
     return ret;
 }