diff options
| author | Janis Schoetterl-Glausch <scgl@linux.ibm.com> | 2022-11-17 11:07:45 +0100 |
|---|---|---|
| committer | Heiko Carstens <hca@linux.ibm.com> | 2022-11-22 19:38:09 +0100 |
| commit | 739ad2e4e15b585a0eaf98b7bdee62b2dd9588c9 (patch) | |
| tree | 47e3bf007b081b621e14f43bb887d085877ae05b /arch | |
| parent | s390/cmpxchg: make loop condition for 1,2 byte cases precise (diff) | |
| download | linux-739ad2e4e15b585a0eaf98b7bdee62b2dd9588c9.tar.xz linux-739ad2e4e15b585a0eaf98b7bdee62b2dd9588c9.zip | |
s390/uaccess: limit number of retries for cmpxchg_user_key()
cmpxchg_user_key() for byte and short values is implemented via a one word
cmpxchg loop. Give up trying to perform the cmpxchg if it fails too often
because of contention on the cache line. This ensures that the thread
cannot become stuck in the kernel.
Signed-off-by: Janis Schoetterl-Glausch <scgl@linux.ibm.com>
Link: https://lore.kernel.org/r/20221117100745.3253896-1-scgl@linux.ibm.com
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
Diffstat (limited to 'arch')
| -rw-r--r-- | arch/s390/include/asm/uaccess.h | 33 |
1 files changed, 25 insertions, 8 deletions
diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index d028ee59e941..7c10f594e747 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -392,6 +392,8 @@ do { \ void __cmpxchg_user_key_called_with_bad_pointer(void); +#define CMPXCHG_USER_KEY_MAX_LOOPS 128 + static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval, __uint128_t old, __uint128_t new, unsigned long key, int size) @@ -401,6 +403,7 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval, switch (size) { case 1: { unsigned int prev, shift, mask, _old, _new; + unsigned long count; shift = (3 ^ (address & 3)) << 3; address ^= address & 3; @@ -410,6 +413,7 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval, asm volatile( " spka 0(%[key])\n" " sacf 256\n" + " llill %[count],%[max_loops]\n" "0: l %[prev],%[address]\n" "1: nr %[prev],%[mask]\n" " xilf %[mask],0xffffffff\n" @@ -421,7 +425,8 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval, " xr %[tmp],%[prev]\n" " xr %[new],%[tmp]\n" " nr %[tmp],%[mask]\n" - " jz 2b\n" + " jnz 5f\n" + " brct %[count],2b\n" "5: sacf 768\n" " spka %[default_key]\n" EX_TABLE_UA_LOAD_REG(0b, 5b, %[rc], %[prev]) @@ -433,15 +438,20 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval, [address] "+Q" (*(int *)address), [tmp] "+&d" (_old), [new] "+&d" (_new), - [mask] "+&d" (mask) - : [key] "a" (key << 4), - [default_key] "J" (PAGE_DEFAULT_KEY) + [mask] "+&d" (mask), + [count] "=a" (count) + : [key] "%[count]" (key << 4), + [default_key] "J" (PAGE_DEFAULT_KEY), + [max_loops] "J" (CMPXCHG_USER_KEY_MAX_LOOPS) : "memory", "cc"); *(unsigned char *)uval = prev >> shift; + if (!count) + rc = -EAGAIN; return rc; } case 2: { unsigned int prev, shift, mask, _old, _new; + unsigned long count; shift = (2 ^ (address & 2)) << 3; address ^= address & 2; @@ -451,6 +461,7 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval, asm volatile( " spka 0(%[key])\n" " sacf 256\n" + " llill %[count],%[max_loops]\n" "0: l %[prev],%[address]\n" "1: nr %[prev],%[mask]\n" " xilf %[mask],0xffffffff\n" @@ -462,7 +473,8 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval, " xr %[tmp],%[prev]\n" " xr %[new],%[tmp]\n" " nr %[tmp],%[mask]\n" - " jz 2b\n" + " jnz 5f\n" + " brct %[count],2b\n" "5: sacf 768\n" " spka %[default_key]\n" EX_TABLE_UA_LOAD_REG(0b, 5b, %[rc], %[prev]) @@ -474,11 +486,15 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval, [address] "+Q" (*(int *)address), [tmp] "+&d" (_old), [new] "+&d" (_new), - [mask] "+&d" (mask) - : [key] "a" (key << 4), - [default_key] "J" (PAGE_DEFAULT_KEY) + [mask] "+&d" (mask), + [count] "=a" (count) + : [key] "%[count]" (key << 4), + [default_key] "J" (PAGE_DEFAULT_KEY), + [max_loops] "J" (CMPXCHG_USER_KEY_MAX_LOOPS) : "memory", "cc"); *(unsigned short *)uval = prev >> shift; + if (!count) + rc = -EAGAIN; return rc; } case 4: { @@ -568,6 +584,7 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval, * * Return: 0: cmpxchg executed * -EFAULT: an exception happened when trying to access *@ptr + * -EAGAIN: maxed out number of retries (byte and short only) */ #define cmpxchg_user_key(ptr, uval, old, new, key) \ ({ \ |