diff options
Diffstat (limited to 'arch/mips/include/asm/cmpxchg.h')
| -rw-r--r-- | arch/mips/include/asm/cmpxchg.h | 27 |
1 files changed, 18 insertions, 9 deletions
diff --git a/arch/mips/include/asm/cmpxchg.h b/arch/mips/include/asm/cmpxchg.h index f345a873742d..f6136871561d 100644 --- a/arch/mips/include/asm/cmpxchg.h +++ b/arch/mips/include/asm/cmpxchg.h @@ -46,6 +46,7 @@ extern unsigned long __xchg_called_with_bad_pointer(void) __typeof(*(m)) __ret; \ \ if (kernel_uses_llsc) { \ + loongson_llsc_mb(); \ __asm__ __volatile__( \ " .set push \n" \ " .set noat \n" \ @@ -60,7 +61,7 @@ extern unsigned long __xchg_called_with_bad_pointer(void) " .set pop \n" \ : "=&r" (__ret), "=" GCC_OFF_SMALL_ASM() (*m) \ : GCC_OFF_SMALL_ASM() (*m), "Jr" (val) \ - : "memory"); \ + : __LLSC_CLOBBER); \ } else { \ unsigned long __flags; \ \ @@ -76,8 +77,8 @@ extern unsigned long __xchg_called_with_bad_pointer(void) extern unsigned long __xchg_small(volatile void *ptr, unsigned long val, unsigned int size); -static inline unsigned long __xchg(volatile void *ptr, unsigned long x, - int size) +static __always_inline +unsigned long __xchg(volatile void *ptr, unsigned long x, int size) { switch (size) { case 1: @@ -117,6 +118,7 @@ static inline unsigned long __xchg(volatile void *ptr, unsigned long x, __typeof(*(m)) __ret; \ \ if (kernel_uses_llsc) { \ + loongson_llsc_mb(); \ __asm__ __volatile__( \ " .set push \n" \ " .set noat \n" \ @@ -132,8 +134,9 @@ static inline unsigned long __xchg(volatile void *ptr, unsigned long x, " .set pop \n" \ "2: \n" \ : "=&r" (__ret), "=" GCC_OFF_SMALL_ASM() (*m) \ - : GCC_OFF_SMALL_ASM() (*m), "Jr" (old), "Jr" (new) \ - : "memory"); \ + : GCC_OFF_SMALL_ASM() (*m), "Jr" (old), "Jr" (new) \ + : __LLSC_CLOBBER); \ + loongson_llsc_mb(); \ } else { \ unsigned long __flags; \ \ @@ -150,8 +153,9 @@ static inline unsigned long __xchg(volatile void *ptr, unsigned long x, extern unsigned long __cmpxchg_small(volatile void *ptr, unsigned long old, unsigned long new, unsigned int size); -static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, - unsigned long new, unsigned int size) +static __always_inline +unsigned long __cmpxchg(volatile void *ptr, unsigned long old, + unsigned long new, unsigned int size) { switch (size) { case 1: @@ -229,6 +233,7 @@ static inline unsigned long __cmpxchg64(volatile void *ptr, */ local_irq_save(flags); + loongson_llsc_mb(); asm volatile( " .set push \n" " .set " MIPS_ISA_ARCH_LEVEL " \n" @@ -274,6 +279,7 @@ static inline unsigned long __cmpxchg64(volatile void *ptr, "r" (old), "r" (new) : "memory"); + loongson_llsc_mb(); local_irq_restore(flags); return ret; @@ -290,10 +296,13 @@ static inline unsigned long __cmpxchg64(volatile void *ptr, * will cause a build error unless cpu_has_64bits is a \ * compile-time constant 1. \ */ \ - if (cpu_has_64bits && kernel_uses_llsc) \ + if (cpu_has_64bits && kernel_uses_llsc) { \ + smp_mb__before_llsc(); \ __res = __cmpxchg64((ptr), __old, __new); \ - else \ + smp_llsc_mb(); \ + } else { \ __res = __cmpxchg64_unsupported(); \ + } \ \ __res; \ }) |