diff options
| author | Will Deacon <will.deacon@arm.com> | 2015-05-29 14:31:10 +0200 |
|---|---|---|
| committer | Will Deacon <will.deacon@arm.com> | 2015-07-27 16:28:53 +0200 |
| commit | 0ea366f5e1b6413a6095dce60ea49ae51e468b61 (patch) | |
| tree | fce4fc690edf16784d21a714415a74a8ce53eb2b /arch/arm64/include/asm/cmpxchg.h | |
| parent | arm64: atomics: tidy up common atomic{,64}_* macros (diff) | |
| download | linux-0ea366f5e1b6413a6095dce60ea49ae51e468b61.tar.xz linux-0ea366f5e1b6413a6095dce60ea49ae51e468b61.zip | |
arm64: atomics: prefetch the destination word for write prior to stxr
The cost of changing a cacheline from shared to exclusive state can be
significant, especially when this is triggered by an exclusive store,
since it may result in having to retry the transaction.
This patch makes use of prfm to prefetch cachelines for write prior to
ldxr/stxr loops when using the ll/sc atomic routines.
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Diffstat (limited to 'arch/arm64/include/asm/cmpxchg.h')
| -rw-r--r-- | arch/arm64/include/asm/cmpxchg.h | 8 |
1 files changed, 8 insertions, 0 deletions
diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h index f70212629d02..7bfda0944c9b 100644 --- a/arch/arm64/include/asm/cmpxchg.h +++ b/arch/arm64/include/asm/cmpxchg.h @@ -33,12 +33,14 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size case 1: asm volatile(ARM64_LSE_ATOMIC_INSN( /* LL/SC */ + " prfm pstl1strm, %2\n" "1: ldxrb %w0, %2\n" " stlxrb %w1, %w3, %2\n" " cbnz %w1, 1b\n" " dmb ish", /* LSE atomics */ " nop\n" + " nop\n" " swpalb %w3, %w0, %2\n" " nop\n" " nop") @@ -49,12 +51,14 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size case 2: asm volatile(ARM64_LSE_ATOMIC_INSN( /* LL/SC */ + " prfm pstl1strm, %2\n" "1: ldxrh %w0, %2\n" " stlxrh %w1, %w3, %2\n" " cbnz %w1, 1b\n" " dmb ish", /* LSE atomics */ " nop\n" + " nop\n" " swpalh %w3, %w0, %2\n" " nop\n" " nop") @@ -65,12 +69,14 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size case 4: asm volatile(ARM64_LSE_ATOMIC_INSN( /* LL/SC */ + " prfm pstl1strm, %2\n" "1: ldxr %w0, %2\n" " stlxr %w1, %w3, %2\n" " cbnz %w1, 1b\n" " dmb ish", /* LSE atomics */ " nop\n" + " nop\n" " swpal %w3, %w0, %2\n" " nop\n" " nop") @@ -81,12 +87,14 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size case 8: asm volatile(ARM64_LSE_ATOMIC_INSN( /* LL/SC */ + " prfm pstl1strm, %2\n" "1: ldxr %0, %2\n" " stlxr %w1, %3, %2\n" " cbnz %w1, 1b\n" " dmb ish", /* LSE atomics */ " nop\n" + " nop\n" " swpal %3, %0, %2\n" " nop\n" " nop") |