From 270c10e00a1e557e068803a22e0556281ceb1830 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Sat, 27 Oct 2012 18:05:20 -0700 Subject: sparc64: Fix cpu strand yielding. For atomic backoff, we just loop over an exponentially backed off counter. This is extremely ineffective as it doesn't actually yield the cpu strand so that other competing strands can use the cpu core. In cpus previous to SPARC-T4 we have to do this in a slightly hackish way, by doing an operation with no side effects that also happens to mark the strand as unavailable. The mechanism we choose for this is three reads of the %ccr (condition-code) register into %g0 (the zero register). SPARC-T4 has an explicit "pause" instruction, and we'll make use of that in a subsequent commit. Yield strands also in cpu_relax(). We really should have done this a very long time ago. Signed-off-by: David S. Miller --- arch/sparc/include/asm/backoff.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'arch/sparc/include/asm/backoff.h') diff --git a/arch/sparc/include/asm/backoff.h b/arch/sparc/include/asm/backoff.h index db3af0d30fb1..64b077b3b13b 100644 --- a/arch/sparc/include/asm/backoff.h +++ b/arch/sparc/include/asm/backoff.h @@ -13,7 +13,10 @@ #define BACKOFF_SPIN(reg, tmp, label) \ mov reg, tmp; \ -88: brnz,pt tmp, 88b; \ +88: rd %ccr, %g0; \ + rd %ccr, %g0; \ + rd %ccr, %g0; \ + brnz,pt tmp, 88b; \ sub tmp, 1, tmp; \ set BACKOFF_LIMIT, tmp; \ cmp reg, tmp; \ -- cgit v1.2.3 From e9b9eb59ffcdee09ec96b040f85c919618f4043e Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Sat, 27 Oct 2012 23:00:41 -0700 Subject: sparc64: Use pause instruction when available. In atomic backoff and cpu_relax(), use the pause instruction found on SPARC-T4 and later. It makes the cpu strand unselectable for the given number of cycles, unless an intervening disrupting trap occurs. Signed-off-by: David S. Miller --- arch/sparc/include/asm/backoff.h | 32 +++++++++++++++++++------------- arch/sparc/include/asm/processor_64.h | 13 ++++++++++--- arch/sparc/kernel/entry.h | 7 +++++++ arch/sparc/kernel/setup_64.c | 21 +++++++++++++++++++++ arch/sparc/kernel/vmlinux.lds.S | 5 +++++ 5 files changed, 62 insertions(+), 16 deletions(-) (limited to 'arch/sparc/include/asm/backoff.h') diff --git a/arch/sparc/include/asm/backoff.h b/arch/sparc/include/asm/backoff.h index 64b077b3b13b..20f01df0871b 100644 --- a/arch/sparc/include/asm/backoff.h +++ b/arch/sparc/include/asm/backoff.h @@ -11,19 +11,25 @@ #define BACKOFF_LABEL(spin_label, continue_label) \ spin_label -#define BACKOFF_SPIN(reg, tmp, label) \ - mov reg, tmp; \ -88: rd %ccr, %g0; \ - rd %ccr, %g0; \ - rd %ccr, %g0; \ - brnz,pt tmp, 88b; \ - sub tmp, 1, tmp; \ - set BACKOFF_LIMIT, tmp; \ - cmp reg, tmp; \ - bg,pn %xcc, label; \ - nop; \ - ba,pt %xcc, label; \ - sllx reg, 1, reg; +#define BACKOFF_SPIN(reg, tmp, label) \ + mov reg, tmp; \ +88: rd %ccr, %g0; \ + rd %ccr, %g0; \ + rd %ccr, %g0; \ + .section .pause_patch,"ax"; \ + .word 88b; \ + sllx tmp, 7, tmp; \ + wr tmp, 0, %asr27; \ + clr tmp; \ + .previous; \ + brnz,pt tmp, 88b; \ + sub tmp, 1, tmp; \ + set BACKOFF_LIMIT, tmp; \ + cmp reg, tmp; \ + bg,pn %xcc, label; \ + nop; \ + ba,pt %xcc, label; \ + sllx reg, 1, reg; #else diff --git a/arch/sparc/include/asm/processor_64.h b/arch/sparc/include/asm/processor_64.h index 986563409469..9cdf52eec48a 100644 --- a/arch/sparc/include/asm/processor_64.h +++ b/arch/sparc/include/asm/processor_64.h @@ -196,9 +196,16 @@ extern unsigned long get_wchan(struct task_struct *task); #define KSTK_EIP(tsk) (task_pt_regs(tsk)->tpc) #define KSTK_ESP(tsk) (task_pt_regs(tsk)->u_regs[UREG_FP]) -#define cpu_relax() asm volatile("rd %%ccr, %%g0\n\t" \ - "rd %%ccr, %%g0\n\t" \ - "rd %%ccr, %%g0" \ +#define cpu_relax() asm volatile("\n99:\n\t" \ + "rd %%ccr, %%g0\n\t" \ + "rd %%ccr, %%g0\n\t" \ + "rd %%ccr, %%g0\n\t" \ + ".section .pause_patch,\"ax\"\n\t"\ + ".word 99b\n\t" \ + "wr %%g0, 128, %%asr27\n\t" \ + "nop\n\t" \ + "nop\n\t" \ + ".previous" \ ::: "memory") /* Prefetch support. This is tuned for UltraSPARC-III and later. diff --git a/arch/sparc/kernel/entry.h b/arch/sparc/kernel/entry.h index 0c218e4c0881..51742df63c75 100644 --- a/arch/sparc/kernel/entry.h +++ b/arch/sparc/kernel/entry.h @@ -59,6 +59,13 @@ struct popc_6insn_patch_entry { extern struct popc_6insn_patch_entry __popc_6insn_patch, __popc_6insn_patch_end; +struct pause_patch_entry { + unsigned int addr; + unsigned int insns[3]; +}; +extern struct pause_patch_entry __pause_patch, + __pause_patch_end; + extern void __init per_cpu_patch(void); extern void sun4v_patch_1insn_range(struct sun4v_1insn_patch_entry *, struct sun4v_1insn_patch_entry *); diff --git a/arch/sparc/kernel/setup_64.c b/arch/sparc/kernel/setup_64.c index 0800e71d8a88..b45cff408de3 100644 --- a/arch/sparc/kernel/setup_64.c +++ b/arch/sparc/kernel/setup_64.c @@ -316,6 +316,25 @@ static void __init popc_patch(void) } } +static void __init pause_patch(void) +{ + struct pause_patch_entry *p; + + p = &__pause_patch; + while (p < &__pause_patch_end) { + unsigned long i, addr = p->addr; + + for (i = 0; i < 3; i++) { + *(unsigned int *) (addr + (i * 4)) = p->insns[i]; + wmb(); + __asm__ __volatile__("flush %0" + : : "r" (addr + (i * 4))); + } + + p++; + } +} + #ifdef CONFIG_SMP void __init boot_cpu_id_too_large(int cpu) { @@ -528,6 +547,8 @@ static void __init init_sparc64_elf_hwcap(void) if (sparc64_elf_hwcap & AV_SPARC_POPC) popc_patch(); + if (sparc64_elf_hwcap & AV_SPARC_PAUSE) + pause_patch(); } void __init setup_arch(char **cmdline_p) diff --git a/arch/sparc/kernel/vmlinux.lds.S b/arch/sparc/kernel/vmlinux.lds.S index 89c2c29f154b..847f9f793618 100644 --- a/arch/sparc/kernel/vmlinux.lds.S +++ b/arch/sparc/kernel/vmlinux.lds.S @@ -132,6 +132,11 @@ SECTIONS *(.popc_6insn_patch) __popc_6insn_patch_end = .; } + .pause_patch : { + __pause_patch = .; + *(.pause_patch) + __pause_patch_end = .; + } PERCPU_SECTION(SMP_CACHE_BYTES) . = ALIGN(PAGE_SIZE); -- cgit v1.2.3 From 187818cd6a5ab6343eac47e52da2f3e40c544b98 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Sun, 28 Oct 2012 13:04:47 -0700 Subject: sparc64: Improvde documentation and readability of atomic backoff code. Document what's going on in asm/backoff.h with a large and descriptive comment. Refer to it above the cpu_relax() definition in asm/processor_64.h Rename the pause patching section to have "3insn" in it's name like the other patching sections do. Based upon feedback from Sam Ravnborg. Signed-off-by: David S. Miller --- arch/sparc/include/asm/backoff.h | 42 ++++++++++++++++++++++++++++++++++- arch/sparc/include/asm/processor_64.h | 7 +++++- arch/sparc/kernel/entry.h | 4 ++-- arch/sparc/kernel/setup_64.c | 4 ++-- arch/sparc/kernel/vmlinux.lds.S | 8 +++---- 5 files changed, 55 insertions(+), 10 deletions(-) (limited to 'arch/sparc/include/asm/backoff.h') diff --git a/arch/sparc/include/asm/backoff.h b/arch/sparc/include/asm/backoff.h index 20f01df0871b..4e02086b839c 100644 --- a/arch/sparc/include/asm/backoff.h +++ b/arch/sparc/include/asm/backoff.h @@ -1,6 +1,46 @@ #ifndef _SPARC64_BACKOFF_H #define _SPARC64_BACKOFF_H +/* The macros in this file implement an exponential backoff facility + * for atomic operations. + * + * When multiple threads compete on an atomic operation, it is + * possible for one thread to be continually denied a successful + * completion of the compare-and-swap instruction. Heavily + * threaded cpu implementations like Niagara can compound this + * problem even further. + * + * When an atomic operation fails and needs to be retried, we spin a + * certain number of times. At each subsequent failure of the same + * operation we double the spin count, realizing an exponential + * backoff. + * + * When we spin, we try to use an operation that will cause the + * current cpu strand to block, and therefore make the core fully + * available to any other other runnable strands. There are two + * options, based upon cpu capabilities. + * + * On all cpus prior to SPARC-T4 we do three dummy reads of the + * condition code register. Each read blocks the strand for something + * between 40 and 50 cpu cycles. + * + * For SPARC-T4 and later we have a special "pause" instruction + * available. This is implemented using writes to register %asr27. + * The cpu will block the number of cycles written into the register, + * unless a disrupting trap happens first. SPARC-T4 specifically + * implements pause with a granularity of 8 cycles. Each strand has + * an internal pause counter which decrements every 8 cycles. So the + * chip shifts the %asr27 value down by 3 bits, and writes the result + * into the pause counter. If a value smaller than 8 is written, the + * chip blocks for 1 cycle. + * + * To achieve the same amount of backoff as the three %ccr reads give + * on earlier chips, we shift the backoff value up by 7 bits. (Three + * %ccr reads block for about 128 cycles, 1 << 7 == 128) We write the + * whole amount we want to block into the pause register, rather than + * loop writing 128 each time. + */ + #define BACKOFF_LIMIT (4 * 1024) #ifdef CONFIG_SMP @@ -16,7 +56,7 @@ 88: rd %ccr, %g0; \ rd %ccr, %g0; \ rd %ccr, %g0; \ - .section .pause_patch,"ax"; \ + .section .pause_3insn_patch,"ax";\ .word 88b; \ sllx tmp, 7, tmp; \ wr tmp, 0, %asr27; \ diff --git a/arch/sparc/include/asm/processor_64.h b/arch/sparc/include/asm/processor_64.h index 9cdf52eec48a..721e25f0e2ea 100644 --- a/arch/sparc/include/asm/processor_64.h +++ b/arch/sparc/include/asm/processor_64.h @@ -196,11 +196,16 @@ extern unsigned long get_wchan(struct task_struct *task); #define KSTK_EIP(tsk) (task_pt_regs(tsk)->tpc) #define KSTK_ESP(tsk) (task_pt_regs(tsk)->u_regs[UREG_FP]) +/* Please see the commentary in asm/backoff.h for a description of + * what these instructions are doing and how they have been choosen. + * To make a long story short, we are trying to yield the current cpu + * strand during busy loops. + */ #define cpu_relax() asm volatile("\n99:\n\t" \ "rd %%ccr, %%g0\n\t" \ "rd %%ccr, %%g0\n\t" \ "rd %%ccr, %%g0\n\t" \ - ".section .pause_patch,\"ax\"\n\t"\ + ".section .pause_3insn_patch,\"ax\"\n\t"\ ".word 99b\n\t" \ "wr %%g0, 128, %%asr27\n\t" \ "nop\n\t" \ diff --git a/arch/sparc/kernel/entry.h b/arch/sparc/kernel/entry.h index 51742df63c75..cc3c5cb47cda 100644 --- a/arch/sparc/kernel/entry.h +++ b/arch/sparc/kernel/entry.h @@ -63,8 +63,8 @@ struct pause_patch_entry { unsigned int addr; unsigned int insns[3]; }; -extern struct pause_patch_entry __pause_patch, - __pause_patch_end; +extern struct pause_patch_entry __pause_3insn_patch, + __pause_3insn_patch_end; extern void __init per_cpu_patch(void); extern void sun4v_patch_1insn_range(struct sun4v_1insn_patch_entry *, diff --git a/arch/sparc/kernel/setup_64.c b/arch/sparc/kernel/setup_64.c index b45cff408de3..0eaf0059aaef 100644 --- a/arch/sparc/kernel/setup_64.c +++ b/arch/sparc/kernel/setup_64.c @@ -320,8 +320,8 @@ static void __init pause_patch(void) { struct pause_patch_entry *p; - p = &__pause_patch; - while (p < &__pause_patch_end) { + p = &__pause_3insn_patch; + while (p < &__pause_3insn_patch_end) { unsigned long i, addr = p->addr; for (i = 0; i < 3; i++) { diff --git a/arch/sparc/kernel/vmlinux.lds.S b/arch/sparc/kernel/vmlinux.lds.S index 847f9f793618..0bacceb19150 100644 --- a/arch/sparc/kernel/vmlinux.lds.S +++ b/arch/sparc/kernel/vmlinux.lds.S @@ -132,10 +132,10 @@ SECTIONS *(.popc_6insn_patch) __popc_6insn_patch_end = .; } - .pause_patch : { - __pause_patch = .; - *(.pause_patch) - __pause_patch_end = .; + .pause_3insn_patch : { + __pause_3insn_patch = .; + *(.pause_3insn_patch) + __pause_3insn_patch_end = .; } PERCPU_SECTION(SMP_CACHE_BYTES) -- cgit v1.2.3