From d2e60075a3d4422dc54b919f3b125d8066b839d4 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Wed, 14 Feb 2018 01:08:12 +1000 Subject: powerpc/64: Use array of paca pointers and allocate pacas individually Change the paca array into an array of pointers to pacas. Allocate pacas individually. This allows flexibility in where the PACAs are allocated. Future work will allocate them node-local. Platforms that don't have address limits on PACAs would be able to defer PACA allocations until later in boot rather than allocate all possible ones up-front then freeing unused. This is slightly more overhead (one additional indirection) for cross CPU paca references, but those aren't too common. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/setup_64.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) (limited to 'arch/powerpc/kernel/setup_64.c') diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c index c388cc3357fa..3ce12af4906f 100644 --- a/arch/powerpc/kernel/setup_64.c +++ b/arch/powerpc/kernel/setup_64.c @@ -110,7 +110,7 @@ void __init setup_tlb_core_data(void) if (cpu_first_thread_sibling(boot_cpuid) == first) first = boot_cpuid; - paca[cpu].tcd_ptr = &paca[first].tcd; + paca_ptrs[cpu]->tcd_ptr = &paca_ptrs[first]->tcd; /* * If we have threads, we need either tlbsrx. @@ -304,7 +304,7 @@ void __init early_setup(unsigned long dt_ptr) early_init_devtree(__va(dt_ptr)); /* Now we know the logical id of our boot cpu, setup the paca. */ - setup_paca(&paca[boot_cpuid]); + setup_paca(paca_ptrs[boot_cpuid]); fixup_boot_paca(); /* @@ -628,15 +628,15 @@ void __init exc_lvl_early_init(void) for_each_possible_cpu(i) { sp = memblock_alloc(THREAD_SIZE, THREAD_SIZE); critirq_ctx[i] = (struct thread_info *)__va(sp); - paca[i].crit_kstack = __va(sp + THREAD_SIZE); + paca_ptrs[i]->crit_kstack = __va(sp + THREAD_SIZE); sp = memblock_alloc(THREAD_SIZE, THREAD_SIZE); dbgirq_ctx[i] = (struct thread_info *)__va(sp); - paca[i].dbg_kstack = __va(sp + THREAD_SIZE); + paca_ptrs[i]->dbg_kstack = __va(sp + THREAD_SIZE); sp = memblock_alloc(THREAD_SIZE, THREAD_SIZE); mcheckirq_ctx[i] = (struct thread_info *)__va(sp); - paca[i].mc_kstack = __va(sp + THREAD_SIZE); + paca_ptrs[i]->mc_kstack = __va(sp + THREAD_SIZE); } if (cpu_has_feature(CPU_FTR_DEBUG_LVL_EXC)) @@ -693,20 +693,20 @@ void __init emergency_stack_init(void) ti = __va(memblock_alloc_base(THREAD_SIZE, THREAD_SIZE, limit)); memset(ti, 0, THREAD_SIZE); emerg_stack_init_thread_info(ti, i); - paca[i].emergency_sp = (void *)ti + THREAD_SIZE; + paca_ptrs[i]->emergency_sp = (void *)ti + THREAD_SIZE; #ifdef CONFIG_PPC_BOOK3S_64 /* emergency stack for NMI exception handling. */ ti = __va(memblock_alloc_base(THREAD_SIZE, THREAD_SIZE, limit)); memset(ti, 0, THREAD_SIZE); emerg_stack_init_thread_info(ti, i); - paca[i].nmi_emergency_sp = (void *)ti + THREAD_SIZE; + paca_ptrs[i]->nmi_emergency_sp = (void *)ti + THREAD_SIZE; /* emergency stack for machine check exception handling. */ ti = __va(memblock_alloc_base(THREAD_SIZE, THREAD_SIZE, limit)); memset(ti, 0, THREAD_SIZE); emerg_stack_init_thread_info(ti, i); - paca[i].mc_emergency_sp = (void *)ti + THREAD_SIZE; + paca_ptrs[i]->mc_emergency_sp = (void *)ti + THREAD_SIZE; #endif } } @@ -762,7 +762,7 @@ void __init setup_per_cpu_areas(void) delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start; for_each_possible_cpu(cpu) { __per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu]; - paca[cpu].data_offset = __per_cpu_offset[cpu]; + paca_ptrs[cpu]->data_offset = __per_cpu_offset[cpu]; } } #endif @@ -875,8 +875,9 @@ static void init_fallback_flush(void) memset(l1d_flush_fallback_area, 0, l1d_size * 2); for_each_possible_cpu(cpu) { - paca[cpu].rfi_flush_fallback_area = l1d_flush_fallback_area; - paca[cpu].l1d_flush_size = l1d_size; + struct paca_struct *paca = paca_ptrs[cpu]; + paca->rfi_flush_fallback_area = l1d_flush_fallback_area; + paca->l1d_flush_size = l1d_size; } } -- cgit v1.2.3 From c0abd0c745bdabe027a8f013a866f385fba717b1 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Wed, 14 Feb 2018 01:08:17 +1000 Subject: powerpc/64: move default SPR recording Move this into the early setup code, and don't iterate over CPU masks. We don't want to call into sysfs so early from setup, and a future patch won't initialize CPU masks by the time this is called. Signed-off-by: Nicholas Piggin [mpe: Fold in incremental fix from Nick for DSCR handling] Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/setup.h | 9 +++------ arch/powerpc/kernel/setup_64.c | 8 ++++++++ arch/powerpc/kernel/sysfs.c | 20 +++++++++----------- 3 files changed, 20 insertions(+), 17 deletions(-) (limited to 'arch/powerpc/kernel/setup_64.c') diff --git a/arch/powerpc/kernel/setup.h b/arch/powerpc/kernel/setup.h index 3fc11e30308f..d144df54ad40 100644 --- a/arch/powerpc/kernel/setup.h +++ b/arch/powerpc/kernel/setup.h @@ -45,14 +45,11 @@ void emergency_stack_init(void); static inline void emergency_stack_init(void) { }; #endif -#ifdef CONFIG_PPC64 -void record_spr_defaults(void); -#else -static inline void record_spr_defaults(void) { }; -#endif - #ifdef CONFIG_PPC64 u64 ppc64_bolted_size(void); + +/* Default SPR values from firmware/kexec */ +extern unsigned long spr_default_dscr; #endif /* diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c index 3ce12af4906f..dde34d35d1e7 100644 --- a/arch/powerpc/kernel/setup_64.c +++ b/arch/powerpc/kernel/setup_64.c @@ -254,6 +254,14 @@ static void cpu_ready_for_interrupts(void) get_paca()->kernel_msr = MSR_KERNEL; } +unsigned long spr_default_dscr = 0; + +void __init record_spr_defaults(void) +{ + if (early_cpu_has_feature(CPU_FTR_DSCR)) + spr_default_dscr = mfspr(SPRN_DSCR); +} + /* * Early initialization entry point. This is called by head.S * with MMU translation disabled. We rely on the "feature" of diff --git a/arch/powerpc/kernel/sysfs.c b/arch/powerpc/kernel/sysfs.c index 1f9d94dac3a6..9f327483008c 100644 --- a/arch/powerpc/kernel/sysfs.c +++ b/arch/powerpc/kernel/sysfs.c @@ -20,6 +20,7 @@ #include #include "cacheinfo.h" +#include "setup.h" #ifdef CONFIG_PPC64 #include @@ -588,21 +589,18 @@ static DEVICE_ATTR(dscr_default, 0600, static void sysfs_create_dscr_default(void) { - int err = 0; - if (cpu_has_feature(CPU_FTR_DSCR)) - err = device_create_file(cpu_subsys.dev_root, &dev_attr_dscr_default); -} - -void __init record_spr_defaults(void) -{ - int cpu; - if (cpu_has_feature(CPU_FTR_DSCR)) { - dscr_default = mfspr(SPRN_DSCR); - for (cpu = 0; cpu < nr_cpu_ids; cpu++) + int err = 0; + int cpu; + + dscr_default = spr_default_dscr; + for_each_possible_cpu(cpu) paca_ptrs[cpu]->dscr_default = dscr_default; + + err = device_create_file(cpu_subsys.dev_root, &dev_attr_dscr_default); } } + #endif /* CONFIG_PPC64 */ #ifdef HAS_PPC_PMC_PA6T -- cgit v1.2.3 From 4890aea65ae7b5d424b5020e8be193b08a545990 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Wed, 14 Feb 2018 01:08:20 +1000 Subject: powerpc/64: Allocate pacas per node Per-node allocations are possible on 64s with radix that does not have the bolted SLB limitation. Hash would be able to do the same if all CPUs had the bottom of their node-local memory bolted as well. This is left as an exercise for the reader. Signed-off-by: Nicholas Piggin [mpe: Add dummy definition of boot_cpuid for !SMP] Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/paca.c | 45 ++++++++++++++++++++++++++++++++++++------ arch/powerpc/kernel/setup_64.c | 4 ++++ 2 files changed, 43 insertions(+), 6 deletions(-) (limited to 'arch/powerpc/kernel/setup_64.c') diff --git a/arch/powerpc/kernel/paca.c b/arch/powerpc/kernel/paca.c index a186911791c7..0f7e2be23fa2 100644 --- a/arch/powerpc/kernel/paca.c +++ b/arch/powerpc/kernel/paca.c @@ -20,6 +20,41 @@ #include "setup.h" +#ifndef CONFIG_SMP +#define boot_cpuid 0 +#endif + +static void *__init alloc_paca_data(unsigned long size, unsigned long align, + unsigned long limit, int cpu) +{ + unsigned long pa; + int nid; + + /* + * boot_cpuid paca is allocated very early before cpu_to_node is up. + * Set bottom-up mode, because the boot CPU should be on node-0, + * which will put its paca in the right place. + */ + if (cpu == boot_cpuid) { + nid = -1; + memblock_set_bottom_up(true); + } else { + nid = early_cpu_to_node(cpu); + } + + pa = memblock_alloc_base_nid(size, align, limit, nid, MEMBLOCK_NONE); + if (!pa) { + pa = memblock_alloc_base(size, align, limit); + if (!pa) + panic("cannot allocate paca data"); + } + + if (cpu == boot_cpuid) + memblock_set_bottom_up(false); + + return __va(pa); +} + #ifdef CONFIG_PPC_PSERIES /* @@ -52,7 +87,7 @@ static struct lppaca * __init new_lppaca(int cpu, unsigned long limit) if (early_cpu_has_feature(CPU_FTR_HVMODE)) return NULL; - lp = __va(memblock_alloc_base(size, 0x400, limit)); + lp = alloc_paca_data(size, 0x400, limit, cpu); init_lppaca(lp); return lp; @@ -82,7 +117,7 @@ static struct slb_shadow * __init new_slb_shadow(int cpu, unsigned long limit) return NULL; } - s = __va(memblock_alloc_base(sizeof(*s), L1_CACHE_BYTES, limit)); + s = alloc_paca_data(sizeof(*s), L1_CACHE_BYTES, limit, cpu); memset(s, 0, sizeof(*s)); s->persistent = cpu_to_be32(SLB_NUM_BOLTED); @@ -170,7 +205,6 @@ void __init allocate_paca_ptrs(void) void __init allocate_paca(int cpu) { u64 limit; - unsigned long pa; struct paca_struct *paca; BUG_ON(cpu >= paca_nr_cpu_ids); @@ -185,9 +219,8 @@ void __init allocate_paca(int cpu) limit = ppc64_rma_size; #endif - pa = memblock_alloc_base(sizeof(struct paca_struct), - L1_CACHE_BYTES, limit); - paca = __va(pa); + paca = alloc_paca_data(sizeof(struct paca_struct), L1_CACHE_BYTES, + limit, cpu); paca_ptrs[cpu] = paca; memset(paca, 0, sizeof(struct paca_struct)); diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c index dde34d35d1e7..02fa358982e6 100644 --- a/arch/powerpc/kernel/setup_64.c +++ b/arch/powerpc/kernel/setup_64.c @@ -312,6 +312,10 @@ void __init early_setup(unsigned long dt_ptr) early_init_devtree(__va(dt_ptr)); /* Now we know the logical id of our boot cpu, setup the paca. */ + if (boot_cpuid != 0) { + /* Poison paca_ptrs[0] again if it's not the boot cpu */ + memset(&paca_ptrs[0], 0x88, sizeof(paca_ptrs[0])); + } setup_paca(paca_ptrs[boot_cpuid]); fixup_boot_paca(); -- cgit v1.2.3 From f3865f9a7112590f0cae02dce05ec3c3a09ff405 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Wed, 14 Feb 2018 01:08:21 +1000 Subject: powerpc/64: Allocate per-cpu stacks node-local if possible Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/setup_64.c | 51 ++++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 19 deletions(-) (limited to 'arch/powerpc/kernel/setup_64.c') diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c index 02fa358982e6..16ea71fa1ead 100644 --- a/arch/powerpc/kernel/setup_64.c +++ b/arch/powerpc/kernel/setup_64.c @@ -611,6 +611,21 @@ __init u64 ppc64_bolted_size(void) #endif } +static void *__init alloc_stack(unsigned long limit, int cpu) +{ + unsigned long pa; + + pa = memblock_alloc_base_nid(THREAD_SIZE, THREAD_SIZE, limit, + early_cpu_to_node(cpu), MEMBLOCK_NONE); + if (!pa) { + pa = memblock_alloc_base(THREAD_SIZE, THREAD_SIZE, limit); + if (!pa) + panic("cannot allocate stacks"); + } + + return __va(pa); +} + void __init irqstack_early_init(void) { u64 limit = ppc64_bolted_size(); @@ -622,12 +637,8 @@ void __init irqstack_early_init(void) * accessed in realmode. */ for_each_possible_cpu(i) { - softirq_ctx[i] = (struct thread_info *) - __va(memblock_alloc_base(THREAD_SIZE, - THREAD_SIZE, limit)); - hardirq_ctx[i] = (struct thread_info *) - __va(memblock_alloc_base(THREAD_SIZE, - THREAD_SIZE, limit)); + softirq_ctx[i] = alloc_stack(limit, i); + hardirq_ctx[i] = alloc_stack(limit, i); } } @@ -635,20 +646,21 @@ void __init irqstack_early_init(void) void __init exc_lvl_early_init(void) { unsigned int i; - unsigned long sp; for_each_possible_cpu(i) { - sp = memblock_alloc(THREAD_SIZE, THREAD_SIZE); - critirq_ctx[i] = (struct thread_info *)__va(sp); - paca_ptrs[i]->crit_kstack = __va(sp + THREAD_SIZE); + void *sp; - sp = memblock_alloc(THREAD_SIZE, THREAD_SIZE); - dbgirq_ctx[i] = (struct thread_info *)__va(sp); - paca_ptrs[i]->dbg_kstack = __va(sp + THREAD_SIZE); + sp = alloc_stack(ULONG_MAX, i); + critirq_ctx[i] = sp; + paca_ptrs[i]->crit_kstack = sp + THREAD_SIZE; - sp = memblock_alloc(THREAD_SIZE, THREAD_SIZE); - mcheckirq_ctx[i] = (struct thread_info *)__va(sp); - paca_ptrs[i]->mc_kstack = __va(sp + THREAD_SIZE); + sp = alloc_stack(ULONG_MAX, i); + dbgirq_ctx[i] = sp; + paca_ptrs[i]->dbg_kstack = sp + THREAD_SIZE; + + sp = alloc_stack(ULONG_MAX, i); + mcheckirq_ctx[i] = sp; + paca_ptrs[i]->mc_kstack = sp + THREAD_SIZE; } if (cpu_has_feature(CPU_FTR_DEBUG_LVL_EXC)) @@ -702,20 +714,21 @@ void __init emergency_stack_init(void) for_each_possible_cpu(i) { struct thread_info *ti; - ti = __va(memblock_alloc_base(THREAD_SIZE, THREAD_SIZE, limit)); + + ti = alloc_stack(limit, i); memset(ti, 0, THREAD_SIZE); emerg_stack_init_thread_info(ti, i); paca_ptrs[i]->emergency_sp = (void *)ti + THREAD_SIZE; #ifdef CONFIG_PPC_BOOK3S_64 /* emergency stack for NMI exception handling. */ - ti = __va(memblock_alloc_base(THREAD_SIZE, THREAD_SIZE, limit)); + ti = alloc_stack(limit, i); memset(ti, 0, THREAD_SIZE); emerg_stack_init_thread_info(ti, i); paca_ptrs[i]->nmi_emergency_sp = (void *)ti + THREAD_SIZE; /* emergency stack for machine check exception handling. */ - ti = __va(memblock_alloc_base(THREAD_SIZE, THREAD_SIZE, limit)); + ti = alloc_stack(limit, i); memset(ti, 0, THREAD_SIZE); emerg_stack_init_thread_info(ti, i); paca_ptrs[i]->mc_emergency_sp = (void *)ti + THREAD_SIZE; -- cgit v1.2.3