summaryrefslogtreecommitdiffstats
path: root/arch/loongarch/mm
diff options
context:
space:
mode:
authorHuacai Chen <chenhuacai@loongson.cn>2022-05-31 12:04:12 +0200
committerHuacai Chen <chenhuacai@loongson.cn>2022-06-03 14:09:29 +0200
commit46859ac8af52ae599e1b51992ddef3eb43f295fc (patch)
treec1e6640316d5d8748bea046b71d2260b81cbe314 /arch/loongarch/mm
parentLoongArch: Add VDSO and VSYSCALL support (diff)
downloadlinux-46859ac8af52ae599e1b51992ddef3eb43f295fc.tar.xz
linux-46859ac8af52ae599e1b51992ddef3eb43f295fc.zip
LoongArch: Add multi-processor (SMP) support
LoongArch-based procesors have 4, 8 or 16 cores per package. This patch adds multi-processor (SMP) support for LoongArch. Reviewed-by: WANG Xuerui <git@xen0n.name> Reviewed-by: Jiaxun Yang <jiaxun.yang@flygoat.com> Signed-off-by: Huacai Chen <chenhuacai@loongson.cn>
Diffstat (limited to 'arch/loongarch/mm')
-rw-r--r--arch/loongarch/mm/tlbex.S69
1 files changed, 69 insertions, 0 deletions
diff --git a/arch/loongarch/mm/tlbex.S b/arch/loongarch/mm/tlbex.S
index bef740710a3b..7eee40271577 100644
--- a/arch/loongarch/mm/tlbex.S
+++ b/arch/loongarch/mm/tlbex.S
@@ -88,7 +88,14 @@ vmalloc_done_load:
slli.d t0, t0, _PTE_T_LOG2
add.d t1, ra, t0
+#ifdef CONFIG_SMP
+smp_pgtable_change_load:
+#endif
+#ifdef CONFIG_SMP
+ ll.d t0, t1, 0
+#else
ld.d t0, t1, 0
+#endif
tlbsrch
srli.d ra, t0, _PAGE_PRESENT_SHIFT
@@ -96,7 +103,12 @@ vmalloc_done_load:
beq ra, $r0, nopage_tlb_load
ori t0, t0, _PAGE_VALID
+#ifdef CONFIG_SMP
+ sc.d t0, t1, 0
+ beq t0, $r0, smp_pgtable_change_load
+#else
st.d t0, t1, 0
+#endif
ori t1, t1, 8
xori t1, t1, 8
ld.d t0, t1, 0
@@ -120,14 +132,24 @@ vmalloc_load:
* spots a huge page.
*/
tlb_huge_update_load:
+#ifdef CONFIG_SMP
+ ll.d t0, t1, 0
+#else
ld.d t0, t1, 0
+#endif
srli.d ra, t0, _PAGE_PRESENT_SHIFT
andi ra, ra, 1
beq ra, $r0, nopage_tlb_load
tlbsrch
ori t0, t0, _PAGE_VALID
+#ifdef CONFIG_SMP
+ sc.d t0, t1, 0
+ beq t0, $r0, tlb_huge_update_load
+ ld.d t0, t1, 0
+#else
st.d t0, t1, 0
+#endif
addu16i.d t1, $r0, -(CSR_TLBIDX_EHINV >> 16)
addi.d ra, t1, 0
csrxchg ra, t1, LOONGARCH_CSR_TLBIDX
@@ -173,6 +195,7 @@ tlb_huge_update_load:
csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
nopage_tlb_load:
+ dbar 0
csrrd ra, EXCEPTION_KS2
la.abs t0, tlb_do_page_fault_0
jirl $r0, t0, 0
@@ -229,7 +252,14 @@ vmalloc_done_store:
slli.d t0, t0, _PTE_T_LOG2
add.d t1, ra, t0
+#ifdef CONFIG_SMP
+smp_pgtable_change_store:
+#endif
+#ifdef CONFIG_SMP
+ ll.d t0, t1, 0
+#else
ld.d t0, t1, 0
+#endif
tlbsrch
srli.d ra, t0, _PAGE_PRESENT_SHIFT
@@ -238,7 +268,12 @@ vmalloc_done_store:
bne ra, $r0, nopage_tlb_store
ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
+#ifdef CONFIG_SMP
+ sc.d t0, t1, 0
+ beq t0, $r0, smp_pgtable_change_store
+#else
st.d t0, t1, 0
+#endif
ori t1, t1, 8
xori t1, t1, 8
@@ -263,7 +298,11 @@ vmalloc_store:
* spots a huge page.
*/
tlb_huge_update_store:
+#ifdef CONFIG_SMP
+ ll.d t0, t1, 0
+#else
ld.d t0, t1, 0
+#endif
srli.d ra, t0, _PAGE_PRESENT_SHIFT
andi ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT)
xori ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT)
@@ -272,7 +311,13 @@ tlb_huge_update_store:
tlbsrch
ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
+#ifdef CONFIG_SMP
+ sc.d t0, t1, 0
+ beq t0, $r0, tlb_huge_update_store
+ ld.d t0, t1, 0
+#else
st.d t0, t1, 0
+#endif
addu16i.d t1, $r0, -(CSR_TLBIDX_EHINV >> 16)
addi.d ra, t1, 0
csrxchg ra, t1, LOONGARCH_CSR_TLBIDX
@@ -318,6 +363,7 @@ tlb_huge_update_store:
csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
nopage_tlb_store:
+ dbar 0
csrrd ra, EXCEPTION_KS2
la.abs t0, tlb_do_page_fault_1
jirl $r0, t0, 0
@@ -373,7 +419,14 @@ vmalloc_done_modify:
slli.d t0, t0, _PTE_T_LOG2
add.d t1, ra, t0
+#ifdef CONFIG_SMP
+smp_pgtable_change_modify:
+#endif
+#ifdef CONFIG_SMP
+ ll.d t0, t1, 0
+#else
ld.d t0, t1, 0
+#endif
tlbsrch
srli.d ra, t0, _PAGE_WRITE_SHIFT
@@ -381,7 +434,12 @@ vmalloc_done_modify:
beq ra, $r0, nopage_tlb_modify
ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
+#ifdef CONFIG_SMP
+ sc.d t0, t1, 0
+ beq t0, $r0, smp_pgtable_change_modify
+#else
st.d t0, t1, 0
+#endif
ori t1, t1, 8
xori t1, t1, 8
ld.d t0, t1, 0
@@ -405,7 +463,11 @@ vmalloc_modify:
* build_tlbchange_handler_head spots a huge page.
*/
tlb_huge_update_modify:
+#ifdef CONFIG_SMP
+ ll.d t0, t1, 0
+#else
ld.d t0, t1, 0
+#endif
srli.d ra, t0, _PAGE_WRITE_SHIFT
andi ra, ra, 1
@@ -414,7 +476,13 @@ tlb_huge_update_modify:
tlbsrch
ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
+#ifdef CONFIG_SMP
+ sc.d t0, t1, 0
+ beq t0, $r0, tlb_huge_update_modify
+ ld.d t0, t1, 0
+#else
st.d t0, t1, 0
+#endif
/*
* A huge PTE describes an area the size of the
* configured huge page size. This is twice the
@@ -454,6 +522,7 @@ tlb_huge_update_modify:
csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
nopage_tlb_modify:
+ dbar 0
csrrd ra, EXCEPTION_KS2
la.abs t0, tlb_do_page_fault_1
jirl $r0, t0, 0