From 28c807e5132ecc9f1607461eabfa1fc67b21e163 Mon Sep 17 00:00:00 2001 From: Martin Schwidefsky Date: Tue, 26 Jul 2016 16:00:22 +0200 Subject: s390/mm: add guest ASCE TLB flush optimization Signed-off-by: Martin Schwidefsky --- arch/s390/include/asm/pgtable.h | 58 +++++++++++++++++++++++++++++----------- arch/s390/include/asm/tlbflush.h | 2 ++ arch/s390/mm/pageattr.c | 2 +- arch/s390/mm/pgtable.c | 36 ++++++++++++++++--------- 4 files changed, 70 insertions(+), 28 deletions(-) (limited to 'arch/s390') diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index c92713893313..b20b0f7170e8 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -953,9 +953,11 @@ static inline pte_t pte_mkhuge(pte_t pte) #define IPTE_LOCAL 1 #define IPTE_NODAT 0x400 +#define IPTE_GUEST_ASCE 0x800 static inline void __ptep_ipte(unsigned long address, pte_t *ptep, - unsigned long opt, int local) + unsigned long opt, unsigned long asce, + int local) { unsigned long pto = (unsigned long) ptep; @@ -969,6 +971,7 @@ static inline void __ptep_ipte(unsigned long address, pte_t *ptep, } /* Invalidate ptes with options + TLB flush of the ptes */ + opt = opt | (asce & _ASCE_ORIGIN); asm volatile( " .insn rrf,0xb2210000,%[r1],%[r2],%[r3],%[m4]" : [r2] "+a" (address), [r3] "+a" (opt) @@ -1355,34 +1358,59 @@ static inline void __pmdp_csp(pmd_t *pmdp) #define IDTE_PTOA 0x0800 #define IDTE_NODAT 0x1000 +#define IDTE_GUEST_ASCE 0x2000 static inline void __pmdp_idte(unsigned long addr, pmd_t *pmdp, - unsigned long opt, int local) + unsigned long opt, unsigned long asce, + int local) { unsigned long sto; sto = (unsigned long) pmdp - pmd_index(addr) * sizeof(pmd_t); - asm volatile( - " .insn rrf,0xb98e0000,%[r1],%[r2],0,%[m4]" - : "+m" (*pmdp) - : [r1] "a" (sto), [r2] "a" ((addr & HPAGE_MASK) | opt), - [m4] "i" (local) - : "cc" ); + if (__builtin_constant_p(opt) && opt == 0) { + /* flush without guest asce */ + asm volatile( + " .insn rrf,0xb98e0000,%[r1],%[r2],0,%[m4]" + : "+m" (*pmdp) + : [r1] "a" (sto), [r2] "a" ((addr & HPAGE_MASK)), + [m4] "i" (local) + : "cc" ); + } else { + /* flush with guest asce */ + asm volatile( + " .insn rrf,0xb98e0000,%[r1],%[r2],%[r3],%[m4]" + : "+m" (*pmdp) + : [r1] "a" (sto), [r2] "a" ((addr & HPAGE_MASK) | opt), + [r3] "a" (asce), [m4] "i" (local) + : "cc" ); + } } static inline void __pudp_idte(unsigned long addr, pud_t *pudp, - unsigned long opt, int local) + unsigned long opt, unsigned long asce, + int local) { unsigned long r3o; r3o = (unsigned long) pudp - pud_index(addr) * sizeof(pud_t); r3o |= _ASCE_TYPE_REGION3; - asm volatile( - " .insn rrf,0xb98e0000,%[r1],%[r2],0,%[m4]" - : "+m" (*pudp) - : [r1] "a" (r3o), [r2] "a" ((addr & PUD_MASK) | opt), - [m4] "i" (local) - : "cc" ); + if (__builtin_constant_p(opt) && opt == 0) { + /* flush without guest asce */ + asm volatile( + " .insn rrf,0xb98e0000,%[r1],%[r2],0,%[m4]" + : "+m" (*pudp) + : [r1] "a" (r3o), [r2] "a" ((addr & PUD_MASK)), + [m4] "i" (local) + : "cc"); + } else { + /* flush with guest asce */ + asm volatile( + " .insn rrf,0xb98e0000,%[r1],%[r2],%[r3],%[m4]" + : "+m" (*pudp) + : [r1] "a" (r3o), [r2] "a" ((addr & PUD_MASK) | opt), + [r3] "a" (asce), [m4] "i" (local) + : "cc" ); + } } pmd_t pmdp_xchg_direct(struct mm_struct *, unsigned long, pmd_t *, pmd_t); diff --git a/arch/s390/include/asm/tlbflush.h b/arch/s390/include/asm/tlbflush.h index 38d82ed60345..4d759f8f4bc7 100644 --- a/arch/s390/include/asm/tlbflush.h +++ b/arch/s390/include/asm/tlbflush.h @@ -23,6 +23,8 @@ static inline void __tlb_flush_idte(unsigned long asce) unsigned long opt; opt = IDTE_PTOA; + if (MACHINE_HAS_TLB_GUEST) + opt |= IDTE_GUEST_ASCE; /* Global TLB flush for the mm */ asm volatile( " .insn rrf,0xb98e0000,0,%0,%1,0" diff --git a/arch/s390/mm/pageattr.c b/arch/s390/mm/pageattr.c index 5734b01ca765..567fe92e2bb8 100644 --- a/arch/s390/mm/pageattr.c +++ b/arch/s390/mm/pageattr.c @@ -328,7 +328,7 @@ static void ipte_range(pte_t *pte, unsigned long address, int nr) return; } for (i = 0; i < nr; i++) { - __ptep_ipte(address, pte, 0, IPTE_GLOBAL); + __ptep_ipte(address, pte, 0, 0, IPTE_GLOBAL); address += PAGE_SIZE; pte++; } diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c index 9696bf89f03a..3f1abc7b5fd2 100644 --- a/arch/s390/mm/pgtable.c +++ b/arch/s390/mm/pgtable.c @@ -35,9 +35,13 @@ static inline void ptep_ipte_local(struct mm_struct *mm, unsigned long addr, asce = READ_ONCE(mm->context.gmap_asce); if (asce == 0UL) opt |= IPTE_NODAT; - __ptep_ipte(addr, ptep, opt, IPTE_LOCAL); + if (asce != -1UL) { + asce = asce ? : mm->context.asce; + opt |= IPTE_GUEST_ASCE; + } + __ptep_ipte(addr, ptep, opt, asce, IPTE_LOCAL); } else { - __ptep_ipte(addr, ptep, 0, IPTE_LOCAL); + __ptep_ipte(addr, ptep, 0, 0, IPTE_LOCAL); } } @@ -51,9 +55,13 @@ static inline void ptep_ipte_global(struct mm_struct *mm, unsigned long addr, asce = READ_ONCE(mm->context.gmap_asce); if (asce == 0UL) opt |= IPTE_NODAT; - __ptep_ipte(addr, ptep, opt, IPTE_GLOBAL); + if (asce != -1UL) { + asce = asce ? : mm->context.asce; + opt |= IPTE_GUEST_ASCE; + } + __ptep_ipte(addr, ptep, opt, asce, IPTE_GLOBAL); } else { - __ptep_ipte(addr, ptep, 0, IPTE_GLOBAL); + __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL); } } @@ -326,18 +334,20 @@ static inline void pmdp_idte_local(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp) { if (MACHINE_HAS_TLB_GUEST) - __pmdp_idte(addr, pmdp, IDTE_NODAT, IDTE_LOCAL); + __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE, + mm->context.asce, IDTE_LOCAL); else - __pmdp_idte(addr, pmdp, 0, IDTE_LOCAL); + __pmdp_idte(addr, pmdp, 0, 0, IDTE_LOCAL); } static inline void pmdp_idte_global(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp) { if (MACHINE_HAS_TLB_GUEST) - __pmdp_idte(addr, pmdp, IDTE_NODAT, IDTE_GLOBAL); + __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE, + mm->context.asce, IDTE_GLOBAL); else if (MACHINE_HAS_IDTE) - __pmdp_idte(addr, pmdp, 0, IDTE_GLOBAL); + __pmdp_idte(addr, pmdp, 0, 0, IDTE_GLOBAL); else __pmdp_csp(pmdp); } @@ -410,18 +420,20 @@ static inline void pudp_idte_local(struct mm_struct *mm, unsigned long addr, pud_t *pudp) { if (MACHINE_HAS_TLB_GUEST) - __pudp_idte(addr, pudp, IDTE_NODAT, IDTE_LOCAL); + __pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE, + mm->context.asce, IDTE_LOCAL); else - __pudp_idte(addr, pudp, 0, IDTE_LOCAL); + __pudp_idte(addr, pudp, 0, 0, IDTE_LOCAL); } static inline void pudp_idte_global(struct mm_struct *mm, unsigned long addr, pud_t *pudp) { if (MACHINE_HAS_TLB_GUEST) - __pudp_idte(addr, pudp, IDTE_NODAT, IDTE_GLOBAL); + __pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE, + mm->context.asce, IDTE_GLOBAL); else if (MACHINE_HAS_IDTE) - __pudp_idte(addr, pudp, 0, IDTE_GLOBAL); + __pudp_idte(addr, pudp, 0, 0, IDTE_GLOBAL); else /* * Invalid bit position is the same for pmd and pud, so we can -- cgit v1.2.3