From dcc1e8dd88d4bc55e32a26dad7633d20ffe606d2 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Wed, 22 Mar 2006 00:49:59 -0800 Subject: [SPARC64]: Add a secondary TSB for hugepage mappings. Signed-off-by: David S. Miller --- arch/sparc64/kernel/tsb.S | 210 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 160 insertions(+), 50 deletions(-) (limited to 'arch/sparc64/kernel/tsb.S') diff --git a/arch/sparc64/kernel/tsb.S b/arch/sparc64/kernel/tsb.S index 118baea44f69..a0c8ba58920b 100644 --- a/arch/sparc64/kernel/tsb.S +++ b/arch/sparc64/kernel/tsb.S @@ -3,8 +3,13 @@ * Copyright (C) 2006 David S. Miller */ +#include + #include #include +#include +#include +#include .text .align 32 @@ -34,34 +39,124 @@ tsb_miss_itlb: ldxa [%g4] ASI_IMMU, %g4 /* At this point we have: - * %g1 -- TSB entry address + * %g1 -- PAGE_SIZE TSB entry address * %g3 -- FAULT_CODE_{D,I}TLB * %g4 -- missing virtual address * %g6 -- TAG TARGET (vaddr >> 22) */ tsb_miss_page_table_walk: - TRAP_LOAD_PGD_PHYS(%g7, %g5) + TRAP_LOAD_TRAP_BLOCK(%g7, %g5) - /* And now we have the PGD base physical address in %g7. */ -tsb_miss_page_table_walk_sun4v_fastpath: - USER_PGTABLE_WALK_TL1(%g4, %g7, %g5, %g2, tsb_do_fault) + /* Before committing to a full page table walk, + * check the huge page TSB. + */ +#ifdef CONFIG_HUGETLB_PAGE + +661: ldx [%g7 + TRAP_PER_CPU_TSB_HUGE], %g5 + nop + .section .sun4v_2insn_patch, "ax" + .word 661b + mov SCRATCHPAD_UTSBREG2, %g5 + ldxa [%g5] ASI_SCRATCHPAD, %g5 + .previous + + cmp %g5, -1 + be,pt %xcc, 80f + nop + + /* We need an aligned pair of registers containing 2 values + * which can be easily rematerialized. %g6 and %g7 foot the + * bill just nicely. We'll save %g6 away into %g2 for the + * huge page TSB TAG comparison. + * + * Perform a huge page TSB lookup. + */ + mov %g6, %g2 + and %g5, 0x7, %g6 + mov 512, %g7 + andn %g5, 0x7, %g5 + sllx %g7, %g6, %g7 + srlx %g4, HPAGE_SHIFT, %g6 + sub %g7, 1, %g7 + and %g6, %g7, %g6 + sllx %g6, 4, %g6 + add %g5, %g6, %g5 + + TSB_LOAD_QUAD(%g5, %g6) + cmp %g6, %g2 + be,a,pt %xcc, tsb_tlb_reload + mov %g7, %g5 + + /* No match, remember the huge page TSB entry address, + * and restore %g6 and %g7. + */ + TRAP_LOAD_TRAP_BLOCK(%g7, %g6) + srlx %g4, 22, %g6 +80: stx %g5, [%g7 + TRAP_PER_CPU_TSB_HUGE_TEMP] + +#endif + + ldx [%g7 + TRAP_PER_CPU_PGD_PADDR], %g7 /* At this point we have: * %g1 -- TSB entry address * %g3 -- FAULT_CODE_{D,I}TLB - * %g5 -- physical address of PTE in Linux page tables + * %g4 -- missing virtual address * %g6 -- TAG TARGET (vaddr >> 22) + * %g7 -- page table physical address + * + * We know that both the base PAGE_SIZE TSB and the HPAGE_SIZE + * TSB both lack a matching entry. */ -tsb_reload: - TSB_LOCK_TAG(%g1, %g2, %g7) +tsb_miss_page_table_walk_sun4v_fastpath: + USER_PGTABLE_WALK_TL1(%g4, %g7, %g5, %g2, tsb_do_fault) /* Load and check PTE. */ ldxa [%g5] ASI_PHYS_USE_EC, %g5 - mov 1, %g7 - sllx %g7, TSB_TAG_INVALID_BIT, %g7 - brgez,a,pn %g5, tsb_do_fault - TSB_STORE(%g1, %g7) + brgez,pn %g5, tsb_do_fault + nop + +#ifdef CONFIG_HUGETLB_PAGE +661: sethi %uhi(_PAGE_SZALL_4U), %g7 + sllx %g7, 32, %g7 + .section .sun4v_2insn_patch, "ax" + .word 661b + mov _PAGE_SZALL_4V, %g7 + nop + .previous + + and %g5, %g7, %g2 + +661: sethi %uhi(_PAGE_SZHUGE_4U), %g7 + sllx %g7, 32, %g7 + .section .sun4v_2insn_patch, "ax" + .word 661b + mov _PAGE_SZHUGE_4V, %g7 + nop + .previous + + cmp %g2, %g7 + bne,pt %xcc, 60f + nop + + /* It is a huge page, use huge page TSB entry address we + * calculated above. + */ + TRAP_LOAD_TRAP_BLOCK(%g7, %g2) + ldx [%g7 + TRAP_PER_CPU_TSB_HUGE_TEMP], %g2 + cmp %g2, -1 + movne %xcc, %g2, %g1 +60: +#endif + /* At this point we have: + * %g1 -- TSB entry address + * %g3 -- FAULT_CODE_{D,I}TLB + * %g5 -- valid PTE + * %g6 -- TAG TARGET (vaddr >> 22) + */ +tsb_reload: + TSB_LOCK_TAG(%g1, %g2, %g7) TSB_WRITE(%g1, %g5, %g6) /* Finally, load TLB and return from trap. */ @@ -240,10 +335,9 @@ tsb_flush: * schedule() time. * * %o0: page table physical address - * %o1: TSB register value - * %o2: TSB virtual address - * %o3: TSB mapping locked PTE - * %o4: Hypervisor TSB descriptor physical address + * %o1: TSB base config pointer + * %o2: TSB huge config pointer, or NULL if none + * %o3: Hypervisor TSB descriptor physical address * * We have to run this whole thing with interrupts * disabled so that the current cpu doesn't change @@ -253,63 +347,79 @@ tsb_flush: .globl __tsb_context_switch .type __tsb_context_switch,#function __tsb_context_switch: - rdpr %pstate, %o5 - wrpr %o5, PSTATE_IE, %pstate + rdpr %pstate, %g1 + wrpr %g1, PSTATE_IE, %pstate + + TRAP_LOAD_TRAP_BLOCK(%g2, %g3) - ldub [%g6 + TI_CPU], %g1 - sethi %hi(trap_block), %g2 - sllx %g1, TRAP_BLOCK_SZ_SHIFT, %g1 - or %g2, %lo(trap_block), %g2 - add %g2, %g1, %g2 stx %o0, [%g2 + TRAP_PER_CPU_PGD_PADDR] - sethi %hi(tlb_type), %g1 - lduw [%g1 + %lo(tlb_type)], %g1 - cmp %g1, 3 - bne,pt %icc, 1f + ldx [%o1 + TSB_CONFIG_REG_VAL], %o0 + brz,pt %o2, 1f + mov -1, %g3 + + ldx [%o2 + TSB_CONFIG_REG_VAL], %g3 + +1: stx %g3, [%g2 + TRAP_PER_CPU_TSB_HUGE] + + sethi %hi(tlb_type), %g2 + lduw [%g2 + %lo(tlb_type)], %g2 + cmp %g2, 3 + bne,pt %icc, 50f nop /* Hypervisor TSB switch. */ - mov SCRATCHPAD_UTSBREG1, %g1 - stxa %o1, [%g1] ASI_SCRATCHPAD - mov -1, %g2 - mov SCRATCHPAD_UTSBREG2, %g1 - stxa %g2, [%g1] ASI_SCRATCHPAD - - /* Save away %o5's %pstate, we have to use %o5 for - * the hypervisor call. - */ - mov %o5, %g1 + mov SCRATCHPAD_UTSBREG1, %o5 + stxa %o0, [%o5] ASI_SCRATCHPAD + mov SCRATCHPAD_UTSBREG2, %o5 + stxa %g3, [%o5] ASI_SCRATCHPAD + + mov 2, %o0 + cmp %g3, -1 + move %xcc, 1, %o0 mov HV_FAST_MMU_TSB_CTXNON0, %o5 - mov 1, %o0 - mov %o4, %o1 + mov %o3, %o1 ta HV_FAST_TRAP - /* Finish up and restore %o5. */ + /* Finish up. */ ba,pt %xcc, 9f - mov %g1, %o5 + nop /* SUN4U TSB switch. */ -1: mov TSB_REG, %g1 - stxa %o1, [%g1] ASI_DMMU +50: mov TSB_REG, %o5 + stxa %o0, [%o5] ASI_DMMU membar #Sync - stxa %o1, [%g1] ASI_IMMU + stxa %o0, [%o5] ASI_IMMU membar #Sync -2: brz %o2, 9f - nop +2: ldx [%o1 + TSB_CONFIG_MAP_VADDR], %o4 + brz %o4, 9f + ldx [%o1 + TSB_CONFIG_MAP_PTE], %o5 sethi %hi(sparc64_highest_unlocked_tlb_ent), %g2 - mov TLB_TAG_ACCESS, %g1 + mov TLB_TAG_ACCESS, %g3 lduw [%g2 + %lo(sparc64_highest_unlocked_tlb_ent)], %g2 - stxa %o2, [%g1] ASI_DMMU + stxa %o4, [%g3] ASI_DMMU membar #Sync sllx %g2, 3, %g2 - stxa %o3, [%g2] ASI_DTLB_DATA_ACCESS + stxa %o5, [%g2] ASI_DTLB_DATA_ACCESS + membar #Sync + + brz,pt %o2, 9f + nop + + ldx [%o2 + TSB_CONFIG_MAP_VADDR], %o4 + ldx [%o2 + TSB_CONFIG_MAP_PTE], %o5 + mov TLB_TAG_ACCESS, %g3 + stxa %o4, [%g3] ASI_DMMU + membar #Sync + sub %g2, (1 << 3), %g2 + stxa %o5, [%g2] ASI_DTLB_DATA_ACCESS membar #Sync + 9: - wrpr %o5, %pstate + wrpr %g1, %pstate retl nop -- cgit v1.2.3