[SPARC64]: Move away from virtual page tables, part 1.

We now use the TSB hardware assist features of the UltraSPARC
MMUs.

SMP is currently knowingly broken, we need to find another place
to store the per-cpu base pointers.  We hid them away in the TSB
base register, and that obviously will not work any more :-)

Another known broken case is non-8KB base page size.

Also noticed that flush_tlb_all() is not referenced anywhere, only
the internal __flush_tlb_all() (local cpu only) is used by the
sparc64 port, so we can get rid of flush_tlb_all().

The kernel gets it's own 8KB TSB (swapper_tsb) and each address space
gets it's own private 8K TSB.  Later we can add code to dynamically
increase the size of per-process TSB as the RSS grows.  An 8KB TSB is
good enough for up to about a 4MB RSS, after which the TSB starts to
incur many capacity and conflict misses.

We even accumulate OBP translations into the kernel TSB.

Another area for refinement is large page size support.  We could use
a secondary address space TSB to handle those.

Signed-off-by: David S. Miller <davem@davemloft.net>

1 files changed, 121 insertions, 142 deletions

diff --git a/arch/sparc64/kernel/ktlb.S b/arch/sparc64/kernel/ktlb.S
index d9244d3c9f73..2b5e71b68882 100644
--- a/arch/sparc64/kernel/ktlb.S
+++ b/arch/sparc64/kernel/ktlb.S
@@ -4,191 +4,170 @@
  * Copyright (C) 1996 Eddie C. Dost        (ecd@brainaid.de)
  * Copyright (C) 1996 Miguel de Icaza      (miguel@nuclecu.unam.mx)
  * Copyright (C) 1996,98,99 Jakub Jelinek  (jj@sunsite.mff.cuni.cz)
-*/
+ */
 
 #include <linux/config.h>
 #include <asm/head.h>
 #include <asm/asi.h>
 #include <asm/page.h>
 #include <asm/pgtable.h>
+#include <asm/tsb.h>
 
         .text
         .align          32
 
-/*
- * On a second level vpte miss, check whether the original fault is to the OBP 
- * range (note that this is only possible for instruction miss, data misses to
- * obp range do not use vpte). If so, go back directly to the faulting address.
- * This is because we want to read the tpc, otherwise we have no way of knowing
- * the 8k aligned faulting address if we are using >8k kernel pagesize. This
- * also ensures no vpte range addresses are dropped into tlb while obp is
- * executing (see inherit_locked_prom_mappings() rant).
- */
-sparc64_vpte_nucleus:
-        /* Note that kvmap below has verified that the address is
-         * in the range MODULES_VADDR --> VMALLOC_END already.  So
-         * here we need only check if it is an OBP address or not.
-         */
+        .globl          kvmap_itlb
+kvmap_itlb:
+        /* g6: TAG TARGET */
+        mov             TLB_TAG_ACCESS, %g4
+        ldxa            [%g4] ASI_IMMU, %g4
+
+kvmap_itlb_nonlinear:
+        /* Catch kernel NULL pointer calls.  */
+        sethi           %hi(PAGE_SIZE), %g5
+        cmp             %g4, %g5
+        bleu,pn         %xcc, kvmap_dtlb_longpath
+         nop
+
+        KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load)
+
+kvmap_itlb_tsb_miss:
         sethi           %hi(LOW_OBP_ADDRESS), %g5
         cmp             %g4, %g5
-        blu,pn          %xcc, kern_vpte
+        blu,pn          %xcc, kvmap_itlb_vmalloc_addr
          mov            0x1, %g5
         sllx            %g5, 32, %g5
         cmp             %g4, %g5
-        blu,pn          %xcc, vpte_insn_obp
+        blu,pn          %xcc, kvmap_itlb_obp
          nop
 
-        /* These two instructions are patched by paginig_init().  */
-kern_vpte:
-        sethi           %hi(swapper_pgd_zero), %g5
-        lduw            [%g5 + %lo(swapper_pgd_zero)], %g5
-
-        /* With kernel PGD in %g5, branch back into dtlb_backend.  */
-        ba,pt           %xcc, sparc64_kpte_continue
-         andn           %g1, 0x3, %g1   /* Finish PMD offset adjustment.  */
-
-vpte_noent:
-        /* Restore previous TAG_ACCESS, %g5 is zero, and we will
-         * skip over the trap instruction so that the top level
-         * TLB miss handler will thing this %g5 value is just an
-         * invalid PTE, thus branching to full fault processing.
-         */
-        mov             TLB_SFSR, %g1
-        stxa            %g4, [%g1 + %g1] ASI_DMMU
-        done
-
-vpte_insn_obp:
-        /* Behave as if we are at TL0.  */
-        wrpr            %g0, 1, %tl
-        rdpr            %tpc, %g4       /* Find original faulting iaddr */
-        srlx            %g4, 13, %g4    /* Throw out context bits */
-        sllx            %g4, 13, %g4    /* g4 has vpn + ctx0 now */
-
-        /* Restore previous TAG_ACCESS.  */
-        mov             TLB_SFSR, %g1
-        stxa            %g4, [%g1 + %g1] ASI_IMMU
-
-        sethi           %hi(prom_trans), %g5
-        or              %g5, %lo(prom_trans), %g5
-
-1:      ldx             [%g5 + 0x00], %g6       ! base
-        brz,a,pn        %g6, longpath           ! no more entries, fail
-         mov            TLB_SFSR, %g1           ! and restore %g1
-        ldx             [%g5 + 0x08], %g1       ! len
-        add             %g6, %g1, %g1           ! end
-        cmp             %g6, %g4
-        bgu,pt          %xcc, 2f
-         cmp            %g4, %g1
-        bgeu,pt         %xcc, 2f
-         ldx            [%g5 + 0x10], %g1       ! PTE
-
-        /* TLB load, restore %g1, and return from trap.  */
-        sub             %g4, %g6, %g6
-        add             %g1, %g6, %g5
-        mov             TLB_SFSR, %g1
-        stxa            %g5, [%g0] ASI_ITLB_DATA_IN
-        retry
+kvmap_itlb_vmalloc_addr:
+        KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)
+
+        TSB_LOCK_TAG(%g1, %g2, %g4)
+
+        /* Load and check PTE.  */
+        ldxa            [%g5] ASI_PHYS_USE_EC, %g5
+        brgez,a,pn      %g5, kvmap_itlb_longpath
+         stx            %g0, [%g1]
 
-2:      ba,pt           %xcc, 1b
-         add            %g5, (3 * 8), %g5       ! next entry
-
-kvmap_do_obp:
-        sethi           %hi(prom_trans), %g5
-        or              %g5, %lo(prom_trans), %g5
-        srlx            %g4, 13, %g4
-        sllx            %g4, 13, %g4
-
-1:      ldx             [%g5 + 0x00], %g6       ! base
-        brz,a,pn        %g6, longpath           ! no more entries, fail
-         mov            TLB_SFSR, %g1           ! and restore %g1
-        ldx             [%g5 + 0x08], %g1       ! len
-        add             %g6, %g1, %g1           ! end
-        cmp             %g6, %g4
-        bgu,pt          %xcc, 2f
-         cmp            %g4, %g1
-        bgeu,pt         %xcc, 2f
-         ldx            [%g5 + 0x10], %g1       ! PTE
-
-        /* TLB load, restore %g1, and return from trap.  */
-        sub             %g4, %g6, %g6
-        add             %g1, %g6, %g5
-        mov             TLB_SFSR, %g1
-        stxa            %g5, [%g0] ASI_DTLB_DATA_IN
+        TSB_WRITE(%g1, %g5, %g6)
+
+        /* fallthrough to TLB load */
+
+kvmap_itlb_load:
+        stxa            %g5, [%g0] ASI_ITLB_DATA_IN     ! Reload TLB
         retry
 
-2:      ba,pt           %xcc, 1b
-         add            %g5, (3 * 8), %g5       ! next entry
+kvmap_itlb_longpath:
+        rdpr    %pstate, %g5
+        wrpr    %g5, PSTATE_AG | PSTATE_MG, %pstate
+        rdpr    %tpc, %g5
+        ba,pt   %xcc, sparc64_realfault_common
+         mov    FAULT_CODE_ITLB, %g4
+
+kvmap_itlb_obp:
+        OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath)
+
+        TSB_LOCK_TAG(%g1, %g2, %g4)
+
+        TSB_WRITE(%g1, %g5, %g6)
+
+        ba,pt           %xcc, kvmap_itlb_load
+         nop
+
+kvmap_dtlb_obp:
+        OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath)
+
+        TSB_LOCK_TAG(%g1, %g2, %g4)
+
+        TSB_WRITE(%g1, %g5, %g6)
+
+        ba,pt           %xcc, kvmap_dtlb_load
+         nop
 
-/*
- * On a first level data miss, check whether this is to the OBP range (note
- * that such accesses can be made by prom, as well as by kernel using
- * prom_getproperty on "address"), and if so, do not use vpte access ...
- * rather, use information saved during inherit_prom_mappings() using 8k
- * pagesize.
- */
         .align          32
-kvmap:
-        brgez,pn        %g4, kvmap_nonlinear
+        .globl          kvmap_dtlb
+kvmap_dtlb:
+        /* %g6: TAG TARGET */
+        mov             TLB_TAG_ACCESS, %g4
+        ldxa            [%g4] ASI_DMMU, %g4
+        brgez,pn        %g4, kvmap_dtlb_nonlinear
          nop
 
-#ifdef CONFIG_DEBUG_PAGEALLOC
+#define KERN_HIGHBITS   ((_PAGE_VALID|_PAGE_SZ4MB)^0xfffff80000000000)
+#define KERN_LOWBITS    (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_W)
+
+        sethi           %uhi(KERN_HIGHBITS), %g2
+        or              %g2, %ulo(KERN_HIGHBITS), %g2
+        sllx            %g2, 32, %g2
+        or              %g2, KERN_LOWBITS, %g2
+
+#undef KERN_HIGHBITS
+#undef KERN_LOWBITS
+
         .globl          kvmap_linear_patch
 kvmap_linear_patch:
-#endif
-        ba,pt           %xcc, kvmap_load
+        ba,pt           %xcc, kvmap_dtlb_load
          xor            %g2, %g4, %g5
 
-#ifdef CONFIG_DEBUG_PAGEALLOC
-        sethi           %hi(swapper_pg_dir), %g5
-        or              %g5, %lo(swapper_pg_dir), %g5
-        sllx            %g4, 64 - (PGDIR_SHIFT + PGDIR_BITS), %g6
-        srlx            %g6, 64 - PAGE_SHIFT, %g6
-        andn            %g6, 0x3, %g6
-        lduw            [%g5 + %g6], %g5
-        brz,pn          %g5, longpath
-         sllx           %g4, 64 - (PMD_SHIFT + PMD_BITS), %g6
-        srlx            %g6, 64 - PAGE_SHIFT, %g6
-        sllx            %g5, 11, %g5
-        andn            %g6, 0x3, %g6
-        lduwa           [%g5 + %g6] ASI_PHYS_USE_EC, %g5
-        brz,pn          %g5, longpath
-         sllx           %g4, 64 - PMD_SHIFT, %g6
-        srlx            %g6, 64 - PAGE_SHIFT, %g6
-        sllx            %g5, 11, %g5
-        andn            %g6, 0x7, %g6
-        ldxa            [%g5 + %g6] ASI_PHYS_USE_EC, %g5
-        brz,pn          %g5, longpath
+kvmap_dtlb_vmalloc_addr:
+        KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
+
+        TSB_LOCK_TAG(%g1, %g2, %g4)
+
+        /* Load and check PTE.  */
+        ldxa            [%g5] ASI_PHYS_USE_EC, %g5
+        brgez,a,pn      %g5, kvmap_dtlb_longpath
+         stx            %g0, [%g1]
+
+        TSB_WRITE(%g1, %g5, %g6)
+
+        /* fallthrough to TLB load */
+
+kvmap_dtlb_load:
+        stxa            %g5, [%g0] ASI_DTLB_DATA_IN     ! Reload TLB
+        retry
+
+kvmap_dtlb_nonlinear:
+        /* Catch kernel NULL pointer derefs.  */
+        sethi           %hi(PAGE_SIZE), %g5
+        cmp             %g4, %g5
+        bleu,pn         %xcc, kvmap_dtlb_longpath
          nop
-        ba,a,pt         %xcc, kvmap_load
-#endif
 
-kvmap_nonlinear:
+        KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
+
+kvmap_dtlb_tsbmiss:
         sethi           %hi(MODULES_VADDR), %g5
         cmp             %g4, %g5
-        blu,pn          %xcc, longpath
+        blu,pn          %xcc, kvmap_dtlb_longpath
          mov            (VMALLOC_END >> 24), %g5
         sllx            %g5, 24, %g5
         cmp             %g4, %g5
-        bgeu,pn         %xcc, longpath
+        bgeu,pn         %xcc, kvmap_dtlb_longpath
          nop
 
 kvmap_check_obp:
         sethi           %hi(LOW_OBP_ADDRESS), %g5
         cmp             %g4, %g5
-        blu,pn          %xcc, kvmap_vmalloc_addr
+        blu,pn          %xcc, kvmap_dtlb_vmalloc_addr
          mov            0x1, %g5
         sllx            %g5, 32, %g5
         cmp             %g4, %g5
-        blu,pn          %xcc, kvmap_do_obp
+        blu,pn          %xcc, kvmap_dtlb_obp
          nop
-
-kvmap_vmalloc_addr:
-        /* If we get here, a vmalloc addr was accessed, load kernel VPTE.  */
-        ldxa            [%g3 + %g6] ASI_N, %g5
-        brgez,pn        %g5, longpath
+        ba,pt           %xcc, kvmap_dtlb_vmalloc_addr
          nop
 
-kvmap_load:
-        /* PTE is valid, load into TLB and return from trap.  */
-        stxa            %g5, [%g0] ASI_DTLB_DATA_IN     ! Reload TLB
-        retry
+kvmap_dtlb_longpath:
+        rdpr    %pstate, %g5
+        wrpr    %g5, PSTATE_AG | PSTATE_MG, %pstate
+        rdpr    %tl, %g4
+        cmp     %g4, 1
+        mov     TLB_TAG_ACCESS, %g4
+        ldxa    [%g4] ASI_DMMU, %g5
+        be,pt   %xcc, sparc64_realfault_common
+         mov    FAULT_CODE_DTLB, %g4
+        ba,pt   %xcc, winfix_trampoline
+         nop