[SPARC64]: Add a secondary TSB for hugepage mappings.

Signed-off-by: David S. Miller <davem@davemloft.net>

8 files changed, 401 insertions, 171 deletions

diff --git a/arch/sparc64/Kconfig b/arch/sparc64/Kconfig
index c3685b314d71..267afddf63cf 100644
--- a/arch/sparc64/Kconfig
+++ b/arch/sparc64/Kconfig
@@ -175,11 +175,11 @@ config HUGETLB_PAGE_SIZE_4MB
         bool "4MB"
 
 config HUGETLB_PAGE_SIZE_512K
-        depends on !SPARC64_PAGE_SIZE_4MB
+        depends on !SPARC64_PAGE_SIZE_4MB && !SPARC64_PAGE_SIZE_512KB
         bool "512K"
 
 config HUGETLB_PAGE_SIZE_64K
-        depends on !SPARC64_PAGE_SIZE_4MB && !SPARC64_PAGE_SIZE_512KB
+        depends on !SPARC64_PAGE_SIZE_4MB && !SPARC64_PAGE_SIZE_512KB && !SPARC64_PAGE_SIZE_64K
         bool "64K"
 
 endchoice
diff --git a/arch/sparc64/kernel/sun4v_tlb_miss.S b/arch/sparc64/kernel/sun4v_tlb_miss.S
index ab23ddb7116e..b731881224e8 100644
--- a/arch/sparc64/kernel/sun4v_tlb_miss.S
+++ b/arch/sparc64/kernel/sun4v_tlb_miss.S
@@ -29,15 +29,15 @@
          *
          * index_mask = (512 << (tsb_reg & 0x7UL)) - 1UL;
          * tsb_base = tsb_reg & ~0x7UL;
-         * tsb_index = ((vaddr >> PAGE_SHIFT) & tsb_mask);
+         * tsb_index = ((vaddr >> HASH_SHIFT) & tsb_mask);
          * tsb_ptr = tsb_base + (tsb_index * 16);
          */
-#define COMPUTE_TSB_PTR(TSB_PTR, VADDR, TMP1, TMP2)     \
+#define COMPUTE_TSB_PTR(TSB_PTR, VADDR, HASH_SHIFT, TMP1, TMP2) \
         and     TSB_PTR, 0x7, TMP1;                     \
         mov     512, TMP2;                              \
         andn    TSB_PTR, 0x7, TSB_PTR;                  \
         sllx    TMP2, TMP1, TMP2;                       \
-        srlx    VADDR, PAGE_SHIFT, TMP1;                \
+        srlx    VADDR, HASH_SHIFT, TMP1;                \
         sub     TMP2, 1, TMP2;                          \
         and     TMP1, TMP2, TMP1;                       \
         sllx    TMP1, 4, TMP1;                          \
@@ -53,7 +53,7 @@ sun4v_itlb_miss:
 
         LOAD_ITLB_INFO(%g2, %g4, %g5)
         COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_itlb_4v)
-        COMPUTE_TSB_PTR(%g1, %g4, %g3, %g7)
+        COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7)
 
         /* Load TSB tag/pte into %g2/%g3 and compare the tag.  */
         ldda    [%g1] ASI_QUAD_LDD_PHYS_4V, %g2
@@ -99,7 +99,7 @@ sun4v_dtlb_miss:
 
         LOAD_DTLB_INFO(%g2, %g4, %g5)
         COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_dtlb_4v)
-        COMPUTE_TSB_PTR(%g1, %g4, %g3, %g7)
+        COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7)
 
         /* Load TSB tag/pte into %g2/%g3 and compare the tag.  */
         ldda    [%g1] ASI_QUAD_LDD_PHYS_4V, %g2
@@ -171,21 +171,26 @@ sun4v_dtsb_miss:
 
         /* fallthrough */
 
-        /* Create TSB pointer into %g1.  This is something like:
-         *
-         * index_mask = (512 << (tsb_reg & 0x7UL)) - 1UL;
-         * tsb_base = tsb_reg & ~0x7UL;
-         * tsb_index = ((vaddr >> PAGE_SHIFT) & tsb_mask);
-         * tsb_ptr = tsb_base + (tsb_index * 16);
-         */
 sun4v_tsb_miss_common:
-        COMPUTE_TSB_PTR(%g1, %g4, %g5, %g7)
+        COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g5, %g7)
 
-        /* Branch directly to page table lookup.  We have SCRATCHPAD_MMU_MISS
-         * still in %g2, so it's quite trivial to get at the PGD PHYS value
-         * so we can preload it into %g7.
-         */
         sub     %g2, TRAP_PER_CPU_FAULT_INFO, %g2
+
+#ifdef CONFIG_HUGETLB_PAGE
+        mov     SCRATCHPAD_UTSBREG2, %g5
+        ldxa    [%g5] ASI_SCRATCHPAD, %g5
+        cmp     %g5, -1
+        be,pt   %xcc, 80f
+         nop
+        COMPUTE_TSB_PTR(%g5, %g4, HPAGE_SHIFT, %g2, %g7)
+
+        /* That clobbered %g2, reload it.  */
+        ldxa    [%g0] ASI_SCRATCHPAD, %g2
+        sub     %g2, TRAP_PER_CPU_FAULT_INFO, %g2
+
+80:     stx     %g5, [%g2 + TRAP_PER_CPU_TSB_HUGE_TEMP]
+#endif
+
         ba,pt   %xcc, tsb_miss_page_table_walk_sun4v_fastpath
          ldx    [%g2 + TRAP_PER_CPU_PGD_PADDR], %g7
 
diff --git a/arch/sparc64/kernel/traps.c b/arch/sparc64/kernel/traps.c
index 7f7dba0ca96a..df612e4f75f9 100644
--- a/arch/sparc64/kernel/traps.c
+++ b/arch/sparc64/kernel/traps.c
@@ -2482,6 +2482,7 @@ void init_cur_cpu_trap(struct thread_info *t)
 
 extern void thread_info_offsets_are_bolixed_dave(void);
 extern void trap_per_cpu_offsets_are_bolixed_dave(void);
+extern void tsb_config_offsets_are_bolixed_dave(void);
 
 /* Only invoked on boot processor. */
 void __init trap_init(void)
@@ -2535,9 +2536,27 @@ void __init trap_init(void)
             (TRAP_PER_CPU_CPU_MONDO_BLOCK_PA !=
              offsetof(struct trap_per_cpu, cpu_mondo_block_pa)) ||
             (TRAP_PER_CPU_CPU_LIST_PA !=
-             offsetof(struct trap_per_cpu, cpu_list_pa)))
+             offsetof(struct trap_per_cpu, cpu_list_pa)) ||
+            (TRAP_PER_CPU_TSB_HUGE !=
+             offsetof(struct trap_per_cpu, tsb_huge)) ||
+            (TRAP_PER_CPU_TSB_HUGE_TEMP !=
+             offsetof(struct trap_per_cpu, tsb_huge_temp)))
                 trap_per_cpu_offsets_are_bolixed_dave();
 
+        if ((TSB_CONFIG_TSB !=
+             offsetof(struct tsb_config, tsb)) ||
+            (TSB_CONFIG_RSS_LIMIT !=
+             offsetof(struct tsb_config, tsb_rss_limit)) ||
+            (TSB_CONFIG_NENTRIES !=
+             offsetof(struct tsb_config, tsb_nentries)) ||
+            (TSB_CONFIG_REG_VAL !=
+             offsetof(struct tsb_config, tsb_reg_val)) ||
+            (TSB_CONFIG_MAP_VADDR !=
+             offsetof(struct tsb_config, tsb_map_vaddr)) ||
+            (TSB_CONFIG_MAP_PTE !=
+             offsetof(struct tsb_config, tsb_map_pte)))
+                tsb_config_offsets_are_bolixed_dave();
+
         /* Attach to the address space of init_task.  On SMP we
          * do this in smp.c:smp_callin for other cpus.
          */
diff --git a/arch/sparc64/kernel/tsb.S b/arch/sparc64/kernel/tsb.S
index 118baea44f69..a0c8ba58920b 100644
--- a/arch/sparc64/kernel/tsb.S
+++ b/arch/sparc64/kernel/tsb.S
@@ -3,8 +3,13 @@
  * Copyright (C) 2006 David S. Miller <davem@davemloft.net>
  */
 
+#include <linux/config.h>
+
 #include <asm/tsb.h>
 #include <asm/hypervisor.h>
+#include <asm/page.h>
+#include <asm/cpudata.h>
+#include <asm/mmu.h>
 
         .text
         .align  32
@@ -34,34 +39,124 @@ tsb_miss_itlb:
          ldxa           [%g4] ASI_IMMU, %g4
 
         /* At this point we have:
-         * %g1 --       TSB entry address
+         * %g1 --       PAGE_SIZE TSB entry address
          * %g3 --       FAULT_CODE_{D,I}TLB
          * %g4 --       missing virtual address
          * %g6 --       TAG TARGET (vaddr >> 22)
          */
 tsb_miss_page_table_walk:
-        TRAP_LOAD_PGD_PHYS(%g7, %g5)
+        TRAP_LOAD_TRAP_BLOCK(%g7, %g5)
 
-        /* And now we have the PGD base physical address in %g7.  */
-tsb_miss_page_table_walk_sun4v_fastpath:
-        USER_PGTABLE_WALK_TL1(%g4, %g7, %g5, %g2, tsb_do_fault)
+        /* Before committing to a full page table walk,
+         * check the huge page TSB.
+         */
+#ifdef CONFIG_HUGETLB_PAGE
+
+661:    ldx             [%g7 + TRAP_PER_CPU_TSB_HUGE], %g5
+        nop
+        .section        .sun4v_2insn_patch, "ax"
+        .word           661b
+        mov             SCRATCHPAD_UTSBREG2, %g5
+        ldxa            [%g5] ASI_SCRATCHPAD, %g5
+        .previous
+
+        cmp             %g5, -1
+        be,pt           %xcc, 80f
+         nop
+
+        /* We need an aligned pair of registers containing 2 values
+         * which can be easily rematerialized.  %g6 and %g7 foot the
+         * bill just nicely.  We'll save %g6 away into %g2 for the
+         * huge page TSB TAG comparison.
+         *
+         * Perform a huge page TSB lookup.
+         */
+        mov             %g6, %g2
+        and             %g5, 0x7, %g6
+        mov             512, %g7
+        andn            %g5, 0x7, %g5
+        sllx            %g7, %g6, %g7
+        srlx            %g4, HPAGE_SHIFT, %g6
+        sub             %g7, 1, %g7
+        and             %g6, %g7, %g6
+        sllx            %g6, 4, %g6
+        add             %g5, %g6, %g5
+
+        TSB_LOAD_QUAD(%g5, %g6)
+        cmp             %g6, %g2
+        be,a,pt         %xcc, tsb_tlb_reload
+         mov            %g7, %g5
+
+        /* No match, remember the huge page TSB entry address,
+         * and restore %g6 and %g7.
+         */
+        TRAP_LOAD_TRAP_BLOCK(%g7, %g6)
+        srlx            %g4, 22, %g6
+80:     stx             %g5, [%g7 + TRAP_PER_CPU_TSB_HUGE_TEMP]
+
+#endif
+
+        ldx             [%g7 + TRAP_PER_CPU_PGD_PADDR], %g7
 
         /* At this point we have:
          * %g1 --       TSB entry address
          * %g3 --       FAULT_CODE_{D,I}TLB
-         * %g5 --       physical address of PTE in Linux page tables
+         * %g4 --       missing virtual address
          * %g6 --       TAG TARGET (vaddr >> 22)
+         * %g7 --       page table physical address
+         *
+         * We know that both the base PAGE_SIZE TSB and the HPAGE_SIZE
+         * TSB both lack a matching entry.
          */
-tsb_reload:
-        TSB_LOCK_TAG(%g1, %g2, %g7)
+tsb_miss_page_table_walk_sun4v_fastpath:
+        USER_PGTABLE_WALK_TL1(%g4, %g7, %g5, %g2, tsb_do_fault)
 
         /* Load and check PTE.  */
         ldxa            [%g5] ASI_PHYS_USE_EC, %g5
-        mov             1, %g7
-        sllx            %g7, TSB_TAG_INVALID_BIT, %g7
-        brgez,a,pn      %g5, tsb_do_fault
-         TSB_STORE(%g1, %g7)
+        brgez,pn        %g5, tsb_do_fault
+         nop
+
+#ifdef CONFIG_HUGETLB_PAGE
+661:    sethi           %uhi(_PAGE_SZALL_4U), %g7
+        sllx            %g7, 32, %g7
+        .section        .sun4v_2insn_patch, "ax"
+        .word           661b
+        mov             _PAGE_SZALL_4V, %g7
+        nop
+        .previous
+
+        and             %g5, %g7, %g2
+
+661:    sethi           %uhi(_PAGE_SZHUGE_4U), %g7
+        sllx            %g7, 32, %g7
+        .section        .sun4v_2insn_patch, "ax"
+        .word           661b
+        mov             _PAGE_SZHUGE_4V, %g7
+        nop
+        .previous
+
+        cmp             %g2, %g7
+        bne,pt          %xcc, 60f
+         nop
+
+        /* It is a huge page, use huge page TSB entry address we
+         * calculated above.
+         */
+        TRAP_LOAD_TRAP_BLOCK(%g7, %g2)
+        ldx             [%g7 + TRAP_PER_CPU_TSB_HUGE_TEMP], %g2
+        cmp             %g2, -1
+        movne           %xcc, %g2, %g1
+60:
+#endif
 
+        /* At this point we have:
+         * %g1 --       TSB entry address
+         * %g3 --       FAULT_CODE_{D,I}TLB
+         * %g5 --       valid PTE
+         * %g6 --       TAG TARGET (vaddr >> 22)
+         */
+tsb_reload:
+        TSB_LOCK_TAG(%g1, %g2, %g7)
         TSB_WRITE(%g1, %g5, %g6)
 
         /* Finally, load TLB and return from trap.  */
@@ -240,10 +335,9 @@ tsb_flush:
          * schedule() time.
          *
          * %o0: page table physical address
-         * %o1: TSB register value
-         * %o2: TSB virtual address
-         * %o3: TSB mapping locked PTE
-         * %o4: Hypervisor TSB descriptor physical address
+         * %o1: TSB base config pointer
+         * %o2: TSB huge config pointer, or NULL if none
+         * %o3: Hypervisor TSB descriptor physical address
          *
          * We have to run this whole thing with interrupts
          * disabled so that the current cpu doesn't change
@@ -253,63 +347,79 @@ tsb_flush:
         .globl  __tsb_context_switch
         .type   __tsb_context_switch,#function
 __tsb_context_switch:
-        rdpr    %pstate, %o5
-        wrpr    %o5, PSTATE_IE, %pstate
+        rdpr    %pstate, %g1
+        wrpr    %g1, PSTATE_IE, %pstate
+
+        TRAP_LOAD_TRAP_BLOCK(%g2, %g3)
 
-        ldub    [%g6 + TI_CPU], %g1
-        sethi   %hi(trap_block), %g2
-        sllx    %g1, TRAP_BLOCK_SZ_SHIFT, %g1
-        or      %g2, %lo(trap_block), %g2
-        add     %g2, %g1, %g2
         stx     %o0, [%g2 + TRAP_PER_CPU_PGD_PADDR]
 
-        sethi   %hi(tlb_type), %g1
-        lduw    [%g1 + %lo(tlb_type)], %g1
-        cmp     %g1, 3
-        bne,pt  %icc, 1f
+        ldx     [%o1 + TSB_CONFIG_REG_VAL], %o0
+        brz,pt  %o2, 1f
+         mov    -1, %g3
+
+        ldx     [%o2 + TSB_CONFIG_REG_VAL], %g3
+
+1:      stx     %g3, [%g2 + TRAP_PER_CPU_TSB_HUGE]
+
+        sethi   %hi(tlb_type), %g2
+        lduw    [%g2 + %lo(tlb_type)], %g2
+        cmp     %g2, 3
+        bne,pt  %icc, 50f
          nop
 
         /* Hypervisor TSB switch. */
-        mov     SCRATCHPAD_UTSBREG1, %g1
-        stxa    %o1, [%g1] ASI_SCRATCHPAD
-        mov     -1, %g2
-        mov     SCRATCHPAD_UTSBREG2, %g1
-        stxa    %g2, [%g1] ASI_SCRATCHPAD
-
-        /* Save away %o5's %pstate, we have to use %o5 for
-         * the hypervisor call.
-         */
-        mov     %o5, %g1
+        mov     SCRATCHPAD_UTSBREG1, %o5
+        stxa    %o0, [%o5] ASI_SCRATCHPAD
+        mov     SCRATCHPAD_UTSBREG2, %o5
+        stxa    %g3, [%o5] ASI_SCRATCHPAD
+
+        mov     2, %o0
+        cmp     %g3, -1
+        move    %xcc, 1, %o0
 
         mov     HV_FAST_MMU_TSB_CTXNON0, %o5
-        mov     1, %o0
-        mov     %o4, %o1
+        mov     %o3, %o1
         ta      HV_FAST_TRAP
 
-        /* Finish up and restore %o5.  */
+        /* Finish up.  */
         ba,pt   %xcc, 9f
-         mov    %g1, %o5
+         nop
 
         /* SUN4U TSB switch.  */
-1:      mov     TSB_REG, %g1
-        stxa    %o1, [%g1] ASI_DMMU
+50:     mov     TSB_REG, %o5
+        stxa    %o0, [%o5] ASI_DMMU
         membar  #Sync
-        stxa    %o1, [%g1] ASI_IMMU
+        stxa    %o0, [%o5] ASI_IMMU
         membar  #Sync
 
-2:      brz     %o2, 9f
-         nop
+2:      ldx     [%o1 + TSB_CONFIG_MAP_VADDR], %o4
+        brz     %o4, 9f
+         ldx    [%o1 + TSB_CONFIG_MAP_PTE], %o5
 
         sethi   %hi(sparc64_highest_unlocked_tlb_ent), %g2
-        mov     TLB_TAG_ACCESS, %g1
+        mov     TLB_TAG_ACCESS, %g3
         lduw    [%g2 + %lo(sparc64_highest_unlocked_tlb_ent)], %g2
-        stxa    %o2, [%g1] ASI_DMMU
+        stxa    %o4, [%g3] ASI_DMMU
         membar  #Sync
         sllx    %g2, 3, %g2
-        stxa    %o3, [%g2] ASI_DTLB_DATA_ACCESS
+        stxa    %o5, [%g2] ASI_DTLB_DATA_ACCESS
+        membar  #Sync
+
+        brz,pt  %o2, 9f
+         nop
+
+        ldx     [%o2 + TSB_CONFIG_MAP_VADDR], %o4
+        ldx     [%o2 + TSB_CONFIG_MAP_PTE], %o5
+        mov     TLB_TAG_ACCESS, %g3
+        stxa    %o4, [%g3] ASI_DMMU
+        membar  #Sync
+        sub     %g2, (1 << 3), %g2
+        stxa    %o5, [%g2] ASI_DTLB_DATA_ACCESS
         membar  #Sync
+
 9:
-        wrpr    %o5, %pstate
+        wrpr    %g1, %pstate
 
         retl
          nop
diff --git a/arch/sparc64/mm/fault.c b/arch/sparc64/mm/fault.c
index 63b6cc0cd5d5..d21ff3230c02 100644
--- a/arch/sparc64/mm/fault.c
+++ b/arch/sparc64/mm/fault.c
@@ -410,9 +410,18 @@ good_area:
         up_read(&mm->mmap_sem);
 
         mm_rss = get_mm_rss(mm);
-        if (unlikely(mm_rss >= mm->context.tsb_rss_limit))
-                tsb_grow(mm, mm_rss);
-
+#ifdef CONFIG_HUGETLB_PAGE
+        mm_rss -= (mm->context.huge_pte_count * (HPAGE_SIZE / PAGE_SIZE));
+#endif
+        if (unlikely(mm_rss >=
+                     mm->context.tsb_block[MM_TSB_BASE].tsb_rss_limit))
+                tsb_grow(mm, MM_TSB_BASE, mm_rss);
+#ifdef CONFIG_HUGETLB_PAGE
+        mm_rss = mm->context.huge_pte_count;
+        if (unlikely(mm_rss >=
+                     mm->context.tsb_block[MM_TSB_HUGE].tsb_rss_limit))
+                tsb_grow(mm, MM_TSB_HUGE, mm_rss);
+#endif
         return;
 
         /*
diff --git a/arch/sparc64/mm/hugetlbpage.c b/arch/sparc64/mm/hugetlbpage.c
index a7a24869d045..0a1d4cd24cda 100644
--- a/arch/sparc64/mm/hugetlbpage.c
+++ b/arch/sparc64/mm/hugetlbpage.c
@@ -199,13 +199,11 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr)
         pte_t *pte = NULL;
 
         pgd = pgd_offset(mm, addr);
-        if (pgd) {
-                pud = pud_offset(pgd, addr);
-                if (pud) {
-                        pmd = pmd_alloc(mm, pud, addr);
-                        if (pmd)
-                                pte = pte_alloc_map(mm, pmd, addr);
-                }
+        pud = pud_alloc(mm, pgd, addr);
+        if (pud) {
+                pmd = pmd_alloc(mm, pud, addr);
+                if (pmd)
+                        pte = pte_alloc_map(mm, pmd, addr);
         }
         return pte;
 }
@@ -231,13 +229,14 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
         return pte;
 }
 
-#define mk_pte_huge(entry) do { pte_val(entry) |= _PAGE_SZHUGE; } while (0)
-
 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
                      pte_t *ptep, pte_t entry)
 {
         int i;
 
+        if (!pte_present(*ptep) && pte_present(entry))
+                mm->context.huge_pte_count++;
+
         for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) {
                 set_pte_at(mm, addr, ptep, entry);
                 ptep++;
@@ -253,6 +252,8 @@ pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
         int i;
 
         entry = *ptep;
+        if (pte_present(entry))
+                mm->context.huge_pte_count--;
 
         for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) {
                 pte_clear(mm, addr, ptep);
@@ -302,6 +303,15 @@ static void context_reload(void *__data)
 
 void hugetlb_prefault_arch_hook(struct mm_struct *mm)
 {
+        struct tsb_config *tp = &mm->context.tsb_block[MM_TSB_HUGE];
+
+        if (likely(tp->tsb != NULL))
+                return;
+
+        tsb_grow(mm, MM_TSB_HUGE, 0);
+        tsb_context_switch(mm);
+        smp_tsb_sync(mm);
+
         /* On UltraSPARC-III+ and later, configure the second half of
          * the Data-TLB for huge pages.
          */
diff --git a/arch/sparc64/mm/init.c b/arch/sparc64/mm/init.c
index c2b556106fc1..16d231703d6a 100644
--- a/arch/sparc64/mm/init.c
+++ b/arch/sparc64/mm/init.c
@@ -283,6 +283,7 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t p
         struct mm_struct *mm;
         struct tsb *tsb;
         unsigned long tag, flags;
+        unsigned long tsb_index, tsb_hash_shift;
 
         if (tlb_type != hypervisor) {
                 unsigned long pfn = pte_pfn(pte);
@@ -312,10 +313,26 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t p
 
         mm = vma->vm_mm;
 
+        tsb_index = MM_TSB_BASE;
+        tsb_hash_shift = PAGE_SHIFT;
+
         spin_lock_irqsave(&mm->context.lock, flags);
 
-        tsb = &mm->context.tsb[(address >> PAGE_SHIFT) &
-                               (mm->context.tsb_nentries - 1UL)];
+#ifdef CONFIG_HUGETLB_PAGE
+        if (mm->context.tsb_block[MM_TSB_HUGE].tsb != NULL) {
+                if ((tlb_type == hypervisor &&
+                     (pte_val(pte) & _PAGE_SZALL_4V) == _PAGE_SZHUGE_4V) ||
+                    (tlb_type != hypervisor &&
+                     (pte_val(pte) & _PAGE_SZALL_4U) == _PAGE_SZHUGE_4U)) {
+                        tsb_index = MM_TSB_HUGE;
+                        tsb_hash_shift = HPAGE_SHIFT;
+                }
+        }
+#endif
+
+        tsb = mm->context.tsb_block[tsb_index].tsb;
+        tsb += ((address >> tsb_hash_shift) &
+                (mm->context.tsb_block[tsb_index].tsb_nentries - 1UL));
         tag = (address >> 22UL);
         tsb_insert(tsb, tag, pte_val(pte));
 
diff --git a/arch/sparc64/mm/tsb.c b/arch/sparc64/mm/tsb.c
index b2064e2a44d6..beaa02810f0e 100644
--- a/arch/sparc64/mm/tsb.c
+++ b/arch/sparc64/mm/tsb.c
@@ -15,9 +15,9 @@
 
 extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES];
 
-static inline unsigned long tsb_hash(unsigned long vaddr, unsigned long nentries)
+static inline unsigned long tsb_hash(unsigned long vaddr, unsigned long hash_shift, unsigned long nentries)
 {
-        vaddr >>= PAGE_SHIFT;
+        vaddr >>= hash_shift;
         return vaddr & (nentries - 1);
 }
 
@@ -36,7 +36,8 @@ void flush_tsb_kernel_range(unsigned long start, unsigned long end)
         unsigned long v;
 
         for (v = start; v < end; v += PAGE_SIZE) {
-                unsigned long hash = tsb_hash(v, KERNEL_TSB_NENTRIES);
+                unsigned long hash = tsb_hash(v, PAGE_SHIFT,
+                                              KERNEL_TSB_NENTRIES);
                 struct tsb *ent = &swapper_tsb[hash];
 
                 if (tag_compare(ent->tag, v)) {
@@ -46,49 +47,91 @@ void flush_tsb_kernel_range(unsigned long start, unsigned long end)
         }
 }
 
-void flush_tsb_user(struct mmu_gather *mp)
+static void __flush_tsb_one(struct mmu_gather *mp, unsigned long hash_shift, unsigned long tsb, unsigned long nentries)
 {
-        struct mm_struct *mm = mp->mm;
-        unsigned long nentries, base, flags;
-        struct tsb *tsb;
-        int i;
-
-        spin_lock_irqsave(&mm->context.lock, flags);
-
-        tsb = mm->context.tsb;
-        nentries = mm->context.tsb_nentries;
+        unsigned long i;
 
-        if (tlb_type == cheetah_plus || tlb_type == hypervisor)
-                base = __pa(tsb);
-        else
-                base = (unsigned long) tsb;
-        
         for (i = 0; i < mp->tlb_nr; i++) {
                 unsigned long v = mp->vaddrs[i];
                 unsigned long tag, ent, hash;
 
                 v &= ~0x1UL;
 
-                hash = tsb_hash(v, nentries);
-                ent = base + (hash * sizeof(struct tsb));
+                hash = tsb_hash(v, hash_shift, nentries);
+                ent = tsb + (hash * sizeof(struct tsb));
                 tag = (v >> 22UL);
 
                 tsb_flush(ent, tag);
         }
+}
+
+void flush_tsb_user(struct mmu_gather *mp)
+{
+        struct mm_struct *mm = mp->mm;
+        unsigned long nentries, base, flags;
+
+        spin_lock_irqsave(&mm->context.lock, flags);
 
+        base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb;
+        nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries;
+        if (tlb_type == cheetah_plus || tlb_type == hypervisor)
+                base = __pa(base);
+        __flush_tsb_one(mp, PAGE_SHIFT, base, nentries);
+
+#ifdef CONFIG_HUGETLB_PAGE
+        if (mm->context.tsb_block[MM_TSB_HUGE].tsb) {
+                base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb;
+                nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries;
+                if (tlb_type == cheetah_plus || tlb_type == hypervisor)
+                        base = __pa(base);
+                __flush_tsb_one(mp, HPAGE_SHIFT, base, nentries);
+        }
+#endif
         spin_unlock_irqrestore(&mm->context.lock, flags);
 }
 
-static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_bytes)
+#if defined(CONFIG_SPARC64_PAGE_SIZE_8KB)
+#define HV_PGSZ_IDX_BASE        HV_PGSZ_IDX_8K
+#define HV_PGSZ_MASK_BASE       HV_PGSZ_MASK_8K
+#elif defined(CONFIG_SPARC64_PAGE_SIZE_64KB)
+#define HV_PGSZ_IDX_BASE        HV_PGSZ_IDX_64K
+#define HV_PGSZ_MASK_BASE       HV_PGSZ_MASK_64K
+#elif defined(CONFIG_SPARC64_PAGE_SIZE_512KB)
+#define HV_PGSZ_IDX_BASE        HV_PGSZ_IDX_512K
+#define HV_PGSZ_MASK_BASE       HV_PGSZ_MASK_512K
+#elif defined(CONFIG_SPARC64_PAGE_SIZE_4MB)
+#define HV_PGSZ_IDX_BASE        HV_PGSZ_IDX_4MB
+#define HV_PGSZ_MASK_BASE       HV_PGSZ_MASK_4MB
+#else
+#error Broken base page size setting...
+#endif
+
+#ifdef CONFIG_HUGETLB_PAGE
+#if defined(CONFIG_HUGETLB_PAGE_SIZE_64K)
+#define HV_PGSZ_IDX_HUGE        HV_PGSZ_IDX_64K
+#define HV_PGSZ_MASK_HUGE       HV_PGSZ_MASK_64K
+#elif defined(CONFIG_HUGETLB_PAGE_SIZE_512K)
+#define HV_PGSZ_IDX_HUGE        HV_PGSZ_IDX_512K
+#define HV_PGSZ_MASK_HUGE       HV_PGSZ_MASK_512K
+#elif defined(CONFIG_HUGETLB_PAGE_SIZE_4MB)
+#define HV_PGSZ_IDX_HUGE        HV_PGSZ_IDX_4MB
+#define HV_PGSZ_MASK_HUGE       HV_PGSZ_MASK_4MB
+#else
+#error Broken huge page size setting...
+#endif
+#endif
+
+static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx, unsigned long tsb_bytes)
 {
         unsigned long tsb_reg, base, tsb_paddr;
         unsigned long page_sz, tte;
 
-        mm->context.tsb_nentries = tsb_bytes / sizeof(struct tsb);
+        mm->context.tsb_block[tsb_idx].tsb_nentries =
+                tsb_bytes / sizeof(struct tsb);
 
         base = TSBMAP_BASE;
         tte = pgprot_val(PAGE_KERNEL_LOCKED);
-        tsb_paddr = __pa(mm->context.tsb);
+        tsb_paddr = __pa(mm->context.tsb_block[tsb_idx].tsb);
         BUG_ON(tsb_paddr & (tsb_bytes - 1UL));
 
         /* Use the smallest page size that can map the whole TSB
@@ -147,61 +190,49 @@ static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_bytes)
                 /* Physical mapping, no locked TLB entry for TSB.  */
                 tsb_reg |= tsb_paddr;
 
-                mm->context.tsb_reg_val = tsb_reg;
-                mm->context.tsb_map_vaddr = 0;
-                mm->context.tsb_map_pte = 0;
+                mm->context.tsb_block[tsb_idx].tsb_reg_val = tsb_reg;
+                mm->context.tsb_block[tsb_idx].tsb_map_vaddr = 0;
+                mm->context.tsb_block[tsb_idx].tsb_map_pte = 0;
         } else {
                 tsb_reg |= base;
                 tsb_reg |= (tsb_paddr & (page_sz - 1UL));
                 tte |= (tsb_paddr & ~(page_sz - 1UL));
 
-                mm->context.tsb_reg_val = tsb_reg;
-                mm->context.tsb_map_vaddr = base;
-                mm->context.tsb_map_pte = tte;
+                mm->context.tsb_block[tsb_idx].tsb_reg_val = tsb_reg;
+                mm->context.tsb_block[tsb_idx].tsb_map_vaddr = base;
+                mm->context.tsb_block[tsb_idx].tsb_map_pte = tte;
         }
 
         /* Setup the Hypervisor TSB descriptor.  */
         if (tlb_type == hypervisor) {
-                struct hv_tsb_descr *hp = &mm->context.tsb_descr;
+                struct hv_tsb_descr *hp = &mm->context.tsb_descr[tsb_idx];
 
-                switch (PAGE_SIZE) {
-                case 8192:
-                default:
-                        hp->pgsz_idx = HV_PGSZ_IDX_8K;
+                switch (tsb_idx) {
+                case MM_TSB_BASE:
+                        hp->pgsz_idx = HV_PGSZ_IDX_BASE;
                         break;
-
-                case 64 * 1024:
-                        hp->pgsz_idx = HV_PGSZ_IDX_64K;
-                        break;
-
-                case 512 * 1024:
-                        hp->pgsz_idx = HV_PGSZ_IDX_512K;
-                        break;
-
-                case 4 * 1024 * 1024:
-                        hp->pgsz_idx = HV_PGSZ_IDX_4MB;
+#ifdef CONFIG_HUGETLB_PAGE
+                case MM_TSB_HUGE:
+                        hp->pgsz_idx = HV_PGSZ_IDX_HUGE;
                         break;
+#endif
+                default:
+                        BUG();
                 };
                 hp->assoc = 1;
                 hp->num_ttes = tsb_bytes / 16;
                 hp->ctx_idx = 0;
-                switch (PAGE_SIZE) {
-                case 8192:
-                default:
-                        hp->pgsz_mask = HV_PGSZ_MASK_8K;
-                        break;
-
-                case 64 * 1024:
-                        hp->pgsz_mask = HV_PGSZ_MASK_64K;
-                        break;
-
-                case 512 * 1024:
-                        hp->pgsz_mask = HV_PGSZ_MASK_512K;
+                switch (tsb_idx) {
+                case MM_TSB_BASE:
+                        hp->pgsz_mask = HV_PGSZ_MASK_BASE;
                         break;
-
-                case 4 * 1024 * 1024:
-                        hp->pgsz_mask = HV_PGSZ_MASK_4MB;
+#ifdef CONFIG_HUGETLB_PAGE
+                case MM_TSB_HUGE:
+                        hp->pgsz_mask = HV_PGSZ_MASK_HUGE;
                         break;
+#endif
+                default:
+                        BUG();
                 };
                 hp->tsb_base = tsb_paddr;
                 hp->resv = 0;
@@ -241,11 +272,11 @@ void __init tsb_cache_init(void)
         }
 }
 
-/* When the RSS of an address space exceeds mm->context.tsb_rss_limit,
- * do_sparc64_fault() invokes this routine to try and grow the TSB.
+/* When the RSS of an address space exceeds tsb_rss_limit for a TSB,
+ * do_sparc64_fault() invokes this routine to try and grow it.
  *
  * When we reach the maximum TSB size supported, we stick ~0UL into
- * mm->context.tsb_rss_limit so the grow checks in update_mmu_cache()
+ * tsb_rss_limit for that TSB so the grow checks in do_sparc64_fault()
  * will not trigger any longer.
  *
  * The TSB can be anywhere from 8K to 1MB in size, in increasing powers
@@ -257,7 +288,7 @@ void __init tsb_cache_init(void)
  * the number of entries that the current TSB can hold at once.  Currently,
  * we trigger when the RSS hits 3/4 of the TSB capacity.
  */
-void tsb_grow(struct mm_struct *mm, unsigned long rss)
+void tsb_grow(struct mm_struct *mm, unsigned long tsb_index, unsigned long rss)
 {
         unsigned long max_tsb_size = 1 * 1024 * 1024;
         unsigned long new_size, old_size, flags;
@@ -297,7 +328,8 @@ retry_tsb_alloc:
                  * down to a 0-order allocation and force no TSB
                  * growing for this address space.
                  */
-                if (mm->context.tsb == NULL && new_cache_index > 0) {
+                if (mm->context.tsb_block[tsb_index].tsb == NULL &&
+                    new_cache_index > 0) {
                         new_cache_index = 0;
                         new_size = 8192;
                         new_rss_limit = ~0UL;
@@ -307,8 +339,8 @@ retry_tsb_alloc:
                 /* If we failed on a TSB grow, we are under serious
                  * memory pressure so don't try to grow any more.
                  */
-                if (mm->context.tsb != NULL)
-                        mm->context.tsb_rss_limit = ~0UL;
+                if (mm->context.tsb_block[tsb_index].tsb != NULL)
+                        mm->context.tsb_block[tsb_index].tsb_rss_limit = ~0UL;
                 return;
         }
 
@@ -339,23 +371,26 @@ retry_tsb_alloc:
          */
         spin_lock_irqsave(&mm->context.lock, flags);
 
-        old_tsb = mm->context.tsb;
-        old_cache_index = (mm->context.tsb_reg_val & 0x7UL);
-        old_size = mm->context.tsb_nentries * sizeof(struct tsb);
+        old_tsb = mm->context.tsb_block[tsb_index].tsb;
+        old_cache_index =
+                (mm->context.tsb_block[tsb_index].tsb_reg_val & 0x7UL);
+        old_size = (mm->context.tsb_block[tsb_index].tsb_nentries *
+                    sizeof(struct tsb));
 
 
         /* Handle multiple threads trying to grow the TSB at the same time.
          * One will get in here first, and bump the size and the RSS limit.
          * The others will get in here next and hit this check.
          */
-        if (unlikely(old_tsb && (rss < mm->context.tsb_rss_limit))) {
+        if (unlikely(old_tsb &&
+                     (rss < mm->context.tsb_block[tsb_index].tsb_rss_limit))) {
                 spin_unlock_irqrestore(&mm->context.lock, flags);
 
                 kmem_cache_free(tsb_caches[new_cache_index], new_tsb);
                 return;
         }
 
-        mm->context.tsb_rss_limit = new_rss_limit;
+        mm->context.tsb_block[tsb_index].tsb_rss_limit = new_rss_limit;
 
         if (old_tsb) {
                 extern void copy_tsb(unsigned long old_tsb_base,
@@ -372,8 +407,8 @@ retry_tsb_alloc:
                 copy_tsb(old_tsb_base, old_size, new_tsb_base, new_size);
         }
 
-        mm->context.tsb = new_tsb;
-        setup_tsb_params(mm, new_size);
+        mm->context.tsb_block[tsb_index].tsb = new_tsb;
+        setup_tsb_params(mm, tsb_index, new_size);
 
         spin_unlock_irqrestore(&mm->context.lock, flags);
 
@@ -394,40 +429,65 @@ retry_tsb_alloc:
 
 int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
 {
+#ifdef CONFIG_HUGETLB_PAGE
+        unsigned long huge_pte_count;
+#endif
+        unsigned int i;
+
         spin_lock_init(&mm->context.lock);
 
         mm->context.sparc64_ctx_val = 0UL;
 
+#ifdef CONFIG_HUGETLB_PAGE
+        /* We reset it to zero because the fork() page copying
+         * will re-increment the counters as the parent PTEs are
+         * copied into the child address space.
+         */
+        huge_pte_count = mm->context.huge_pte_count;
+        mm->context.huge_pte_count = 0;
+#endif
+
         /* copy_mm() copies over the parent's mm_struct before calling
          * us, so we need to zero out the TSB pointer or else tsb_grow()
          * will be confused and think there is an older TSB to free up.
          */
-        mm->context.tsb = NULL;
+        for (i = 0; i < MM_NUM_TSBS; i++)
+                mm->context.tsb_block[i].tsb = NULL;
 
         /* If this is fork, inherit the parent's TSB size.  We would
          * grow it to that size on the first page fault anyways.
          */
-        tsb_grow(mm, get_mm_rss(mm));
+        tsb_grow(mm, MM_TSB_BASE, get_mm_rss(mm));
 
-        if (unlikely(!mm->context.tsb))
+#ifdef CONFIG_HUGETLB_PAGE
+        if (unlikely(huge_pte_count))
+                tsb_grow(mm, MM_TSB_HUGE, huge_pte_count);
+#endif
+
+        if (unlikely(!mm->context.tsb_block[MM_TSB_BASE].tsb))
                 return -ENOMEM;
 
         return 0;
 }
 
-void destroy_context(struct mm_struct *mm)
+static void tsb_destroy_one(struct tsb_config *tp)
 {
-        unsigned long flags, cache_index;
+        unsigned long cache_index;
 
-        cache_index = (mm->context.tsb_reg_val & 0x7UL);
-        kmem_cache_free(tsb_caches[cache_index], mm->context.tsb);
+        if (!tp->tsb)
+                return;
+        cache_index = tp->tsb_reg_val & 0x7UL;
+        kmem_cache_free(tsb_caches[cache_index], tp->tsb);
+        tp->tsb = NULL;
+        tp->tsb_reg_val = 0UL;
+}
 
-        /* We can remove these later, but for now it's useful
-         * to catch any bogus post-destroy_context() references
-         * to the TSB.
-         */
-        mm->context.tsb = NULL;
-        mm->context.tsb_reg_val = 0UL;
+void destroy_context(struct mm_struct *mm)
+{
+        unsigned long flags, i;
+
+        for (i = 0; i < MM_NUM_TSBS; i++)
+                tsb_destroy_one(&mm->context.tsb_block[i]);
 
         spin_lock_irqsave(&ctx_alloc_lock, flags);