1 files changed, 487 insertions, 64 deletions

diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index d58edf5fefdb..9e28a118e6a4 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -51,22 +51,40 @@
 
 #include "init_64.h"
 
-unsigned long kern_linear_pte_xor[2] __read_mostly;
+unsigned long kern_linear_pte_xor[4] __read_mostly;
 
-/* A bitmap, one bit for every 256MB of physical memory.  If the bit
- * is clear, we should use a 4MB page (via kern_linear_pte_xor[0]) else
- * if set we should use a 256MB page (via kern_linear_pte_xor[1]).
+/* A bitmap, two bits for every 256MB of physical memory.  These two
+ * bits determine what page size we use for kernel linear
+ * translations.  They form an index into kern_linear_pte_xor[].  The
+ * value in the indexed slot is XOR'd with the TLB miss virtual
+ * address to form the resulting TTE.  The mapping is:
+ *
+ *      0       ==>     4MB
+ *      1       ==>     256MB
+ *      2       ==>     2GB
+ *      3       ==>     16GB
+ *
+ * All sun4v chips support 256MB pages.  Only SPARC-T4 and later
+ * support 2GB pages, and hopefully future cpus will support the 16GB
+ * pages as well.  For slots 2 and 3, we encode a 256MB TTE xor there
+ * if these larger page sizes are not supported by the cpu.
+ *
+ * It would be nice to determine this from the machine description
+ * 'cpu' properties, but we need to have this table setup before the
+ * MDESC is initialized.
  */
 unsigned long kpte_linear_bitmap[KPTE_BITMAP_BYTES / sizeof(unsigned long)];
 
 #ifndef CONFIG_DEBUG_PAGEALLOC
-/* A special kernel TSB for 4MB and 256MB linear mappings.
- * Space is allocated for this right after the trap table
- * in arch/sparc64/kernel/head.S
+/* A special kernel TSB for 4MB, 256MB, 2GB and 16GB linear mappings.
+ * Space is allocated for this right after the trap table in
+ * arch/sparc64/kernel/head.S
  */
 extern struct tsb swapper_4m_tsb[KERNEL_TSB4M_NENTRIES];
 #endif
 
+static unsigned long cpu_pgsz_mask;
+
 #define MAX_BANKS       32
 
 static struct linux_prom64_registers pavail[MAX_BANKS] __devinitdata;
@@ -101,7 +119,8 @@ static void __init read_obp_memory(const char *property,
 
         ret = prom_getproperty(node, property, (char *) regs, prop_size);
         if (ret == -1) {
-                prom_printf("Couldn't get %s property from /memory.\n");
+                prom_printf("Couldn't get %s property from /memory.\n",
+                                property);
                 prom_halt();
         }
 
@@ -257,7 +276,6 @@ static inline void tsb_insert(struct tsb *ent, unsigned long tag, unsigned long
 }
 
 unsigned long _PAGE_ALL_SZ_BITS __read_mostly;
-unsigned long _PAGE_SZBITS __read_mostly;
 
 static void flush_dcache(unsigned long pfn)
 {
@@ -288,12 +306,24 @@ static void flush_dcache(unsigned long pfn)
         }
 }
 
+/* mm->context.lock must be held */
+static void __update_mmu_tsb_insert(struct mm_struct *mm, unsigned long tsb_index,
+                                    unsigned long tsb_hash_shift, unsigned long address,
+                                    unsigned long tte)
+{
+        struct tsb *tsb = mm->context.tsb_block[tsb_index].tsb;
+        unsigned long tag;
+
+        tsb += ((address >> tsb_hash_shift) &
+                (mm->context.tsb_block[tsb_index].tsb_nentries - 1UL));
+        tag = (address >> 22UL);
+        tsb_insert(tsb, tag, tte);
+}
+
 void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep)
 {
+        unsigned long tsb_index, tsb_hash_shift, flags;
         struct mm_struct *mm;
-        struct tsb *tsb;
-        unsigned long tag, flags;
-        unsigned long tsb_index, tsb_hash_shift;
         pte_t pte = *ptep;
 
         if (tlb_type != hypervisor) {
@@ -310,7 +340,7 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *
 
         spin_lock_irqsave(&mm->context.lock, flags);
 
-#ifdef CONFIG_HUGETLB_PAGE
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
         if (mm->context.tsb_block[MM_TSB_HUGE].tsb != NULL) {
                 if ((tlb_type == hypervisor &&
                      (pte_val(pte) & _PAGE_SZALL_4V) == _PAGE_SZHUGE_4V) ||
@@ -322,11 +352,8 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *
         }
 #endif
 
-        tsb = mm->context.tsb_block[tsb_index].tsb;
-        tsb += ((address >> tsb_hash_shift) &
-                (mm->context.tsb_block[tsb_index].tsb_nentries - 1UL));
-        tag = (address >> 22UL);
-        tsb_insert(tsb, tag, pte_val(pte));
+        __update_mmu_tsb_insert(mm, tsb_index, tsb_hash_shift,
+                                address, pte_val(pte));
 
         spin_unlock_irqrestore(&mm->context.lock, flags);
 }
@@ -403,6 +430,12 @@ EXPORT_SYMBOL(flush_icache_range);
 
 void mmu_info(struct seq_file *m)
 {
+        static const char *pgsz_strings[] = {
+                "8K", "64K", "512K", "4MB", "32MB",
+                "256MB", "2GB", "16GB",
+        };
+        int i, printed;
+
         if (tlb_type == cheetah)
                 seq_printf(m, "MMU Type\t: Cheetah\n");
         else if (tlb_type == cheetah_plus)
@@ -414,6 +447,17 @@ void mmu_info(struct seq_file *m)
         else
                 seq_printf(m, "MMU Type\t: ???\n");
 
+        seq_printf(m, "MMU PGSZs\t: ");
+        printed = 0;
+        for (i = 0; i < ARRAY_SIZE(pgsz_strings); i++) {
+                if (cpu_pgsz_mask & (1UL << i)) {
+                        seq_printf(m, "%s%s",
+                                   printed ? "," : "", pgsz_strings[i]);
+                        printed++;
+                }
+        }
+        seq_putc(m, '\n');
+
 #ifdef CONFIG_DEBUG_DCFLUSH
         seq_printf(m, "DCPageFlushes\t: %d\n",
                    atomic_read(&dcpage_flushes));
@@ -462,7 +506,7 @@ static void __init read_obp_translations(void)
                 prom_halt();
         }
         if (unlikely(n > sizeof(prom_trans))) {
-                prom_printf("prom_mappings: Size %Zd is too big.\n", n);
+                prom_printf("prom_mappings: Size %d is too big.\n", n);
                 prom_halt();
         }
 
@@ -524,7 +568,7 @@ static void __init hypervisor_tlb_lock(unsigned long vaddr,
         unsigned long ret = sun4v_mmu_map_perm_addr(vaddr, 0, pte, mmu);
 
         if (ret != 0) {
-                prom_printf("hypervisor_tlb_lock[%lx:%lx:%lx:%lx]: "
+                prom_printf("hypervisor_tlb_lock[%lx:%x:%lx:%lx]: "
                             "errors with %lx\n", vaddr, 0, pte, mmu, ret);
                 prom_halt();
         }
@@ -1358,32 +1402,75 @@ static unsigned long __ref kernel_map_range(unsigned long pstart,
 extern unsigned int kvmap_linear_patch[1];
 #endif /* CONFIG_DEBUG_PAGEALLOC */
 
-static void __init mark_kpte_bitmap(unsigned long start, unsigned long end)
+static void __init kpte_set_val(unsigned long index, unsigned long val)
 {
-        const unsigned long shift_256MB = 28;
-        const unsigned long mask_256MB = ((1UL << shift_256MB) - 1UL);
-        const unsigned long size_256MB = (1UL << shift_256MB);
+        unsigned long *ptr = kpte_linear_bitmap;
 
-        while (start < end) {
-                long remains;
+        val <<= ((index % (BITS_PER_LONG / 2)) * 2);
+        ptr += (index / (BITS_PER_LONG / 2));
 
-                remains = end - start;
-                if (remains < size_256MB)
-                        break;
+        *ptr |= val;
+}
 
-                if (start & mask_256MB) {
-                        start = (start + size_256MB) & ~mask_256MB;
-                        continue;
-                }
+static const unsigned long kpte_shift_min = 28; /* 256MB */
+static const unsigned long kpte_shift_max = 34; /* 16GB */
+static const unsigned long kpte_shift_incr = 3;
 
-                while (remains >= size_256MB) {
-                        unsigned long index = start >> shift_256MB;
+static unsigned long kpte_mark_using_shift(unsigned long start, unsigned long end,
+                                           unsigned long shift)
+{
+        unsigned long size = (1UL << shift);
+        unsigned long mask = (size - 1UL);
+        unsigned long remains = end - start;
+        unsigned long val;
 
-                        __set_bit(index, kpte_linear_bitmap);
+        if (remains < size || (start & mask))
+                return start;
 
-                        start += size_256MB;
-                        remains -= size_256MB;
+        /* VAL maps:
+         *
+         *      shift 28 --> kern_linear_pte_xor index 1
+         *      shift 31 --> kern_linear_pte_xor index 2
+         *      shift 34 --> kern_linear_pte_xor index 3
+         */
+        val = ((shift - kpte_shift_min) / kpte_shift_incr) + 1;
+
+        remains &= ~mask;
+        if (shift != kpte_shift_max)
+                remains = size;
+
+        while (remains) {
+                unsigned long index = start >> kpte_shift_min;
+
+                kpte_set_val(index, val);
+
+                start += 1UL << kpte_shift_min;
+                remains -= 1UL << kpte_shift_min;
+        }
+
+        return start;
+}
+
+static void __init mark_kpte_bitmap(unsigned long start, unsigned long end)
+{
+        unsigned long smallest_size, smallest_mask;
+        unsigned long s;
+
+        smallest_size = (1UL << kpte_shift_min);
+        smallest_mask = (smallest_size - 1UL);
+
+        while (start < end) {
+                unsigned long orig_start = start;
+
+                for (s = kpte_shift_max; s >= kpte_shift_min; s -= kpte_shift_incr) {
+                        start = kpte_mark_using_shift(start, end, s);
+
+                        if (start != orig_start)
+                                break;
                 }
+
+                if (start == orig_start)
+                        start = (start + smallest_size) & ~smallest_mask;
         }
 }
 
@@ -1577,13 +1664,16 @@ static void __init sun4v_ktsb_init(void)
         ktsb_descr[0].resv = 0;
 
 #ifndef CONFIG_DEBUG_PAGEALLOC
-        /* Second KTSB for 4MB/256MB mappings.  */
+        /* Second KTSB for 4MB/256MB/2GB/16GB mappings.  */
         ktsb_pa = (kern_base +
                    ((unsigned long)&swapper_4m_tsb[0] - KERNBASE));
 
         ktsb_descr[1].pgsz_idx = HV_PGSZ_IDX_4MB;
-        ktsb_descr[1].pgsz_mask = (HV_PGSZ_MASK_4MB |
-                                   HV_PGSZ_MASK_256MB);
+        ktsb_descr[1].pgsz_mask = ((HV_PGSZ_MASK_4MB |
+                                    HV_PGSZ_MASK_256MB |
+                                    HV_PGSZ_MASK_2GB |
+                                    HV_PGSZ_MASK_16GB) &
+                                   cpu_pgsz_mask);
         ktsb_descr[1].assoc = 1;
         ktsb_descr[1].num_ttes = KERNEL_TSB4M_NENTRIES;
         ktsb_descr[1].ctx_idx = 0;
@@ -1606,6 +1696,47 @@ void __cpuinit sun4v_ktsb_register(void)
         }
 }
 
+static void __init sun4u_linear_pte_xor_finalize(void)
+{
+#ifndef CONFIG_DEBUG_PAGEALLOC
+        /* This is where we would add Panther support for
+         * 32MB and 256MB pages.
+         */
+#endif
+}
+
+static void __init sun4v_linear_pte_xor_finalize(void)
+{
+#ifndef CONFIG_DEBUG_PAGEALLOC
+        if (cpu_pgsz_mask & HV_PGSZ_MASK_256MB) {
+                kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZ256MB_4V) ^
+                        0xfffff80000000000UL;
+                kern_linear_pte_xor[1] |= (_PAGE_CP_4V | _PAGE_CV_4V |
+                                           _PAGE_P_4V | _PAGE_W_4V);
+        } else {
+                kern_linear_pte_xor[1] = kern_linear_pte_xor[0];
+        }
+
+        if (cpu_pgsz_mask & HV_PGSZ_MASK_2GB) {
+                kern_linear_pte_xor[2] = (_PAGE_VALID | _PAGE_SZ2GB_4V) ^
+                        0xfffff80000000000UL;
+                kern_linear_pte_xor[2] |= (_PAGE_CP_4V | _PAGE_CV_4V |
+                                           _PAGE_P_4V | _PAGE_W_4V);
+        } else {
+                kern_linear_pte_xor[2] = kern_linear_pte_xor[1];
+        }
+
+        if (cpu_pgsz_mask & HV_PGSZ_MASK_16GB) {
+                kern_linear_pte_xor[3] = (_PAGE_VALID | _PAGE_SZ16GB_4V) ^
+                        0xfffff80000000000UL;
+                kern_linear_pte_xor[3] |= (_PAGE_CP_4V | _PAGE_CV_4V |
+                                           _PAGE_P_4V | _PAGE_W_4V);
+        } else {
+                kern_linear_pte_xor[3] = kern_linear_pte_xor[2];
+        }
+#endif
+}
+
 /* paging_init() sets up the page tables */
 
 static unsigned long last_valid_pfn;
@@ -1665,10 +1796,8 @@ void __init paging_init(void)
                 ktsb_phys_patch();
         }
 
-        if (tlb_type == hypervisor) {
+        if (tlb_type == hypervisor)
                 sun4v_patch_tlb_handlers();
-                sun4v_ktsb_init();
-        }
 
         /* Find available physical memory...
          *
@@ -1727,9 +1856,6 @@ void __init paging_init(void)
 
         __flush_tlb_all();
 
-        if (tlb_type == hypervisor)
-                sun4v_ktsb_register();
-
         prom_build_devicetree();
         of_populate_present_mask();
 #ifndef CONFIG_SMP
@@ -1742,8 +1868,36 @@ void __init paging_init(void)
 #ifndef CONFIG_SMP
                 mdesc_fill_in_cpu_data(cpu_all_mask);
 #endif
+                mdesc_get_page_sizes(cpu_all_mask, &cpu_pgsz_mask);
+
+                sun4v_linear_pte_xor_finalize();
+
+                sun4v_ktsb_init();
+                sun4v_ktsb_register();
+        } else {
+                unsigned long impl, ver;
+
+                cpu_pgsz_mask = (HV_PGSZ_MASK_8K | HV_PGSZ_MASK_64K |
+                                 HV_PGSZ_MASK_512K | HV_PGSZ_MASK_4MB);
+
+                __asm__ __volatile__("rdpr %%ver, %0" : "=r" (ver));
+                impl = ((ver >> 32) & 0xffff);
+                if (impl == PANTHER_IMPL)
+                        cpu_pgsz_mask |= (HV_PGSZ_MASK_32MB |
+                                          HV_PGSZ_MASK_256MB);
+
+                sun4u_linear_pte_xor_finalize();
         }
 
+        /* Flush the TLBs and the 4M TSB so that the updated linear
+         * pte XOR settings are realized for all mappings.
+         */
+        __flush_tlb_all();
+#ifndef CONFIG_DEBUG_PAGEALLOC
+        memset(swapper_4m_tsb, 0x40, sizeof(swapper_4m_tsb));
+#endif
+        __flush_tlb_all();
+
         /* Setup bootmem... */
         last_valid_pfn = end_pfn = bootmem_init(phys_base);
 
@@ -2110,6 +2264,7 @@ static void __init sun4u_pgprot_init(void)
 {
         unsigned long page_none, page_shared, page_copy, page_readonly;
         unsigned long page_exec_bit;
+        int i;
 
         PAGE_KERNEL = __pgprot (_PAGE_PRESENT_4U | _PAGE_VALID |
                                 _PAGE_CACHE_4U | _PAGE_P_4U |
@@ -2128,8 +2283,7 @@ static void __init sun4u_pgprot_init(void)
                      __ACCESS_BITS_4U | _PAGE_E_4U);
 
 #ifdef CONFIG_DEBUG_PAGEALLOC
-        kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZBITS_4U) ^
-                0xfffff80000000000UL;
+        kern_linear_pte_xor[0] = _PAGE_VALID ^ 0xfffff80000000000UL;
 #else
         kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4U) ^
                 0xfffff80000000000UL;
@@ -2137,10 +2291,9 @@ static void __init sun4u_pgprot_init(void)
         kern_linear_pte_xor[0] |= (_PAGE_CP_4U | _PAGE_CV_4U |
                                    _PAGE_P_4U | _PAGE_W_4U);
 
-        /* XXX Should use 256MB on Panther. XXX */
-        kern_linear_pte_xor[1] = kern_linear_pte_xor[0];
+        for (i = 1; i < 4; i++)
+                kern_linear_pte_xor[i] = kern_linear_pte_xor[0];
 
-        _PAGE_SZBITS = _PAGE_SZBITS_4U;
         _PAGE_ALL_SZ_BITS =  (_PAGE_SZ4MB_4U | _PAGE_SZ512K_4U |
                               _PAGE_SZ64K_4U | _PAGE_SZ8K_4U |
                               _PAGE_SZ32MB_4U | _PAGE_SZ256MB_4U);
@@ -2164,6 +2317,7 @@ static void __init sun4v_pgprot_init(void)
 {
         unsigned long page_none, page_shared, page_copy, page_readonly;
         unsigned long page_exec_bit;
+        int i;
 
         PAGE_KERNEL = __pgprot (_PAGE_PRESENT_4V | _PAGE_VALID |
                                 _PAGE_CACHE_4V | _PAGE_P_4V |
@@ -2176,8 +2330,7 @@ static void __init sun4v_pgprot_init(void)
         _PAGE_CACHE = _PAGE_CACHE_4V;
 
 #ifdef CONFIG_DEBUG_PAGEALLOC
-        kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZBITS_4V) ^
-                0xfffff80000000000UL;
+        kern_linear_pte_xor[0] = _PAGE_VALID ^ 0xfffff80000000000UL;
 #else
         kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4V) ^
                 0xfffff80000000000UL;
@@ -2185,20 +2338,12 @@ static void __init sun4v_pgprot_init(void)
         kern_linear_pte_xor[0] |= (_PAGE_CP_4V | _PAGE_CV_4V |
                                    _PAGE_P_4V | _PAGE_W_4V);
 
-#ifdef CONFIG_DEBUG_PAGEALLOC
-        kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZBITS_4V) ^
-                0xfffff80000000000UL;
-#else
-        kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZ256MB_4V) ^
-                0xfffff80000000000UL;
-#endif
-        kern_linear_pte_xor[1] |= (_PAGE_CP_4V | _PAGE_CV_4V |
-                                   _PAGE_P_4V | _PAGE_W_4V);
+        for (i = 1; i < 4; i++)
+                kern_linear_pte_xor[i] = kern_linear_pte_xor[0];
 
         pg_iobits = (_PAGE_VALID | _PAGE_PRESENT_4V | __DIRTY_BITS_4V |
                      __ACCESS_BITS_4V | _PAGE_E_4V);
 
-        _PAGE_SZBITS = _PAGE_SZBITS_4V;
         _PAGE_ALL_SZ_BITS = (_PAGE_SZ16GB_4V | _PAGE_SZ2GB_4V |
                              _PAGE_SZ256MB_4V | _PAGE_SZ32MB_4V |
                              _PAGE_SZ4MB_4V | _PAGE_SZ512K_4V |
@@ -2331,3 +2476,281 @@ void __flush_tlb_all(void)
         __asm__ __volatile__("wrpr      %0, 0, %%pstate"
                              : : "r" (pstate));
 }
+
+static pte_t *get_from_cache(struct mm_struct *mm)
+{
+        struct page *page;
+        pte_t *ret;
+
+        spin_lock(&mm->page_table_lock);
+        page = mm->context.pgtable_page;
+        ret = NULL;
+        if (page) {
+                void *p = page_address(page);
+
+                mm->context.pgtable_page = NULL;
+
+                ret = (pte_t *) (p + (PAGE_SIZE / 2));
+        }
+        spin_unlock(&mm->page_table_lock);
+
+        return ret;
+}
+
+static struct page *__alloc_for_cache(struct mm_struct *mm)
+{
+        struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK |
+                                       __GFP_REPEAT | __GFP_ZERO);
+
+        if (page) {
+                spin_lock(&mm->page_table_lock);
+                if (!mm->context.pgtable_page) {
+                        atomic_set(&page->_count, 2);
+                        mm->context.pgtable_page = page;
+                }
+                spin_unlock(&mm->page_table_lock);
+        }
+        return page;
+}
+
+pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
+                            unsigned long address)
+{
+        struct page *page;
+        pte_t *pte;
+
+        pte = get_from_cache(mm);
+        if (pte)
+                return pte;
+
+        page = __alloc_for_cache(mm);
+        if (page)
+                pte = (pte_t *) page_address(page);
+
+        return pte;
+}
+
+pgtable_t pte_alloc_one(struct mm_struct *mm,
+                        unsigned long address)
+{
+        struct page *page;
+        pte_t *pte;
+
+        pte = get_from_cache(mm);
+        if (pte)
+                return pte;
+
+        page = __alloc_for_cache(mm);
+        if (page) {
+                pgtable_page_ctor(page);
+                pte = (pte_t *) page_address(page);
+        }
+
+        return pte;
+}
+
+void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
+{
+        struct page *page = virt_to_page(pte);
+        if (put_page_testzero(page))
+                free_hot_cold_page(page, 0);
+}
+
+static void __pte_free(pgtable_t pte)
+{
+        struct page *page = virt_to_page(pte);
+        if (put_page_testzero(page)) {
+                pgtable_page_dtor(page);
+                free_hot_cold_page(page, 0);
+        }
+}
+
+void pte_free(struct mm_struct *mm, pgtable_t pte)
+{
+        __pte_free(pte);
+}
+
+void pgtable_free(void *table, bool is_page)
+{
+        if (is_page)
+                __pte_free(table);
+        else
+                kmem_cache_free(pgtable_cache, table);
+}
+
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+static pmd_t pmd_set_protbits(pmd_t pmd, pgprot_t pgprot, bool for_modify)
+{
+        if (pgprot_val(pgprot) & _PAGE_VALID)
+                pmd_val(pmd) |= PMD_HUGE_PRESENT;
+        if (tlb_type == hypervisor) {
+                if (pgprot_val(pgprot) & _PAGE_WRITE_4V)
+                        pmd_val(pmd) |= PMD_HUGE_WRITE;
+                if (pgprot_val(pgprot) & _PAGE_EXEC_4V)
+                        pmd_val(pmd) |= PMD_HUGE_EXEC;
+
+                if (!for_modify) {
+                        if (pgprot_val(pgprot) & _PAGE_ACCESSED_4V)
+                                pmd_val(pmd) |= PMD_HUGE_ACCESSED;
+                        if (pgprot_val(pgprot) & _PAGE_MODIFIED_4V)
+                                pmd_val(pmd) |= PMD_HUGE_DIRTY;
+                }
+        } else {
+                if (pgprot_val(pgprot) & _PAGE_WRITE_4U)
+                        pmd_val(pmd) |= PMD_HUGE_WRITE;
+                if (pgprot_val(pgprot) & _PAGE_EXEC_4U)
+                        pmd_val(pmd) |= PMD_HUGE_EXEC;
+
+                if (!for_modify) {
+                        if (pgprot_val(pgprot) & _PAGE_ACCESSED_4U)
+                                pmd_val(pmd) |= PMD_HUGE_ACCESSED;
+                        if (pgprot_val(pgprot) & _PAGE_MODIFIED_4U)
+                                pmd_val(pmd) |= PMD_HUGE_DIRTY;
+                }
+        }
+
+        return pmd;
+}
+
+pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot)
+{
+        pmd_t pmd;
+
+        pmd_val(pmd) = (page_nr << ((PAGE_SHIFT - PMD_PADDR_SHIFT)));
+        pmd_val(pmd) |= PMD_ISHUGE;
+        pmd = pmd_set_protbits(pmd, pgprot, false);
+        return pmd;
+}
+
+pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
+{
+        pmd_val(pmd) &= ~(PMD_HUGE_PRESENT |
+                          PMD_HUGE_WRITE |
+                          PMD_HUGE_EXEC);
+        pmd = pmd_set_protbits(pmd, newprot, true);
+        return pmd;
+}
+
+pgprot_t pmd_pgprot(pmd_t entry)
+{
+        unsigned long pte = 0;
+
+        if (pmd_val(entry) & PMD_HUGE_PRESENT)
+                pte |= _PAGE_VALID;
+
+        if (tlb_type == hypervisor) {
+                if (pmd_val(entry) & PMD_HUGE_PRESENT)
+                        pte |= _PAGE_PRESENT_4V;
+                if (pmd_val(entry) & PMD_HUGE_EXEC)
+                        pte |= _PAGE_EXEC_4V;
+                if (pmd_val(entry) & PMD_HUGE_WRITE)
+                        pte |= _PAGE_W_4V;
+                if (pmd_val(entry) & PMD_HUGE_ACCESSED)
+                        pte |= _PAGE_ACCESSED_4V;
+                if (pmd_val(entry) & PMD_HUGE_DIRTY)
+                        pte |= _PAGE_MODIFIED_4V;
+                pte |= _PAGE_CP_4V|_PAGE_CV_4V;
+        } else {
+                if (pmd_val(entry) & PMD_HUGE_PRESENT)
+                        pte |= _PAGE_PRESENT_4U;
+                if (pmd_val(entry) & PMD_HUGE_EXEC)
+                        pte |= _PAGE_EXEC_4U;
+                if (pmd_val(entry) & PMD_HUGE_WRITE)
+                        pte |= _PAGE_W_4U;
+                if (pmd_val(entry) & PMD_HUGE_ACCESSED)
+                        pte |= _PAGE_ACCESSED_4U;
+                if (pmd_val(entry) & PMD_HUGE_DIRTY)
+                        pte |= _PAGE_MODIFIED_4U;
+                pte |= _PAGE_CP_4U|_PAGE_CV_4U;
+        }
+
+        return __pgprot(pte);
+}
+
+void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
+                          pmd_t *pmd)
+{
+        unsigned long pte, flags;
+        struct mm_struct *mm;
+        pmd_t entry = *pmd;
+        pgprot_t prot;
+
+        if (!pmd_large(entry) || !pmd_young(entry))
+                return;
+
+        pte = (pmd_val(entry) & ~PMD_HUGE_PROTBITS);
+        pte <<= PMD_PADDR_SHIFT;
+        pte |= _PAGE_VALID;
+
+        prot = pmd_pgprot(entry);
+
+        if (tlb_type == hypervisor)
+                pgprot_val(prot) |= _PAGE_SZHUGE_4V;
+        else
+                pgprot_val(prot) |= _PAGE_SZHUGE_4U;
+
+        pte |= pgprot_val(prot);
+
+        mm = vma->vm_mm;
+
+        spin_lock_irqsave(&mm->context.lock, flags);
+
+        if (mm->context.tsb_block[MM_TSB_HUGE].tsb != NULL)
+                __update_mmu_tsb_insert(mm, MM_TSB_HUGE, HPAGE_SHIFT,
+                                        addr, pte);
+
+        spin_unlock_irqrestore(&mm->context.lock, flags);
+}
+#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
+
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
+static void context_reload(void *__data)
+{
+        struct mm_struct *mm = __data;
+
+        if (mm == current->mm)
+                load_secondary_context(mm);
+}
+
+void hugetlb_setup(struct mm_struct *mm)
+{
+        struct tsb_config *tp = &mm->context.tsb_block[MM_TSB_HUGE];
+
+        if (likely(tp->tsb != NULL))
+                return;
+
+        tsb_grow(mm, MM_TSB_HUGE, 0);
+        tsb_context_switch(mm);
+        smp_tsb_sync(mm);
+
+        /* On UltraSPARC-III+ and later, configure the second half of
+         * the Data-TLB for huge pages.
+         */
+        if (tlb_type == cheetah_plus) {
+                unsigned long ctx;
+
+                spin_lock(&ctx_alloc_lock);
+                ctx = mm->context.sparc64_ctx_val;
+                ctx &= ~CTX_PGSZ_MASK;
+                ctx |= CTX_PGSZ_BASE << CTX_PGSZ0_SHIFT;
+                ctx |= CTX_PGSZ_HUGE << CTX_PGSZ1_SHIFT;
+
+                if (ctx != mm->context.sparc64_ctx_val) {
+                        /* When changing the page size fields, we
+                         * must perform a context flush so that no
+                         * stale entries match.  This flush must
+                         * occur with the original context register
+                         * settings.
+                         */
+                        do_flush_tlb_mm(mm);
+
+                        /* Reload the context register of all processors
+                         * also executing in this address space.
+                         */
+                        mm->context.sparc64_ctx_val = ctx;
+                        on_each_cpu(context_reload, mm, 0);
+                }
+                spin_unlock(&ctx_alloc_lock);
+        }
+}
+#endif