9 files changed, 163 insertions, 214 deletions

diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c
index 2ebec263d685..69bb818fdd79 100644
--- a/arch/sparc/mm/fault_64.c
+++ b/arch/sparc/mm/fault_64.c
@@ -21,6 +21,7 @@
 #include <linux/kprobes.h>
 #include <linux/kdebug.h>
 #include <linux/percpu.h>
+#include <linux/context_tracking.h>
 
 #include <asm/page.h>
 #include <asm/pgtable.h>
@@ -272,6 +273,7 @@ static void noinline __kprobes bogus_32bit_fault_address(struct pt_regs *regs,
 
 asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs)
 {
+        enum ctx_state prev_state = exception_enter();
         struct mm_struct *mm = current->mm;
         struct vm_area_struct *vma;
         unsigned int insn = 0;
@@ -282,7 +284,7 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs)
         fault_code = get_thread_fault_code();
 
         if (notify_page_fault(regs))
-                return;
+                goto exit_exception;
 
         si_code = SEGV_MAPERR;
         address = current_thread_info()->fault_address;
@@ -313,7 +315,7 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs)
                         /* Valid, no problems... */
                 } else {
                         bad_kernel_pc(regs, address);
-                        return;
+                        goto exit_exception;
                 }
         } else
                 flags |= FAULT_FLAG_USER;
@@ -430,7 +432,7 @@ good_area:
         fault = handle_mm_fault(mm, vma, address, flags);
 
         if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
-                return;
+                goto exit_exception;
 
         if (unlikely(fault & VM_FAULT_ERROR)) {
                 if (fault & VM_FAULT_OOM)
@@ -482,6 +484,8 @@ good_area:
 
         }
 #endif
+exit_exception:
+        exception_exit(prev_state);
         return;
 
         /*
@@ -494,7 +498,7 @@ bad_area:
 
 handle_kernel_fault:
         do_kernel_fault(regs, si_code, fault_code, insn, address);
-        return;
+        goto exit_exception;
 
 /*
  * We ran out of memory, or some other thing happened to us that made
@@ -505,7 +509,7 @@ out_of_memory:
         up_read(&mm->mmap_sem);
         if (!(regs->tstate & TSTATE_PRIV)) {
                 pagefault_out_of_memory();
-                return;
+                goto exit_exception;
         }
         goto handle_kernel_fault;
 
diff --git a/arch/sparc/mm/gup.c b/arch/sparc/mm/gup.c
index 01ee23dd724d..c4d3da68b800 100644
--- a/arch/sparc/mm/gup.c
+++ b/arch/sparc/mm/gup.c
@@ -71,13 +71,12 @@ static int gup_huge_pmd(pmd_t *pmdp, pmd_t pmd, unsigned long addr,
                         int *nr)
 {
         struct page *head, *page, *tail;
-        u32 mask;
         int refs;
 
-        mask = PMD_HUGE_PRESENT;
-        if (write)
-                mask |= PMD_HUGE_WRITE;
-        if ((pmd_val(pmd) & mask) != mask)
+        if (!pmd_large(pmd))
+                return 0;
+
+        if (write && !pmd_write(pmd))
                 return 0;
 
         refs = 0;
diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c
index 96399646570a..30963178d7e9 100644
--- a/arch/sparc/mm/hugetlbpage.c
+++ b/arch/sparc/mm/hugetlbpage.c
@@ -21,8 +21,6 @@
 /* Slightly simplified from the non-hugepage variant because by
  * definition we don't have to worry about any page coloring stuff
  */
-#define VA_EXCLUDE_START (0x0000080000000000UL - (1UL << 32UL))
-#define VA_EXCLUDE_END   (0xfffff80000000000UL + (1UL << 32UL))
 
 static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *filp,
                                                         unsigned long addr,
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index ed82edad1a39..6b643790e4fe 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -354,7 +354,7 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *
 
 #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
         if (mm->context.huge_pte_count && is_hugetlb_pte(pte))
-                __update_mmu_tsb_insert(mm, MM_TSB_HUGE, HPAGE_SHIFT,
+                __update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT,
                                         address, pte_val(pte));
         else
 #endif
@@ -1557,6 +1557,96 @@ unsigned long __init find_ecache_flush_span(unsigned long size)
         return ~0UL;
 }
 
+unsigned long PAGE_OFFSET;
+EXPORT_SYMBOL(PAGE_OFFSET);
+
+static void __init page_offset_shift_patch_one(unsigned int *insn, unsigned long phys_bits)
+{
+        unsigned long final_shift;
+        unsigned int val = *insn;
+        unsigned int cnt;
+
+        /* We are patching in ilog2(max_supported_phys_address), and
+         * we are doing so in a manner similar to a relocation addend.
+         * That is, we are adding the shift value to whatever value
+         * is in the shift instruction count field already.
+         */
+        cnt = (val & 0x3f);
+        val &= ~0x3f;
+
+        /* If we are trying to shift >= 64 bits, clear the destination
+         * register.  This can happen when phys_bits ends up being equal
+         * to MAX_PHYS_ADDRESS_BITS.
+         */
+        final_shift = (cnt + (64 - phys_bits));
+        if (final_shift >= 64) {
+                unsigned int rd = (val >> 25) & 0x1f;
+
+                val = 0x80100000 | (rd << 25);
+        } else {
+                val |= final_shift;
+        }
+        *insn = val;
+
+        __asm__ __volatile__("flush     %0"
+                             : /* no outputs */
+                             : "r" (insn));
+}
+
+static void __init page_offset_shift_patch(unsigned long phys_bits)
+{
+        extern unsigned int __page_offset_shift_patch;
+        extern unsigned int __page_offset_shift_patch_end;
+        unsigned int *p;
+
+        p = &__page_offset_shift_patch;
+        while (p < &__page_offset_shift_patch_end) {
+                unsigned int *insn = (unsigned int *)(unsigned long)*p;
+
+                page_offset_shift_patch_one(insn, phys_bits);
+
+                p++;
+        }
+}
+
+static void __init setup_page_offset(void)
+{
+        unsigned long max_phys_bits = 40;
+
+        if (tlb_type == cheetah || tlb_type == cheetah_plus) {
+                max_phys_bits = 42;
+        } else if (tlb_type == hypervisor) {
+                switch (sun4v_chip_type) {
+                case SUN4V_CHIP_NIAGARA1:
+                case SUN4V_CHIP_NIAGARA2:
+                        max_phys_bits = 39;
+                        break;
+                case SUN4V_CHIP_NIAGARA3:
+                        max_phys_bits = 43;
+                        break;
+                case SUN4V_CHIP_NIAGARA4:
+                case SUN4V_CHIP_NIAGARA5:
+                case SUN4V_CHIP_SPARC64X:
+                default:
+                        max_phys_bits = 47;
+                        break;
+                }
+        }
+
+        if (max_phys_bits > MAX_PHYS_ADDRESS_BITS) {
+                prom_printf("MAX_PHYS_ADDRESS_BITS is too small, need %lu\n",
+                            max_phys_bits);
+                prom_halt();
+        }
+
+        PAGE_OFFSET = PAGE_OFFSET_BY_BITS(max_phys_bits);
+
+        pr_info("PAGE_OFFSET is 0x%016lx (max_phys_bits == %lu)\n",
+                PAGE_OFFSET, max_phys_bits);
+
+        page_offset_shift_patch(max_phys_bits);
+}
+
 static void __init tsb_phys_patch(void)
 {
         struct tsb_ldquad_phys_patch_entry *pquad;
@@ -1722,7 +1812,7 @@ static void __init sun4v_linear_pte_xor_finalize(void)
 #ifndef CONFIG_DEBUG_PAGEALLOC
         if (cpu_pgsz_mask & HV_PGSZ_MASK_256MB) {
                 kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZ256MB_4V) ^
-                        0xfffff80000000000UL;
+                        PAGE_OFFSET;
                 kern_linear_pte_xor[1] |= (_PAGE_CP_4V | _PAGE_CV_4V |
                                            _PAGE_P_4V | _PAGE_W_4V);
         } else {
@@ -1731,7 +1821,7 @@ static void __init sun4v_linear_pte_xor_finalize(void)
 
         if (cpu_pgsz_mask & HV_PGSZ_MASK_2GB) {
                 kern_linear_pte_xor[2] = (_PAGE_VALID | _PAGE_SZ2GB_4V) ^
-                        0xfffff80000000000UL;
+                        PAGE_OFFSET;
                 kern_linear_pte_xor[2] |= (_PAGE_CP_4V | _PAGE_CV_4V |
                                            _PAGE_P_4V | _PAGE_W_4V);
         } else {
@@ -1740,7 +1830,7 @@ static void __init sun4v_linear_pte_xor_finalize(void)
 
         if (cpu_pgsz_mask & HV_PGSZ_MASK_16GB) {
                 kern_linear_pte_xor[3] = (_PAGE_VALID | _PAGE_SZ16GB_4V) ^
-                        0xfffff80000000000UL;
+                        PAGE_OFFSET;
                 kern_linear_pte_xor[3] |= (_PAGE_CP_4V | _PAGE_CV_4V |
                                            _PAGE_P_4V | _PAGE_W_4V);
         } else {
@@ -1752,7 +1842,7 @@ static void __init sun4v_linear_pte_xor_finalize(void)
 /* paging_init() sets up the page tables */
 
 static unsigned long last_valid_pfn;
-pgd_t swapper_pg_dir[2048];
+pgd_t swapper_pg_dir[PTRS_PER_PGD];
 
 static void sun4u_pgprot_init(void);
 static void sun4v_pgprot_init(void);
@@ -1763,6 +1853,8 @@ void __init paging_init(void)
         unsigned long real_end, i;
         int node;
 
+        setup_page_offset();
+
         /* These build time checkes make sure that the dcache_dirty_cpu()
          * page->flags usage will work.
          *
@@ -2261,10 +2353,10 @@ static void __init sun4u_pgprot_init(void)
                      __ACCESS_BITS_4U | _PAGE_E_4U);
 
 #ifdef CONFIG_DEBUG_PAGEALLOC
-        kern_linear_pte_xor[0] = _PAGE_VALID ^ 0xfffff80000000000UL;
+        kern_linear_pte_xor[0] = _PAGE_VALID ^ PAGE_OFFSET;
 #else
         kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4U) ^
-                0xfffff80000000000UL;
+                PAGE_OFFSET;
 #endif
         kern_linear_pte_xor[0] |= (_PAGE_CP_4U | _PAGE_CV_4U |
                                    _PAGE_P_4U | _PAGE_W_4U);
@@ -2308,10 +2400,10 @@ static void __init sun4v_pgprot_init(void)
         _PAGE_CACHE = _PAGE_CACHE_4V;
 
 #ifdef CONFIG_DEBUG_PAGEALLOC
-        kern_linear_pte_xor[0] = _PAGE_VALID ^ 0xfffff80000000000UL;
+        kern_linear_pte_xor[0] = _PAGE_VALID ^ PAGE_OFFSET;
 #else
         kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4V) ^
-                0xfffff80000000000UL;
+                PAGE_OFFSET;
 #endif
         kern_linear_pte_xor[0] |= (_PAGE_CP_4V | _PAGE_CV_4V |
                                    _PAGE_P_4V | _PAGE_W_4V);
@@ -2455,53 +2547,13 @@ void __flush_tlb_all(void)
                              : : "r" (pstate));
 }
 
-static pte_t *get_from_cache(struct mm_struct *mm)
-{
-        struct page *page;
-        pte_t *ret;
-
-        spin_lock(&mm->page_table_lock);
-        page = mm->context.pgtable_page;
-        ret = NULL;
-        if (page) {
-                void *p = page_address(page);
-
-                mm->context.pgtable_page = NULL;
-
-                ret = (pte_t *) (p + (PAGE_SIZE / 2));
-        }
-        spin_unlock(&mm->page_table_lock);
-
-        return ret;
-}
-
-static struct page *__alloc_for_cache(struct mm_struct *mm)
-{
-        struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK |
-                                       __GFP_REPEAT | __GFP_ZERO);
-
-        if (page) {
-                spin_lock(&mm->page_table_lock);
-                if (!mm->context.pgtable_page) {
-                        atomic_set(&page->_count, 2);
-                        mm->context.pgtable_page = page;
-                }
-                spin_unlock(&mm->page_table_lock);
-        }
-        return page;
-}
-
 pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
                             unsigned long address)
 {
-        struct page *page;
-        pte_t *pte;
-
-        pte = get_from_cache(mm);
-        if (pte)
-                return pte;
+        struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK |
+                                       __GFP_REPEAT | __GFP_ZERO);
+        pte_t *pte = NULL;
 
-        page = __alloc_for_cache(mm);
         if (page)
                 pte = (pte_t *) page_address(page);
 
@@ -2511,36 +2563,30 @@ pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
 pgtable_t pte_alloc_one(struct mm_struct *mm,
                         unsigned long address)
 {
-        struct page *page;
-        pte_t *pte;
-
-        pte = get_from_cache(mm);
-        if (pte)
-                return pte;
+        struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK |
+                                       __GFP_REPEAT | __GFP_ZERO);
+        pte_t *pte = NULL;
 
-        page = __alloc_for_cache(mm);
-        if (page) {
-                pgtable_page_ctor(page);
-                pte = (pte_t *) page_address(page);
+        if (!page)
+                return NULL;
+        if (!pgtable_page_ctor(page)) {
+                free_hot_cold_page(page, 0);
+                return NULL;
         }
-
-        return pte;
+        return (pte_t *) page_address(page);
 }
 
 void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
 {
-        struct page *page = virt_to_page(pte);
-        if (put_page_testzero(page))
-                free_hot_cold_page(page, 0);
+        free_page((unsigned long)pte);
 }
 
 static void __pte_free(pgtable_t pte)
 {
         struct page *page = virt_to_page(pte);
-        if (put_page_testzero(page)) {
-                pgtable_page_dtor(page);
-                free_hot_cold_page(page, 0);
-        }
+
+        pgtable_page_dtor(page);
+        __free_page(page);
 }
 
 void pte_free(struct mm_struct *mm, pgtable_t pte)
@@ -2557,124 +2603,27 @@ void pgtable_free(void *table, bool is_page)
 }
 
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
-static pmd_t pmd_set_protbits(pmd_t pmd, pgprot_t pgprot, bool for_modify)
-{
-        if (pgprot_val(pgprot) & _PAGE_VALID)
-                pmd_val(pmd) |= PMD_HUGE_PRESENT;
-        if (tlb_type == hypervisor) {
-                if (pgprot_val(pgprot) & _PAGE_WRITE_4V)
-                        pmd_val(pmd) |= PMD_HUGE_WRITE;
-                if (pgprot_val(pgprot) & _PAGE_EXEC_4V)
-                        pmd_val(pmd) |= PMD_HUGE_EXEC;
-
-                if (!for_modify) {
-                        if (pgprot_val(pgprot) & _PAGE_ACCESSED_4V)
-                                pmd_val(pmd) |= PMD_HUGE_ACCESSED;
-                        if (pgprot_val(pgprot) & _PAGE_MODIFIED_4V)
-                                pmd_val(pmd) |= PMD_HUGE_DIRTY;
-                }
-        } else {
-                if (pgprot_val(pgprot) & _PAGE_WRITE_4U)
-                        pmd_val(pmd) |= PMD_HUGE_WRITE;
-                if (pgprot_val(pgprot) & _PAGE_EXEC_4U)
-                        pmd_val(pmd) |= PMD_HUGE_EXEC;
-
-                if (!for_modify) {
-                        if (pgprot_val(pgprot) & _PAGE_ACCESSED_4U)
-                                pmd_val(pmd) |= PMD_HUGE_ACCESSED;
-                        if (pgprot_val(pgprot) & _PAGE_MODIFIED_4U)
-                                pmd_val(pmd) |= PMD_HUGE_DIRTY;
-                }
-        }
-
-        return pmd;
-}
-
-pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot)
-{
-        pmd_t pmd;
-
-        pmd_val(pmd) = (page_nr << ((PAGE_SHIFT - PMD_PADDR_SHIFT)));
-        pmd_val(pmd) |= PMD_ISHUGE;
-        pmd = pmd_set_protbits(pmd, pgprot, false);
-        return pmd;
-}
-
-pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
-{
-        pmd_val(pmd) &= ~(PMD_HUGE_PRESENT |
-                          PMD_HUGE_WRITE |
-                          PMD_HUGE_EXEC);
-        pmd = pmd_set_protbits(pmd, newprot, true);
-        return pmd;
-}
-
-pgprot_t pmd_pgprot(pmd_t entry)
-{
-        unsigned long pte = 0;
-
-        if (pmd_val(entry) & PMD_HUGE_PRESENT)
-                pte |= _PAGE_VALID;
-
-        if (tlb_type == hypervisor) {
-                if (pmd_val(entry) & PMD_HUGE_PRESENT)
-                        pte |= _PAGE_PRESENT_4V;
-                if (pmd_val(entry) & PMD_HUGE_EXEC)
-                        pte |= _PAGE_EXEC_4V;
-                if (pmd_val(entry) & PMD_HUGE_WRITE)
-                        pte |= _PAGE_W_4V;
-                if (pmd_val(entry) & PMD_HUGE_ACCESSED)
-                        pte |= _PAGE_ACCESSED_4V;
-                if (pmd_val(entry) & PMD_HUGE_DIRTY)
-                        pte |= _PAGE_MODIFIED_4V;
-                pte |= _PAGE_CP_4V|_PAGE_CV_4V;
-        } else {
-                if (pmd_val(entry) & PMD_HUGE_PRESENT)
-                        pte |= _PAGE_PRESENT_4U;
-                if (pmd_val(entry) & PMD_HUGE_EXEC)
-                        pte |= _PAGE_EXEC_4U;
-                if (pmd_val(entry) & PMD_HUGE_WRITE)
-                        pte |= _PAGE_W_4U;
-                if (pmd_val(entry) & PMD_HUGE_ACCESSED)
-                        pte |= _PAGE_ACCESSED_4U;
-                if (pmd_val(entry) & PMD_HUGE_DIRTY)
-                        pte |= _PAGE_MODIFIED_4U;
-                pte |= _PAGE_CP_4U|_PAGE_CV_4U;
-        }
-
-        return __pgprot(pte);
-}
-
 void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
                           pmd_t *pmd)
 {
         unsigned long pte, flags;
         struct mm_struct *mm;
         pmd_t entry = *pmd;
-        pgprot_t prot;
 
         if (!pmd_large(entry) || !pmd_young(entry))
                 return;
 
-        pte = (pmd_val(entry) & ~PMD_HUGE_PROTBITS);
-        pte <<= PMD_PADDR_SHIFT;
-        pte |= _PAGE_VALID;
-
-        prot = pmd_pgprot(entry);
-
-        if (tlb_type == hypervisor)
-                pgprot_val(prot) |= _PAGE_SZHUGE_4V;
-        else
-                pgprot_val(prot) |= _PAGE_SZHUGE_4U;
+        pte = pmd_val(entry);
 
-        pte |= pgprot_val(prot);
+        /* We are fabricating 8MB pages using 4MB real hw pages.  */
+        pte |= (addr & (1UL << REAL_HPAGE_SHIFT));
 
         mm = vma->vm_mm;
 
         spin_lock_irqsave(&mm->context.lock, flags);
 
         if (mm->context.tsb_block[MM_TSB_HUGE].tsb != NULL)
-                __update_mmu_tsb_insert(mm, MM_TSB_HUGE, HPAGE_SHIFT,
+                __update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT,
                                         addr, pte);
 
         spin_unlock_irqrestore(&mm->context.lock, flags);
diff --git a/arch/sparc/mm/init_64.h b/arch/sparc/mm/init_64.h
index 0661aa606dec..5d3782deb403 100644
--- a/arch/sparc/mm/init_64.h
+++ b/arch/sparc/mm/init_64.h
@@ -1,11 +1,13 @@
 #ifndef _SPARC64_MM_INIT_H
 #define _SPARC64_MM_INIT_H
 
+#include <asm/page.h>
+
 /* Most of the symbols in this file are defined in init.c and
  * marked non-static so that assembler code can get at them.
  */
 
-#define MAX_PHYS_ADDRESS        (1UL << 41UL)
+#define MAX_PHYS_ADDRESS        (1UL << MAX_PHYS_ADDRESS_BITS)
 #define KPTE_BITMAP_CHUNK_SZ            (256UL * 1024UL * 1024UL)
 #define KPTE_BITMAP_BYTES       \
         ((MAX_PHYS_ADDRESS / KPTE_BITMAP_CHUNK_SZ) / 4)
diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c
index 5d721df48a72..869023abe5a4 100644
--- a/arch/sparc/mm/srmmu.c
+++ b/arch/sparc/mm/srmmu.c
@@ -345,7 +345,10 @@ pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
         if ((pte = (unsigned long)pte_alloc_one_kernel(mm, address)) == 0)
                 return NULL;
         page = pfn_to_page(__nocache_pa(pte) >> PAGE_SHIFT);
-        pgtable_page_ctor(page);
+        if (!pgtable_page_ctor(page)) {
+                __free_page(page);
+                return NULL;
+        }
         return page;
 }
 
diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c
index 7a91f288c708..ad3bf4b4324d 100644
--- a/arch/sparc/mm/tlb.c
+++ b/arch/sparc/mm/tlb.c
@@ -161,8 +161,8 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr,
         if (mm == &init_mm)
                 return;
 
-        if ((pmd_val(pmd) ^ pmd_val(orig)) & PMD_ISHUGE) {
-                if (pmd_val(pmd) & PMD_ISHUGE)
+        if ((pmd_val(pmd) ^ pmd_val(orig)) & _PAGE_PMD_HUGE) {
+                if (pmd_val(pmd) & _PAGE_PMD_HUGE)
                         mm->context.huge_pte_count++;
                 else
                         mm->context.huge_pte_count--;
@@ -178,13 +178,16 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr,
         }
 
         if (!pmd_none(orig)) {
-                bool exec = ((pmd_val(orig) & PMD_HUGE_EXEC) != 0);
+                pte_t orig_pte = __pte(pmd_val(orig));
+                bool exec = pte_exec(orig_pte);
 
                 addr &= HPAGE_MASK;
-                if (pmd_val(orig) & PMD_ISHUGE)
+                if (pmd_trans_huge(orig)) {
                         tlb_batch_add_one(mm, addr, exec);
-                else
+                        tlb_batch_add_one(mm, addr + REAL_HPAGE_SIZE, exec);
+                } else {
                         tlb_batch_pmd_scan(mm, addr, orig, exec);
+                }
         }
 }
 
@@ -196,11 +199,11 @@ void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
         assert_spin_locked(&mm->page_table_lock);
 
         /* FIFO */
-        if (!mm->pmd_huge_pte)
+        if (!pmd_huge_pte(mm, pmdp))
                 INIT_LIST_HEAD(lh);
         else
-                list_add(lh, (struct list_head *) mm->pmd_huge_pte);
-        mm->pmd_huge_pte = pgtable;
+                list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
+        pmd_huge_pte(mm, pmdp) = pgtable;
 }
 
 pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
@@ -211,12 +214,12 @@ pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
         assert_spin_locked(&mm->page_table_lock);
 
         /* FIFO */
-        pgtable = mm->pmd_huge_pte;
+        pgtable = pmd_huge_pte(mm, pmdp);
         lh = (struct list_head *) pgtable;
         if (list_empty(lh))
-                mm->pmd_huge_pte = NULL;
+                pmd_huge_pte(mm, pmdp) = NULL;
         else {
-                mm->pmd_huge_pte = (pgtable_t) lh->next;
+                pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
                 list_del(lh);
         }
         pte_val(pgtable[0]) = 0;
diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c
index 2cc3bce5ee91..3b3a360b429a 100644
--- a/arch/sparc/mm/tsb.c
+++ b/arch/sparc/mm/tsb.c
@@ -87,7 +87,7 @@ void flush_tsb_user(struct tlb_batch *tb)
                 nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries;
                 if (tlb_type == cheetah_plus || tlb_type == hypervisor)
                         base = __pa(base);
-                __flush_tsb_one(tb, HPAGE_SHIFT, base, nentries);
+                __flush_tsb_one(tb, REAL_HPAGE_SHIFT, base, nentries);
         }
 #endif
         spin_unlock_irqrestore(&mm->context.lock, flags);
@@ -111,7 +111,7 @@ void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr)
                 nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries;
                 if (tlb_type == cheetah_plus || tlb_type == hypervisor)
                         base = __pa(base);
-                __flush_tsb_one_entry(base, vaddr, HPAGE_SHIFT, nentries);
+                __flush_tsb_one_entry(base, vaddr, REAL_HPAGE_SHIFT, nentries);
         }
 #endif
         spin_unlock_irqrestore(&mm->context.lock, flags);
@@ -472,8 +472,6 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
         mm->context.huge_pte_count = 0;
 #endif
 
-        mm->context.pgtable_page = NULL;
-
         /* copy_mm() copies over the parent's mm_struct before calling
          * us, so we need to zero out the TSB pointer or else tsb_grow()
          * will be confused and think there is an older TSB to free up.
@@ -512,17 +510,10 @@ static void tsb_destroy_one(struct tsb_config *tp)
 void destroy_context(struct mm_struct *mm)
 {
         unsigned long flags, i;
-        struct page *page;
 
         for (i = 0; i < MM_NUM_TSBS; i++)
                 tsb_destroy_one(&mm->context.tsb_block[i]);
 
-        page = mm->context.pgtable_page;
-        if (page && put_page_testzero(page)) {
-                pgtable_page_dtor(page);
-                free_hot_cold_page(page, 0);
-        }
-
         spin_lock_irqsave(&ctx_alloc_lock, flags);
 
         if (CTX_VALID(mm->context)) {
diff --git a/arch/sparc/mm/ultra.S b/arch/sparc/mm/ultra.S
index 432aa0cb1b38..b4f4733abc6e 100644
--- a/arch/sparc/mm/ultra.S
+++ b/arch/sparc/mm/ultra.S
@@ -153,10 +153,10 @@ __spitfire_flush_tlb_mm_slow:
         .globl          __flush_icache_page
 __flush_icache_page:    /* %o0 = phys_page */
         srlx            %o0, PAGE_SHIFT, %o0
-        sethi           %uhi(PAGE_OFFSET), %g1
+        sethi           %hi(PAGE_OFFSET), %g1
         sllx            %o0, PAGE_SHIFT, %o0
         sethi           %hi(PAGE_SIZE), %g2
-        sllx            %g1, 32, %g1
+        ldx             [%g1 + %lo(PAGE_OFFSET)], %g1
         add             %o0, %g1, %o0
 1:      subcc           %g2, 32, %g2
         bne,pt          %icc, 1b
@@ -178,8 +178,8 @@ __flush_icache_page:	/* %o0 = phys_page */
         .align          64
         .globl          __flush_dcache_page
 __flush_dcache_page:    /* %o0=kaddr, %o1=flush_icache */
-        sethi           %uhi(PAGE_OFFSET), %g1
-        sllx            %g1, 32, %g1
+        sethi           %hi(PAGE_OFFSET), %g1
+        ldx             [%g1 + %lo(PAGE_OFFSET)], %g1
         sub             %o0, %g1, %o0                   ! physical address
         srlx            %o0, 11, %o0                    ! make D-cache TAG
         sethi           %hi(1 << 14), %o2               ! D-cache size
@@ -287,8 +287,8 @@ __cheetah_flush_tlb_pending:	/* 27 insns */
 
 #ifdef DCACHE_ALIASING_POSSIBLE
 __cheetah_flush_dcache_page: /* 11 insns */
-        sethi           %uhi(PAGE_OFFSET), %g1
-        sllx            %g1, 32, %g1
+        sethi           %hi(PAGE_OFFSET), %g1
+        ldx             [%g1 + %lo(PAGE_OFFSET)], %g1
         sub             %o0, %g1, %o0
         sethi           %hi(PAGE_SIZE), %o4
 1:      subcc           %o4, (1 << 5), %o4