3 files changed, 67 insertions, 38 deletions

diff --git a/arch/sparc/include/asm/page_64.h b/arch/sparc/include/asm/page_64.h
index 2211a8036bfa..732ba178a289 100644
--- a/arch/sparc/include/asm/page_64.h
+++ b/arch/sparc/include/asm/page_64.h
@@ -128,9 +128,6 @@ extern unsigned long PAGE_OFFSET;
  */
 #define MAX_PHYS_ADDRESS_BITS   47
 
-/* These two shift counts are used when indexing sparc64_valid_addr_bitmap
- * and kpte_linear_bitmap.
- */
 #define ILOG2_4MB               22
 #define ILOG2_256MB             28
 
diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h
index 31ac919920a9..a305b22ab581 100644
--- a/arch/sparc/include/asm/pgtable_64.h
+++ b/arch/sparc/include/asm/pgtable_64.h
@@ -79,22 +79,7 @@
 
 #include <linux/sched.h>
 
-extern unsigned long sparc64_valid_addr_bitmap[];
-
-/* Needs to be defined here and not in linux/mm.h, as it is arch dependent */
-static inline bool __kern_addr_valid(unsigned long paddr)
-{
-        if ((paddr >> MAX_PHYS_ADDRESS_BITS) != 0UL)
-                return false;
-        return test_bit(paddr >> ILOG2_4MB, sparc64_valid_addr_bitmap);
-}
-
-static inline bool kern_addr_valid(unsigned long addr)
-{
-        unsigned long paddr = __pa(addr);
-
-        return __kern_addr_valid(paddr);
-}
+bool kern_addr_valid(unsigned long addr);
 
 /* Entries per page directory level. */
 #define PTRS_PER_PTE    (1UL << (PAGE_SHIFT-3))
@@ -122,6 +107,7 @@ static inline bool kern_addr_valid(unsigned long addr)
 #define _PAGE_R           _AC(0x8000000000000000,UL) /* Keep ref bit uptodate*/
 #define _PAGE_SPECIAL     _AC(0x0200000000000000,UL) /* Special page         */
 #define _PAGE_PMD_HUGE    _AC(0x0100000000000000,UL) /* Huge page            */
+#define _PAGE_PUD_HUGE    _PAGE_PMD_HUGE
 
 /* Advertise support for _PAGE_SPECIAL */
 #define __HAVE_ARCH_PTE_SPECIAL
@@ -668,26 +654,26 @@ static inline unsigned long pmd_large(pmd_t pmd)
         return pte_val(pte) & _PAGE_PMD_HUGE;
 }
 
-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
-static inline unsigned long pmd_young(pmd_t pmd)
+static inline unsigned long pmd_pfn(pmd_t pmd)
 {
         pte_t pte = __pte(pmd_val(pmd));
 
-        return pte_young(pte);
+        return pte_pfn(pte);
 }
 
-static inline unsigned long pmd_write(pmd_t pmd)
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+static inline unsigned long pmd_young(pmd_t pmd)
 {
         pte_t pte = __pte(pmd_val(pmd));
 
-        return pte_write(pte);
+        return pte_young(pte);
 }
 
-static inline unsigned long pmd_pfn(pmd_t pmd)
+static inline unsigned long pmd_write(pmd_t pmd)
 {
         pte_t pte = __pte(pmd_val(pmd));
 
-        return pte_pfn(pte);
+        return pte_write(pte);
 }
 
 static inline unsigned long pmd_trans_huge(pmd_t pmd)
@@ -781,18 +767,15 @@ static inline int pmd_present(pmd_t pmd)
  * the top bits outside of the range of any physical address size we
  * support are clear as well.  We also validate the physical itself.
  */
-#define pmd_bad(pmd)                    ((pmd_val(pmd) & ~PAGE_MASK) || \
-                                         !__kern_addr_valid(pmd_val(pmd)))
+#define pmd_bad(pmd)                    (pmd_val(pmd) & ~PAGE_MASK)
 
 #define pud_none(pud)                   (!pud_val(pud))
 
-#define pud_bad(pud)                    ((pud_val(pud) & ~PAGE_MASK) || \
-                                         !__kern_addr_valid(pud_val(pud)))
+#define pud_bad(pud)                    (pud_val(pud) & ~PAGE_MASK)
 
 #define pgd_none(pgd)                   (!pgd_val(pgd))
 
-#define pgd_bad(pgd)                    ((pgd_val(pgd) & ~PAGE_MASK) || \
-                                         !__kern_addr_valid(pgd_val(pgd)))
+#define pgd_bad(pgd)                    (pgd_val(pgd) & ~PAGE_MASK)
 
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 void set_pmd_at(struct mm_struct *mm, unsigned long addr,
@@ -835,6 +818,20 @@ static inline unsigned long __pmd_page(pmd_t pmd)
 #define pgd_present(pgd)                (pgd_val(pgd) != 0U)
 #define pgd_clear(pgdp)                 (pgd_val(*(pgd)) = 0UL)
 
+static inline unsigned long pud_large(pud_t pud)
+{
+        pte_t pte = __pte(pud_val(pud));
+
+        return pte_val(pte) & _PAGE_PMD_HUGE;
+}
+
+static inline unsigned long pud_pfn(pud_t pud)
+{
+        pte_t pte = __pte(pud_val(pud));
+
+        return pte_pfn(pte);
+}
+
 /* Same in both SUN4V and SUN4U.  */
 #define pte_none(pte)                   (!pte_val(pte))
 
diff --git a/arch/sparc/include/asm/tsb.h b/arch/sparc/include/asm/tsb.h
index a2f541905715..ecb49cfa3be9 100644
--- a/arch/sparc/include/asm/tsb.h
+++ b/arch/sparc/include/asm/tsb.h
@@ -133,9 +133,24 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
         sub     TSB, 0x8, TSB;   \
         TSB_STORE(TSB, TAG);
 
-        /* Do a kernel page table walk.  Leaves physical PTE pointer in
-         * REG1.  Jumps to FAIL_LABEL on early page table walk termination.
-         * VADDR will not be clobbered, but REG2 will.
+        /* Do a kernel page table walk.  Leaves valid PTE value in
+         * REG1.  Jumps to FAIL_LABEL on early page table walk
+         * termination.  VADDR will not be clobbered, but REG2 will.
+         *
+         * There are two masks we must apply to propagate bits from
+         * the virtual address into the PTE physical address field
+         * when dealing with huge pages.  This is because the page
+         * table boundaries do not match the huge page size(s) the
+         * hardware supports.
+         *
+         * In these cases we propagate the bits that are below the
+         * page table level where we saw the huge page mapping, but
+         * are still within the relevant physical bits for the huge
+         * page size in question.  So for PMD mappings (which fall on
+         * bit 23, for 8MB per PMD) we must propagate bit 22 for a
+         * 4MB huge page.  For huge PUDs (which fall on bit 33, for
+         * 8GB per PUD), we have to accomodate 256MB and 2GB huge
+         * pages.  So for those we propagate bits 32 to 28.
          */
 #define KERN_PGTABLE_WALK(VADDR, REG1, REG2, FAIL_LABEL)        \
         sethi           %hi(swapper_pg_dir), REG1; \
@@ -150,15 +165,35 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
         andn            REG2, 0x7, REG2; \
         ldxa            [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
         brz,pn          REG1, FAIL_LABEL; \
-         sllx           VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \
+        sethi           %uhi(_PAGE_PUD_HUGE), REG2; \
+        brz,pn          REG1, FAIL_LABEL; \
+         sllx           REG2, 32, REG2; \
+        andcc           REG1, REG2, %g0; \
+        sethi           %hi(0xf8000000), REG2; \
+        bne,pt          %xcc, 697f; \
+         sllx           REG2, 1, REG2; \
+        sllx            VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \
         srlx            REG2, 64 - PAGE_SHIFT, REG2; \
         andn            REG2, 0x7, REG2; \
         ldxa            [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
+        sethi           %uhi(_PAGE_PMD_HUGE), REG2; \
         brz,pn          REG1, FAIL_LABEL; \
-         sllx           VADDR, 64 - PMD_SHIFT, REG2; \
+         sllx           REG2, 32, REG2; \
+        andcc           REG1, REG2, %g0; \
+        be,pn           %xcc, 698f; \
+         sethi          %hi(0x400000), REG2; \
+697:    brgez,pn        REG1, FAIL_LABEL; \
+         andn           REG1, REG2, REG1; \
+        and             VADDR, REG2, REG2; \
+        ba,pt           %xcc, 699f; \
+         or             REG1, REG2, REG1; \
+698:    sllx            VADDR, 64 - PMD_SHIFT, REG2; \
         srlx            REG2, 64 - PAGE_SHIFT, REG2; \
         andn            REG2, 0x7, REG2; \
-        add             REG1, REG2, REG1;
+        ldxa            [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
+        brgez,pn        REG1, FAIL_LABEL; \
+         nop; \
+699:
 
         /* PMD has been loaded into REG1, interpret the value, seeing
          * if it is a HUGE PMD or a normal one.  If it is not valid