sparc64: Switch to 4-level page tables.

This has become necessary with chips that support more than 43-bits
of physical addressing.

Based almost entirely upon a patch by Bob Picco.

Signed-off-by: David S. Miller <davem@davemloft.net>
Acked-by: Bob Picco <bob.picco@oracle.com>

6 files changed, 109 insertions, 10 deletions

diff --git a/arch/sparc/include/asm/page_64.h b/arch/sparc/include/asm/page_64.h
index bf109984a032..09ceb68e72b7 100644
--- a/arch/sparc/include/asm/page_64.h
+++ b/arch/sparc/include/asm/page_64.h
@@ -57,18 +57,21 @@ void copy_user_page(void *to, void *from, unsigned long vaddr, struct page *topa
 typedef struct { unsigned long pte; } pte_t;
 typedef struct { unsigned long iopte; } iopte_t;
 typedef struct { unsigned long pmd; } pmd_t;
+typedef struct { unsigned long pud; } pud_t;
 typedef struct { unsigned long pgd; } pgd_t;
 typedef struct { unsigned long pgprot; } pgprot_t;
 
 #define pte_val(x)      ((x).pte)
 #define iopte_val(x)    ((x).iopte)
 #define pmd_val(x)      ((x).pmd)
+#define pud_val(x)      ((x).pud)
 #define pgd_val(x)      ((x).pgd)
 #define pgprot_val(x)   ((x).pgprot)
 
 #define __pte(x)        ((pte_t) { (x) } )
 #define __iopte(x)      ((iopte_t) { (x) } )
 #define __pmd(x)        ((pmd_t) { (x) } )
+#define __pud(x)        ((pud_t) { (x) } )
 #define __pgd(x)        ((pgd_t) { (x) } )
 #define __pgprot(x)     ((pgprot_t) { (x) } )
 
@@ -77,18 +80,21 @@ typedef struct { unsigned long pgprot; } pgprot_t;
 typedef unsigned long pte_t;
 typedef unsigned long iopte_t;
 typedef unsigned long pmd_t;
+typedef unsigned long pud_t;
 typedef unsigned long pgd_t;
 typedef unsigned long pgprot_t;
 
 #define pte_val(x)      (x)
 #define iopte_val(x)    (x)
 #define pmd_val(x)      (x)
+#define pud_val(x)      (x)
 #define pgd_val(x)      (x)
 #define pgprot_val(x)   (x)
 
 #define __pte(x)        (x)
 #define __iopte(x)      (x)
 #define __pmd(x)        (x)
+#define __pud(x)        (x)
 #define __pgd(x)        (x)
 #define __pgprot(x)     (x)
 
diff --git a/arch/sparc/include/asm/pgalloc_64.h b/arch/sparc/include/asm/pgalloc_64.h
index 39a7ac49b00c..5e3187185b4a 100644
--- a/arch/sparc/include/asm/pgalloc_64.h
+++ b/arch/sparc/include/asm/pgalloc_64.h
@@ -15,6 +15,13 @@
 
 extern struct kmem_cache *pgtable_cache;
 
+static inline void __pgd_populate(pgd_t *pgd, pud_t *pud)
+{
+        pgd_set(pgd, pud);
+}
+
+#define pgd_populate(MM, PGD, PUD)      __pgd_populate(PGD, PUD)
+
 static inline pgd_t *pgd_alloc(struct mm_struct *mm)
 {
         return kmem_cache_alloc(pgtable_cache, GFP_KERNEL);
@@ -25,7 +32,23 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
         kmem_cache_free(pgtable_cache, pgd);
 }
 
-#define pud_populate(MM, PUD, PMD)      pud_set(PUD, PMD)
+static inline void __pud_populate(pud_t *pud, pmd_t *pmd)
+{
+        pud_set(pud, pmd);
+}
+
+#define pud_populate(MM, PUD, PMD)      __pud_populate(PUD, PMD)
+
+static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
+{
+        return kmem_cache_alloc(pgtable_cache,
+                                GFP_KERNEL|__GFP_REPEAT);
+}
+
+static inline void pud_free(struct mm_struct *mm, pud_t *pud)
+{
+        kmem_cache_free(pgtable_cache, pud);
+}
 
 static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
 {
@@ -91,4 +114,7 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pte_t *pte,
 #define __pmd_free_tlb(tlb, pmd, addr)                \
         pgtable_free_tlb(tlb, pmd, false)
 
+#define __pud_free_tlb(tlb, pud, addr)                \
+        pgtable_free_tlb(tlb, pud, false)
+
 #endif /* _SPARC64_PGALLOC_H */
diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h
index 3770bf5c6e1b..31ac919920a9 100644
--- a/arch/sparc/include/asm/pgtable_64.h
+++ b/arch/sparc/include/asm/pgtable_64.h
@@ -20,8 +20,6 @@
 #include <asm/page.h>
 #include <asm/processor.h>
 
-#include <asm-generic/pgtable-nopud.h>
-
 /* The kernel image occupies 0x4000000 to 0x6000000 (4MB --> 96MB).
  * The page copy blockops can use 0x6000000 to 0x8000000.
  * The 8K TSB is mapped in the 0x8000000 to 0x8400000 range.
@@ -55,13 +53,21 @@
 #define PMD_MASK        (~(PMD_SIZE-1))
 #define PMD_BITS        (PAGE_SHIFT - 3)
 
-/* PGDIR_SHIFT determines what a third-level page table entry can map */
-#define PGDIR_SHIFT     (PAGE_SHIFT + (PAGE_SHIFT-3) + PMD_BITS)
+/* PUD_SHIFT determines the size of the area a third-level page
+ * table can map
+ */
+#define PUD_SHIFT       (PMD_SHIFT + PMD_BITS)
+#define PUD_SIZE        (_AC(1,UL) << PUD_SHIFT)
+#define PUD_MASK        (~(PUD_SIZE-1))
+#define PUD_BITS        (PAGE_SHIFT - 3)
+
+/* PGDIR_SHIFT determines what a fourth-level page table entry can map */
+#define PGDIR_SHIFT     (PUD_SHIFT + PUD_BITS)
 #define PGDIR_SIZE      (_AC(1,UL) << PGDIR_SHIFT)
 #define PGDIR_MASK      (~(PGDIR_SIZE-1))
 #define PGDIR_BITS      (PAGE_SHIFT - 3)
 
-#if (PGDIR_SHIFT + PGDIR_BITS) != 43
+#if (PGDIR_SHIFT + PGDIR_BITS) != 53
 #error Page table parameters do not cover virtual address space properly.
 #endif
 
@@ -93,6 +99,7 @@ static inline bool kern_addr_valid(unsigned long addr)
 /* Entries per page directory level. */
 #define PTRS_PER_PTE    (1UL << (PAGE_SHIFT-3))
 #define PTRS_PER_PMD    (1UL << PMD_BITS)
+#define PTRS_PER_PUD    (1UL << PUD_BITS)
 #define PTRS_PER_PGD    (1UL << PGDIR_BITS)
 
 /* Kernel has a separate 44bit address space. */
@@ -101,6 +108,9 @@ static inline bool kern_addr_valid(unsigned long addr)
 #define pmd_ERROR(e)                                                    \
         pr_err("%s:%d: bad pmd %p(%016lx) seen at (%pS)\n",             \
                __FILE__, __LINE__, &(e), pmd_val(e), __builtin_return_address(0))
+#define pud_ERROR(e)                                                    \
+        pr_err("%s:%d: bad pud %p(%016lx) seen at (%pS)\n",             \
+               __FILE__, __LINE__, &(e), pud_val(e), __builtin_return_address(0))
 #define pgd_ERROR(e)                                                    \
         pr_err("%s:%d: bad pgd %p(%016lx) seen at (%pS)\n",             \
                __FILE__, __LINE__, &(e), pgd_val(e), __builtin_return_address(0))
@@ -779,6 +789,11 @@ static inline int pmd_present(pmd_t pmd)
 #define pud_bad(pud)                    ((pud_val(pud) & ~PAGE_MASK) || \
                                          !__kern_addr_valid(pud_val(pud)))
 
+#define pgd_none(pgd)                   (!pgd_val(pgd))
+
+#define pgd_bad(pgd)                    ((pgd_val(pgd) & ~PAGE_MASK) || \
+                                         !__kern_addr_valid(pgd_val(pgd)))
+
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 void set_pmd_at(struct mm_struct *mm, unsigned long addr,
                 pmd_t *pmdp, pmd_t pmd);
@@ -815,10 +830,17 @@ static inline unsigned long __pmd_page(pmd_t pmd)
 #define pmd_clear(pmdp)                 (pmd_val(*(pmdp)) = 0UL)
 #define pud_present(pud)                (pud_val(pud) != 0U)
 #define pud_clear(pudp)                 (pud_val(*(pudp)) = 0UL)
+#define pgd_page_vaddr(pgd)             \
+        ((unsigned long) __va(pgd_val(pgd)))
+#define pgd_present(pgd)                (pgd_val(pgd) != 0U)
+#define pgd_clear(pgdp)                 (pgd_val(*(pgd)) = 0UL)
 
 /* Same in both SUN4V and SUN4U.  */
 #define pte_none(pte)                   (!pte_val(pte))
 
+#define pgd_set(pgdp, pudp)     \
+        (pgd_val(*(pgdp)) = (__pa((unsigned long) (pudp))))
+
 /* to find an entry in a page-table-directory. */
 #define pgd_index(address)      (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))
 #define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address))
@@ -826,6 +848,11 @@ static inline unsigned long __pmd_page(pmd_t pmd)
 /* to find an entry in a kernel page-table-directory */
 #define pgd_offset_k(address) pgd_offset(&init_mm, address)
 
+/* Find an entry in the third-level page table.. */
+#define pud_index(address)      (((address) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
+#define pud_offset(pgdp, address)       \
+        ((pud_t *) pgd_page_vaddr(*(pgdp)) + pud_index(address))
+
 /* Find an entry in the second-level page table.. */
 #define pmd_offset(pudp, address)       \
         ((pmd_t *) pud_page_vaddr(*(pudp)) + \
diff --git a/arch/sparc/include/asm/tsb.h b/arch/sparc/include/asm/tsb.h
index 90916f955cac..2e268b646348 100644
--- a/arch/sparc/include/asm/tsb.h
+++ b/arch/sparc/include/asm/tsb.h
@@ -145,6 +145,11 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
         andn            REG2, 0x7, REG2; \
         ldx             [REG1 + REG2], REG1; \
         brz,pn          REG1, FAIL_LABEL; \
+         sllx           VADDR, 64 - (PUD_SHIFT + PUD_BITS), REG2; \
+        srlx            REG2, 64 - PAGE_SHIFT, REG2; \
+        andn            REG2, 0x7, REG2; \
+        ldxa            [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
+        brz,pn          REG1, FAIL_LABEL; \
          sllx           VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \
         srlx            REG2, 64 - PAGE_SHIFT, REG2; \
         andn            REG2, 0x7, REG2; \
@@ -198,6 +203,11 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
         andn            REG2, 0x7, REG2; \
         ldxa            [PHYS_PGD + REG2] ASI_PHYS_USE_EC, REG1; \
         brz,pn          REG1, FAIL_LABEL; \
+         sllx           VADDR, 64 - (PUD_SHIFT + PUD_BITS), REG2; \
+        srlx            REG2, 64 - PAGE_SHIFT, REG2; \
+        andn            REG2, 0x7, REG2; \
+        ldxa            [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
+        brz,pn          REG1, FAIL_LABEL; \
          sllx           VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \
         srlx            REG2, 64 - PAGE_SHIFT, REG2; \
         andn            REG2, 0x7, REG2; \
diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c
index f7ba87543e5f..c9300bfaee5a 100644
--- a/arch/sparc/kernel/smp_64.c
+++ b/arch/sparc/kernel/smp_64.c
@@ -1467,6 +1467,13 @@ static void __init pcpu_populate_pte(unsigned long addr)
         pud_t *pud;
         pmd_t *pmd;
 
+        if (pgd_none(*pgd)) {
+                pud_t *new;
+
+                new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
+                pgd_populate(&init_mm, pgd, new);
+        }
+
         pud = pud_offset(pgd, addr);
         if (pud_none(*pud)) {
                 pmd_t *new;
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index bd08ed49bbaa..091f846e6192 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -1390,6 +1390,13 @@ static unsigned long __ref kernel_map_range(unsigned long pstart,
                 pmd_t *pmd;
                 pte_t *pte;
 
+                if (pgd_none(*pgd)) {
+                        pud_t *new;
+
+                        new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
+                        alloc_bytes += PAGE_SIZE;
+                        pgd_populate(&init_mm, pgd, new);
+                }
                 pud = pud_offset(pgd, vstart);
                 if (pud_none(*pud)) {
                         pmd_t *new;
@@ -1856,7 +1863,12 @@ static void __init sun4v_linear_pte_xor_finalize(void)
 /* paging_init() sets up the page tables */
 
 static unsigned long last_valid_pfn;
-pgd_t swapper_pg_dir[PTRS_PER_PGD];
+
+/* These must be page aligned in order to not trigger the
+ * alignment tests of pgd_bad() and pud_bad().
+ */
+pgd_t swapper_pg_dir[PTRS_PER_PGD] __attribute__ ((aligned (PAGE_SIZE)));
+static pud_t swapper_pud_dir[PTRS_PER_PUD] __attribute__ ((aligned (PAGE_SIZE)));
 
 static void sun4u_pgprot_init(void);
 static void sun4v_pgprot_init(void);
@@ -1911,6 +1923,8 @@ void __init paging_init(void)
 {
         unsigned long end_pfn, shift, phys_base;
         unsigned long real_end, i;
+        pud_t *pud;
+        pmd_t *pmd;
         int node;
 
         setup_page_offset();
@@ -2008,9 +2022,18 @@ void __init paging_init(void)
         
         memset(swapper_low_pmd_dir, 0, sizeof(swapper_low_pmd_dir));
 
-        /* Now can init the kernel/bad page tables. */
-        pud_set(pud_offset(&swapper_pg_dir[0], 0),
-                swapper_low_pmd_dir + (shift / sizeof(pgd_t)));
+        /* The kernel page tables we publish into what the rest of the
+         * world sees must be adjusted so that they see the PAGE_OFFSET
+         * address of these in-kerenel data structures.  However right
+         * here we must access them from the kernel image side, because
+         * the trap tables haven't been taken over and therefore we cannot
+         * take TLB misses in the PAGE_OFFSET linear mappings yet.
+         */
+        pud = swapper_pud_dir + (shift / sizeof(pud_t));
+        pgd_set(&swapper_pg_dir[0], pud);
+
+        pmd = swapper_low_pmd_dir + (shift / sizeof(pmd_t));
+        pud_set(&swapper_pud_dir[0], pmd);
         
         inherit_prom_mappings();