sparc64: Switch to 4-level page tables.

This has become necessary with chips that support more than 43-bits
of physical addressing.

Based almost entirely upon a patch by Bob Picco.

Signed-off-by: David S. Miller <davem@davemloft.net>
Acked-by: Bob Picco <bob.picco@oracle.com>

1 files changed, 27 insertions, 4 deletions

diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index bd08ed49bbaa..091f846e6192 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -1390,6 +1390,13 @@ static unsigned long __ref kernel_map_range(unsigned long pstart,
                 pmd_t *pmd;
                 pte_t *pte;
 
+                if (pgd_none(*pgd)) {
+                        pud_t *new;
+
+                        new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
+                        alloc_bytes += PAGE_SIZE;
+                        pgd_populate(&init_mm, pgd, new);
+                }
                 pud = pud_offset(pgd, vstart);
                 if (pud_none(*pud)) {
                         pmd_t *new;
@@ -1856,7 +1863,12 @@ static void __init sun4v_linear_pte_xor_finalize(void)
 /* paging_init() sets up the page tables */
 
 static unsigned long last_valid_pfn;
-pgd_t swapper_pg_dir[PTRS_PER_PGD];
+
+/* These must be page aligned in order to not trigger the
+ * alignment tests of pgd_bad() and pud_bad().
+ */
+pgd_t swapper_pg_dir[PTRS_PER_PGD] __attribute__ ((aligned (PAGE_SIZE)));
+static pud_t swapper_pud_dir[PTRS_PER_PUD] __attribute__ ((aligned (PAGE_SIZE)));
 
 static void sun4u_pgprot_init(void);
 static void sun4v_pgprot_init(void);
@@ -1911,6 +1923,8 @@ void __init paging_init(void)
 {
         unsigned long end_pfn, shift, phys_base;
         unsigned long real_end, i;
+        pud_t *pud;
+        pmd_t *pmd;
         int node;
 
         setup_page_offset();
@@ -2008,9 +2022,18 @@ void __init paging_init(void)
         
         memset(swapper_low_pmd_dir, 0, sizeof(swapper_low_pmd_dir));
 
-        /* Now can init the kernel/bad page tables. */
-        pud_set(pud_offset(&swapper_pg_dir[0], 0),
-                swapper_low_pmd_dir + (shift / sizeof(pgd_t)));
+        /* The kernel page tables we publish into what the rest of the
+         * world sees must be adjusted so that they see the PAGE_OFFSET
+         * address of these in-kerenel data structures.  However right
+         * here we must access them from the kernel image side, because
+         * the trap tables haven't been taken over and therefore we cannot
+         * take TLB misses in the PAGE_OFFSET linear mappings yet.
+         */
+        pud = swapper_pud_dir + (shift / sizeof(pud_t));
+        pgd_set(&swapper_pg_dir[0], pud);
+
+        pmd = swapper_low_pmd_dir + (shift / sizeof(pmd_t));
+        pud_set(&swapper_pud_dir[0], pmd);
         
         inherit_prom_mappings();