ARM: LPAE: Page table maintenance for the 3-level format

This patch modifies the pgd/pmd/pte manipulation functions to support
the 3-level page table format. Since there is no need for an 'ext'
argument to cpu_set_pte_ext(), this patch conditionally defines a
different prototype for this function when CONFIG_ARM_LPAE.

The patch also introduces the L_PGD_SWAPPER flag to mark pgd entries
pointing to pmd tables pre-allocated in the swapper_pg_dir and avoid
trying to free them at run-time. This flag is 0 with the classic page
table format.

Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

2 files changed, 52 insertions, 7 deletions

diff --git a/arch/arm/mm/ioremap.c b/arch/arm/mm/ioremap.c
index c3fa40da3b75..d1f78bacb015 100644
--- a/arch/arm/mm/ioremap.c
+++ b/arch/arm/mm/ioremap.c
@@ -64,7 +64,7 @@ void __check_kvm_seq(struct mm_struct *mm)
         } while (seq != init_mm.context.kvm_seq);
 }
 
-#ifndef CONFIG_SMP
+#if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE)
 /*
  * Section support is unsafe on SMP - If you iounmap and ioremap a region,
  * the other CPUs will not see this change until their next context switch.
@@ -202,11 +202,13 @@ void __iomem * __arm_ioremap_pfn_caller(unsigned long pfn,
         unsigned long addr;
         struct vm_struct * area;
 
+#ifndef CONFIG_ARM_LPAE
         /*
          * High mappings must be supersection aligned
          */
         if (pfn >= 0x100000 && (__pfn_to_phys(pfn) & ~SUPERSECTION_MASK))
                 return NULL;
+#endif
 
         /*
          * Don't allow RAM to be mapped - this causes problems with ARMv6+
@@ -228,7 +230,7 @@ void __iomem * __arm_ioremap_pfn_caller(unsigned long pfn,
                 return NULL;
         addr = (unsigned long)area->addr;
 
-#ifndef CONFIG_SMP
+#if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE)
         if (DOMAIN_IO == 0 &&
             (((cpu_architecture() >= CPU_ARCH_ARMv6) && (get_cr() & CR_XP)) ||
                cpu_is_xsc3()) && pfn >= 0x100000 &&
@@ -320,7 +322,7 @@ __arm_ioremap_exec(unsigned long phys_addr, size_t size, bool cached)
 void __iounmap(volatile void __iomem *io_addr)
 {
         void *addr = (void *)(PAGE_MASK & (unsigned long)io_addr);
-#ifndef CONFIG_SMP
+#if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE)
         struct vm_struct **p, *tmp;
 
         /*
diff --git a/arch/arm/mm/pgd.c b/arch/arm/mm/pgd.c
index b2027c154b2a..a3e78ccabd65 100644
--- a/arch/arm/mm/pgd.c
+++ b/arch/arm/mm/pgd.c
@@ -10,6 +10,7 @@
 #include <linux/mm.h>
 #include <linux/gfp.h>
 #include <linux/highmem.h>
+#include <linux/slab.h>
 
 #include <asm/pgalloc.h>
 #include <asm/page.h>
@@ -17,6 +18,14 @@
 
 #include "mm.h"
 
+#ifdef CONFIG_ARM_LPAE
+#define __pgd_alloc()   kmalloc(PTRS_PER_PGD * sizeof(pgd_t), GFP_KERNEL)
+#define __pgd_free(pgd) kfree(pgd)
+#else
+#define __pgd_alloc()   (pgd_t *)__get_free_pages(GFP_KERNEL, 2)
+#define __pgd_free(pgd) free_pages((unsigned long)pgd, 2)
+#endif
+
 /*
  * need to get a 16k page for level 1
  */
@@ -27,7 +36,7 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
         pmd_t *new_pmd, *init_pmd;
         pte_t *new_pte, *init_pte;
 
-        new_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL, 2);
+        new_pgd = __pgd_alloc();
         if (!new_pgd)
                 goto no_pgd;
 
@@ -42,10 +51,25 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
 
         clean_dcache_area(new_pgd, PTRS_PER_PGD * sizeof(pgd_t));
 
+#ifdef CONFIG_ARM_LPAE
+        /*
+         * Allocate PMD table for modules and pkmap mappings.
+         */
+        new_pud = pud_alloc(mm, new_pgd + pgd_index(MODULES_VADDR),
+                            MODULES_VADDR);
+        if (!new_pud)
+                goto no_pud;
+
+        new_pmd = pmd_alloc(mm, new_pud, 0);
+        if (!new_pmd)
+                goto no_pmd;
+#endif
+
         if (!vectors_high()) {
                 /*
                  * On ARM, first page must always be allocated since it
-                 * contains the machine vectors.
+                 * contains the machine vectors. The vectors are always high
+                 * with LPAE.
                  */
                 new_pud = pud_alloc(mm, new_pgd, 0);
                 if (!new_pud)
@@ -74,7 +98,7 @@ no_pte:
 no_pmd:
         pud_free(mm, new_pud);
 no_pud:
-        free_pages((unsigned long)new_pgd, 2);
+        __pgd_free(new_pgd);
 no_pgd:
         return NULL;
 }
@@ -111,5 +135,24 @@ no_pud:
         pgd_clear(pgd);
         pud_free(mm, pud);
 no_pgd:
-        free_pages((unsigned long) pgd_base, 2);
+#ifdef CONFIG_ARM_LPAE
+        /*
+         * Free modules/pkmap or identity pmd tables.
+         */
+        for (pgd = pgd_base; pgd < pgd_base + PTRS_PER_PGD; pgd++) {
+                if (pgd_none_or_clear_bad(pgd))
+                        continue;
+                if (pgd_val(*pgd) & L_PGD_SWAPPER)
+                        continue;
+                pud = pud_offset(pgd, 0);
+                if (pud_none_or_clear_bad(pud))
+                        continue;
+                pmd = pmd_offset(pud, 0);
+                pud_clear(pud);
+                pmd_free(mm, pmd);
+                pgd_clear(pgd);
+                pud_free(mm, pud);
+        }
+#endif
+        __pgd_free(pgd_base);
 }