1 files changed, 123 insertions, 67 deletions

diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index 50159764f694..557b2abceef8 100644
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -2,6 +2,7 @@
 #include <asm/pgalloc.h>
 #include <asm/pgtable.h>
 #include <asm/tlb.h>
+#include <asm/fixmap.h>
 
 pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
 {
@@ -65,12 +66,6 @@ static inline void pgd_list_del(pgd_t *pgd)
 static void pgd_ctor(void *p)
 {
         pgd_t *pgd = p;
-        unsigned long flags;
-
-        /* Clear usermode parts of PGD */
-        memset(pgd, 0, KERNEL_PGD_BOUNDARY*sizeof(pgd_t));
-
-        spin_lock_irqsave(&pgd_lock, flags);
 
         /* If the pgd points to a shared pagetable level (either the
            ptes in non-PAE, or shared PMD in PAE), then just copy the
@@ -90,8 +85,6 @@ static void pgd_ctor(void *p)
         /* list required to sync kernel mapping updates */
         if (!SHARED_KERNEL_PMD)
                 pgd_list_add(pgd);
-
-        spin_unlock_irqrestore(&pgd_lock, flags);
 }
 
 static void pgd_dtor(void *pgd)
@@ -119,6 +112,72 @@ static void pgd_dtor(void *pgd)
 
 #ifdef CONFIG_X86_PAE
 /*
+ * In PAE mode, we need to do a cr3 reload (=tlb flush) when
+ * updating the top-level pagetable entries to guarantee the
+ * processor notices the update.  Since this is expensive, and
+ * all 4 top-level entries are used almost immediately in a
+ * new process's life, we just pre-populate them here.
+ *
+ * Also, if we're in a paravirt environment where the kernel pmd is
+ * not shared between pagetables (!SHARED_KERNEL_PMDS), we allocate
+ * and initialize the kernel pmds here.
+ */
+#define PREALLOCATED_PMDS       UNSHARED_PTRS_PER_PGD
+
+void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd)
+{
+        paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);
+
+        /* Note: almost everything apart from _PAGE_PRESENT is
+           reserved at the pmd (PDPT) level. */
+        set_pud(pudp, __pud(__pa(pmd) | _PAGE_PRESENT));
+
+        /*
+         * According to Intel App note "TLBs, Paging-Structure Caches,
+         * and Their Invalidation", April 2007, document 317080-001,
+         * section 8.1: in PAE mode we explicitly have to flush the
+         * TLB via cr3 if the top-level pgd is changed...
+         */
+        if (mm == current->active_mm)
+                write_cr3(read_cr3());
+}
+#else  /* !CONFIG_X86_PAE */
+
+/* No need to prepopulate any pagetable entries in non-PAE modes. */
+#define PREALLOCATED_PMDS       0
+
+#endif  /* CONFIG_X86_PAE */
+
+static void free_pmds(pmd_t *pmds[])
+{
+        int i;
+
+        for(i = 0; i < PREALLOCATED_PMDS; i++)
+                if (pmds[i])
+                        free_page((unsigned long)pmds[i]);
+}
+
+static int preallocate_pmds(pmd_t *pmds[])
+{
+        int i;
+        bool failed = false;
+
+        for(i = 0; i < PREALLOCATED_PMDS; i++) {
+                pmd_t *pmd = (pmd_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT);
+                if (pmd == NULL)
+                        failed = true;
+                pmds[i] = pmd;
+        }
+
+        if (failed) {
+                free_pmds(pmds);
+                return -ENOMEM;
+        }
+
+        return 0;
+}
+
+/*
  * Mop up any pmd pages which may still be attached to the pgd.
  * Normally they will be freed by munmap/exit_mmap, but any pmd we
  * preallocate which never got a corresponding vma will need to be
@@ -128,7 +187,7 @@ static void pgd_mop_up_pmds(struct mm_struct *mm, pgd_t *pgdp)
 {
         int i;
 
-        for(i = 0; i < UNSHARED_PTRS_PER_PGD; i++) {
+        for(i = 0; i < PREALLOCATED_PMDS; i++) {
                 pgd_t pgd = pgdp[i];
 
                 if (pgd_val(pgd) != 0) {
@@ -142,32 +201,17 @@ static void pgd_mop_up_pmds(struct mm_struct *mm, pgd_t *pgdp)
         }
 }
 
-/*
- * In PAE mode, we need to do a cr3 reload (=tlb flush) when
- * updating the top-level pagetable entries to guarantee the
- * processor notices the update.  Since this is expensive, and
- * all 4 top-level entries are used almost immediately in a
- * new process's life, we just pre-populate them here.
- *
- * Also, if we're in a paravirt environment where the kernel pmd is
- * not shared between pagetables (!SHARED_KERNEL_PMDS), we allocate
- * and initialize the kernel pmds here.
- */
-static int pgd_prepopulate_pmd(struct mm_struct *mm, pgd_t *pgd)
+static void pgd_prepopulate_pmd(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmds[])
 {
         pud_t *pud;
         unsigned long addr;
         int i;
 
         pud = pud_offset(pgd, 0);
-        for (addr = i = 0; i < UNSHARED_PTRS_PER_PGD;
-             i++, pud++, addr += PUD_SIZE) {
-                pmd_t *pmd = pmd_alloc_one(mm, addr);
 
-                if (!pmd) {
-                        pgd_mop_up_pmds(mm, pgd);
-                        return 0;
-                }
+        for (addr = i = 0; i < PREALLOCATED_PMDS;
+             i++, pud++, addr += PUD_SIZE) {
+                pmd_t *pmd = pmds[i];
 
                 if (i >= KERNEL_PGD_BOUNDARY)
                         memcpy(pmd, (pmd_t *)pgd_page_vaddr(swapper_pg_dir[i]),
@@ -175,61 +219,54 @@ static int pgd_prepopulate_pmd(struct mm_struct *mm, pgd_t *pgd)
 
                 pud_populate(mm, pud, pmd);
         }
-
-        return 1;
 }
 
-void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd)
+pgd_t *pgd_alloc(struct mm_struct *mm)
 {
-        paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);
+        pgd_t *pgd;
+        pmd_t *pmds[PREALLOCATED_PMDS];
+        unsigned long flags;
 
-        /* Note: almost everything apart from _PAGE_PRESENT is
-           reserved at the pmd (PDPT) level. */
-        set_pud(pudp, __pud(__pa(pmd) | _PAGE_PRESENT));
+        pgd = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
 
-        /*
-         * According to Intel App note "TLBs, Paging-Structure Caches,
-         * and Their Invalidation", April 2007, document 317080-001,
-         * section 8.1: in PAE mode we explicitly have to flush the
-         * TLB via cr3 if the top-level pgd is changed...
-         */
-        if (mm == current->active_mm)
-                write_cr3(read_cr3());
-}
-#else  /* !CONFIG_X86_PAE */
-/* No need to prepopulate any pagetable entries in non-PAE modes. */
-static int pgd_prepopulate_pmd(struct mm_struct *mm, pgd_t *pgd)
-{
-        return 1;
-}
+        if (pgd == NULL)
+                goto out;
 
-static void pgd_mop_up_pmds(struct mm_struct *mm, pgd_t *pgd)
-{
-}
-#endif  /* CONFIG_X86_PAE */
+        mm->pgd = pgd;
 
-pgd_t *pgd_alloc(struct mm_struct *mm)
-{
-        pgd_t *pgd = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
+        if (preallocate_pmds(pmds) != 0)
+                goto out_free_pgd;
 
-        /* so that alloc_pmd can use it */
-        mm->pgd = pgd;
-        if (pgd)
-                pgd_ctor(pgd);
+        if (paravirt_pgd_alloc(mm) != 0)
+                goto out_free_pmds;
 
-        if (pgd && !pgd_prepopulate_pmd(mm, pgd)) {
-                pgd_dtor(pgd);
-                free_page((unsigned long)pgd);
-                pgd = NULL;
-        }
+        /*
+         * Make sure that pre-populating the pmds is atomic with
+         * respect to anything walking the pgd_list, so that they
+         * never see a partially populated pgd.
+         */
+        spin_lock_irqsave(&pgd_lock, flags);
+
+        pgd_ctor(pgd);
+        pgd_prepopulate_pmd(mm, pgd, pmds);
+
+        spin_unlock_irqrestore(&pgd_lock, flags);
 
         return pgd;
+
+out_free_pmds:
+        free_pmds(pmds);
+out_free_pgd:
+        free_page((unsigned long)pgd);
+out:
+        return NULL;
 }
 
 void pgd_free(struct mm_struct *mm, pgd_t *pgd)
 {
         pgd_mop_up_pmds(mm, pgd);
         pgd_dtor(pgd);
+        paravirt_pgd_free(mm, pgd);
         free_page((unsigned long)pgd);
 }
 
@@ -255,7 +292,7 @@ int ptep_test_and_clear_young(struct vm_area_struct *vma,
 
         if (pte_young(*ptep))
                 ret = test_and_clear_bit(_PAGE_BIT_ACCESSED,
-                                         &ptep->pte);
+                                         (unsigned long *) &ptep->pte);
 
         if (ret)
                 pte_update(vma->vm_mm, addr, ptep);
@@ -274,3 +311,22 @@ int ptep_clear_flush_young(struct vm_area_struct *vma,
 
         return young;
 }
+
+int fixmaps_set;
+
+void __native_set_fixmap(enum fixed_addresses idx, pte_t pte)
+{
+        unsigned long address = __fix_to_virt(idx);
+
+        if (idx >= __end_of_fixed_addresses) {
+                BUG();
+                return;
+        }
+        set_pte_vaddr(address, pte);
+        fixmaps_set++;
+}
+
+void native_set_fixmap(enum fixed_addresses idx, unsigned long phys, pgprot_t flags)
+{
+        __native_set_fixmap(idx, pfn_pte(phys >> PAGE_SHIFT, flags));
+}