4 files changed, 19 insertions, 8 deletions

diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index 2b97525246d4..0e883364abb5 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -378,10 +378,12 @@ static noinline __kprobes int vmalloc_fault(unsigned long address)
         if (pgd_none(*pgd_ref))
                 return -1;
 
-        if (pgd_none(*pgd))
+        if (pgd_none(*pgd)) {
                 set_pgd(pgd, *pgd_ref);
-        else
+                arch_flush_lazy_mmu_mode();
+        } else {
                 BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));
+        }
 
         /*
          * Below here mismatches are bugs because these lower tables
diff --git a/arch/x86/mm/pageattr-test.c b/arch/x86/mm/pageattr-test.c
index b0086567271c..0e38951e65eb 100644
--- a/arch/x86/mm/pageattr-test.c
+++ b/arch/x86/mm/pageattr-test.c
@@ -68,7 +68,7 @@ static int print_split(struct split_state *s)
                         s->gpg++;
                         i += GPS/PAGE_SIZE;
                 } else if (level == PG_LEVEL_2M) {
-                        if (!(pte_val(*pte) & _PAGE_PSE)) {
+                        if ((pte_val(*pte) & _PAGE_PRESENT) && !(pte_val(*pte) & _PAGE_PSE)) {
                                 printk(KERN_ERR
                                         "%lx level %d but not PSE %Lx\n",
                                         addr, level, (u64)pte_val(*pte));
diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c
index 091934e1d0d9..fb4e73ec24d8 100644
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -467,7 +467,7 @@ try_preserve_large_page(pte_t *kpte, unsigned long address,
          * We are safe now. Check whether the new pgprot is the same:
          */
         old_pte = *kpte;
-        old_prot = new_prot = req_prot = pte_pgprot(old_pte);
+        old_prot = req_prot = pte_pgprot(old_pte);
 
         pgprot_val(req_prot) &= ~pgprot_val(cpa->mask_clr);
         pgprot_val(req_prot) |= pgprot_val(cpa->mask_set);
@@ -478,12 +478,12 @@ try_preserve_large_page(pte_t *kpte, unsigned long address,
          * a non present pmd. The canon_pgprot will clear _PAGE_GLOBAL
          * for the ancient hardware that doesn't support it.
          */
-        if (pgprot_val(new_prot) & _PAGE_PRESENT)
-                pgprot_val(new_prot) |= _PAGE_PSE | _PAGE_GLOBAL;
+        if (pgprot_val(req_prot) & _PAGE_PRESENT)
+                pgprot_val(req_prot) |= _PAGE_PSE | _PAGE_GLOBAL;
         else
-                pgprot_val(new_prot) &= ~(_PAGE_PSE | _PAGE_GLOBAL);
+                pgprot_val(req_prot) &= ~(_PAGE_PSE | _PAGE_GLOBAL);
 
-        new_prot = canon_pgprot(new_prot);
+        req_prot = canon_pgprot(req_prot);
 
         /*
          * old_pte points to the large page base address. So we need
@@ -1413,6 +1413,8 @@ void kernel_map_pages(struct page *page, int numpages, int enable)
          * but that can deadlock->flush only current cpu:
          */
         __flush_tlb_all();
+
+        arch_flush_lazy_mmu_mode();
 }
 
 #ifdef CONFIG_HIBERNATION
diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index 193350b51f90..17fda6a8b3c2 100644
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -58,6 +58,13 @@ void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte)
 void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd)
 {
         paravirt_release_pmd(__pa(pmd) >> PAGE_SHIFT);
+        /*
+         * NOTE! For PAE, any changes to the top page-directory-pointer-table
+         * entries need a full cr3 reload to flush.
+         */
+#ifdef CONFIG_X86_PAE
+        tlb->need_flush_all = 1;
+#endif
         tlb_remove_page(tlb, virt_to_page(pmd));
 }