4 files changed, 15 insertions, 2 deletions

diff --git a/arch/x86/include/asm/tlb.h b/arch/x86/include/asm/tlb.h
index 4fef20773b8f..c7797307fc2b 100644
--- a/arch/x86/include/asm/tlb.h
+++ b/arch/x86/include/asm/tlb.h
@@ -7,7 +7,7 @@
 
 #define tlb_flush(tlb)                                                  \
 {                                                                       \
-        if (tlb->fullmm == 0)                                           \
+        if (!tlb->fullmm && !tlb->need_flush_all)                       \
                 flush_tlb_mm_range(tlb->mm, tlb->start, tlb->end, 0UL); \
         else                                                            \
                 flush_tlb_mm_range(tlb->mm, 0UL, TLB_FLUSH_ALL, 0UL);   \
diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index 193350b51f90..17fda6a8b3c2 100644
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -58,6 +58,13 @@ void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte)
 void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd)
 {
         paravirt_release_pmd(__pa(pmd) >> PAGE_SHIFT);
+        /*
+         * NOTE! For PAE, any changes to the top page-directory-pointer-table
+         * entries need a full cr3 reload to flush.
+         */
+#ifdef CONFIG_X86_PAE
+        tlb->need_flush_all = 1;
+#endif
         tlb_remove_page(tlb, virt_to_page(pmd));
 }
 
diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
index 25f01d0bc149..b1b1fa6ffffe 100644
--- a/include/asm-generic/tlb.h
+++ b/include/asm-generic/tlb.h
@@ -99,7 +99,12 @@ struct mmu_gather {
         unsigned int            need_flush : 1, /* Did free PTEs */
                                 fast_mode  : 1; /* No batching   */
 
-        unsigned int            fullmm;
+        /* we are in the middle of an operation to clear
+         * a full mm and can make some optimizations */
+        unsigned int            fullmm : 1,
+        /* we have performed an operation which
+         * requires a complete flush of the tlb */
+                                need_flush_all : 1;
 
         struct mmu_gather_batch *active;
         struct mmu_gather_batch local;
diff --git a/mm/memory.c b/mm/memory.c
index 494526ae024a..13cbc420fead 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -216,6 +216,7 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm)
         tlb->mm = mm;
 
         tlb->fullmm     = fullmm;
+        tlb->need_flush_all = 0;
         tlb->start      = -1UL;
         tlb->end        = 0;
         tlb->need_flush = 0;