Merge git://git.kernel.org/pub/scm/linux/kernel/git/x86/linux-2.6-x86

* git://git.kernel.org/pub/scm/linux/kernel/git/x86/linux-2.6-x86: (78 commits)
  x86: fix RTC lockdep warning: potential hardirq recursion
  x86: cpa, micro-optimization
  x86: cpa, clean up code flow
  x86: cpa, eliminate CPA_ enum
  x86: cpa, cleanups
  x86: implement gbpages support in change_page_attr()
  x86: support gbpages in pagetable dump
  x86: add gbpages support to lookup_address
  x86: add pgtable accessor functions for gbpages
  x86: add PUD_PAGE_SIZE
  x86: add feature macros for the gbpages cpuid bit
  x86: switch direct mapping setup over to set_pte
  x86: fix page-present check in cpa_flush_range
  x86: remove cpa warning
  x86: remove now unused clear_kernel_mapping
  x86: switch pci-gart over to using set_memory_np() instead of clear_kernel_mapping()
  x86: cpa selftest, skip non present entries
  x86: CPA fix pagetable split
  x86: rename LARGE_PAGE_SIZE to PMD_PAGE_SIZE
  x86: cpa, fix lookup_address
  ...

1 files changed, 11 insertions, 15 deletions

diff --git a/include/asm-x86/pgtable-3level.h b/include/asm-x86/pgtable-3level.h
index a195c3e757b9..1d763eec740f 100644
--- a/include/asm-x86/pgtable-3level.h
+++ b/include/asm-x86/pgtable-3level.h
@@ -93,26 +93,22 @@ static inline void native_pmd_clear(pmd_t *pmd)
 
 static inline void pud_clear(pud_t *pudp)
 {
+        unsigned long pgd;
+
         set_pud(pudp, __pud(0));
 
         /*
-         * In principle we need to do a cr3 reload here to make sure
-         * the processor recognizes the changed pgd.  In practice, all
-         * the places where pud_clear() gets called are followed by
-         * full tlb flushes anyway, so we can defer the cost here.
-         *
-         * Specifically:
-         *
-         * mm/memory.c:free_pmd_range() - immediately after the
-         * pud_clear() it does a pmd_free_tlb().  We change the
-         * mmu_gather structure to do a full tlb flush (which has the
-         * effect of reloading cr3) when the pagetable free is
-         * complete.
+         * According to Intel App note "TLBs, Paging-Structure Caches,
+         * and Their Invalidation", April 2007, document 317080-001,
+         * section 8.1: in PAE mode we explicitly have to flush the
+         * TLB via cr3 if the top-level pgd is changed...
          *
-         * arch/x86/mm/hugetlbpage.c:huge_pmd_unshare() - the call to
-         * this is followed by a flush_tlb_range, which on x86 does a
-         * full tlb flush.
+         * Make sure the pud entry we're updating is within the
+         * current pgd to avoid unnecessary TLB flushes.
          */
+        pgd = read_cr3();
+        if (__pa(pudp) >= pgd && __pa(pudp) < (pgd + sizeof(pgd_t)*PTRS_PER_PGD))
+                write_cr3(pgd);
 }
 
 #define pud_page(pud) \