11 files changed, 51 insertions, 27 deletions

diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index 5edd1742cfd0..7361e47db79f 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -703,7 +703,10 @@ static inline void arch_leave_lazy_mmu_mode(void)
         PVOP_VCALL0(pv_mmu_ops.lazy_mode.leave);
 }
 
-void arch_flush_lazy_mmu_mode(void);
+static inline void arch_flush_lazy_mmu_mode(void)
+{
+        PVOP_VCALL0(pv_mmu_ops.lazy_mode.flush);
+}
 
 static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx,
                                 phys_addr_t phys, pgprot_t flags)
diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h
index 142236ed83af..b3b0ec1dac86 100644
--- a/arch/x86/include/asm/paravirt_types.h
+++ b/arch/x86/include/asm/paravirt_types.h
@@ -91,6 +91,7 @@ struct pv_lazy_ops {
         /* Set deferred update mode, used for batching operations. */
         void (*enter)(void);
         void (*leave)(void);
+        void (*flush)(void);
 };
 
 struct pv_time_ops {
@@ -679,6 +680,7 @@ void paravirt_end_context_switch(struct task_struct *next);
 
 void paravirt_enter_lazy_mmu(void);
 void paravirt_leave_lazy_mmu(void);
+void paravirt_flush_lazy_mmu(void);
 
 void _paravirt_nop(void);
 u32 _paravirt_ident_32(u32);
diff --git a/arch/x86/include/asm/tlb.h b/arch/x86/include/asm/tlb.h
index 4fef20773b8f..c7797307fc2b 100644
--- a/arch/x86/include/asm/tlb.h
+++ b/arch/x86/include/asm/tlb.h
@@ -7,7 +7,7 @@
 
 #define tlb_flush(tlb)                                                  \
 {                                                                       \
-        if (tlb->fullmm == 0)                                           \
+        if (!tlb->fullmm && !tlb->need_flush_all)                       \
                 flush_tlb_mm_range(tlb->mm, tlb->start, tlb->end, 0UL); \
         else                                                            \
                 flush_tlb_mm_range(tlb->mm, 0UL, TLB_FLUSH_ALL, 0UL);   \
diff --git a/arch/x86/kernel/cpu/perf_event_intel_ds.c b/arch/x86/kernel/cpu/perf_event_intel_ds.c
index b05a575d56f4..26830f3af0df 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_ds.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_ds.c
@@ -314,10 +314,11 @@ int intel_pmu_drain_bts_buffer(void)
         if (top <= at)
                 return 0;
 
+        memset(&regs, 0, sizeof(regs));
+
         ds->bts_index = ds->bts_buffer_base;
 
         perf_sample_data_init(&data, 0, event->hw.last_period);
-        regs.ip     = 0;
 
         /*
          * Prepare a generic sample, i.e. fill in the invariant fields.
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index 17fff18a1031..8bfb335f74bb 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -263,6 +263,18 @@ void paravirt_leave_lazy_mmu(void)
         leave_lazy(PARAVIRT_LAZY_MMU);
 }
 
+void paravirt_flush_lazy_mmu(void)
+{
+        preempt_disable();
+
+        if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU) {
+                arch_leave_lazy_mmu_mode();
+                arch_enter_lazy_mmu_mode();
+        }
+
+        preempt_enable();
+}
+
 void paravirt_start_context_switch(struct task_struct *prev)
 {
         BUG_ON(preemptible());
@@ -292,18 +304,6 @@ enum paravirt_lazy_mode paravirt_get_lazy_mode(void)
         return this_cpu_read(paravirt_lazy_mode);
 }
 
-void arch_flush_lazy_mmu_mode(void)
-{
-        preempt_disable();
-
-        if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU) {
-                arch_leave_lazy_mmu_mode();
-                arch_enter_lazy_mmu_mode();
-        }
-
-        preempt_enable();
-}
-
 struct pv_info pv_info = {
         .name = "bare hardware",
         .paravirt_enabled = 0,
@@ -475,6 +475,7 @@ struct pv_mmu_ops pv_mmu_ops = {
         .lazy_mode = {
                 .enter = paravirt_nop,
                 .leave = paravirt_nop,
+                .flush = paravirt_nop,
         },
 
         .set_fixmap = native_set_fixmap,
diff --git a/arch/x86/lguest/boot.c b/arch/x86/lguest/boot.c
index 1cbd89ca5569..7114c63f047d 100644
--- a/arch/x86/lguest/boot.c
+++ b/arch/x86/lguest/boot.c
@@ -1334,6 +1334,7 @@ __init void lguest_init(void)
         pv_mmu_ops.read_cr3 = lguest_read_cr3;
         pv_mmu_ops.lazy_mode.enter = paravirt_enter_lazy_mmu;
         pv_mmu_ops.lazy_mode.leave = lguest_leave_lazy_mmu_mode;
+        pv_mmu_ops.lazy_mode.flush = paravirt_flush_lazy_mmu;
         pv_mmu_ops.pte_update = lguest_pte_update;
         pv_mmu_ops.pte_update_defer = lguest_pte_update;
 
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index 2b97525246d4..0e883364abb5 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -378,10 +378,12 @@ static noinline __kprobes int vmalloc_fault(unsigned long address)
         if (pgd_none(*pgd_ref))
                 return -1;
 
-        if (pgd_none(*pgd))
+        if (pgd_none(*pgd)) {
                 set_pgd(pgd, *pgd_ref);
-        else
+                arch_flush_lazy_mmu_mode();
+        } else {
                 BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));
+        }
 
         /*
          * Below here mismatches are bugs because these lower tables
diff --git a/arch/x86/mm/pageattr-test.c b/arch/x86/mm/pageattr-test.c
index b0086567271c..0e38951e65eb 100644
--- a/arch/x86/mm/pageattr-test.c
+++ b/arch/x86/mm/pageattr-test.c
@@ -68,7 +68,7 @@ static int print_split(struct split_state *s)
                         s->gpg++;
                         i += GPS/PAGE_SIZE;
                 } else if (level == PG_LEVEL_2M) {
-                        if (!(pte_val(*pte) & _PAGE_PSE)) {
+                        if ((pte_val(*pte) & _PAGE_PRESENT) && !(pte_val(*pte) & _PAGE_PSE)) {
                                 printk(KERN_ERR
                                         "%lx level %d but not PSE %Lx\n",
                                         addr, level, (u64)pte_val(*pte));
diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c
index 091934e1d0d9..fb4e73ec24d8 100644
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -467,7 +467,7 @@ try_preserve_large_page(pte_t *kpte, unsigned long address,
          * We are safe now. Check whether the new pgprot is the same:
          */
         old_pte = *kpte;
-        old_prot = new_prot = req_prot = pte_pgprot(old_pte);
+        old_prot = req_prot = pte_pgprot(old_pte);
 
         pgprot_val(req_prot) &= ~pgprot_val(cpa->mask_clr);
         pgprot_val(req_prot) |= pgprot_val(cpa->mask_set);
@@ -478,12 +478,12 @@ try_preserve_large_page(pte_t *kpte, unsigned long address,
          * a non present pmd. The canon_pgprot will clear _PAGE_GLOBAL
          * for the ancient hardware that doesn't support it.
          */
-        if (pgprot_val(new_prot) & _PAGE_PRESENT)
-                pgprot_val(new_prot) |= _PAGE_PSE | _PAGE_GLOBAL;
+        if (pgprot_val(req_prot) & _PAGE_PRESENT)
+                pgprot_val(req_prot) |= _PAGE_PSE | _PAGE_GLOBAL;
         else
-                pgprot_val(new_prot) &= ~(_PAGE_PSE | _PAGE_GLOBAL);
+                pgprot_val(req_prot) &= ~(_PAGE_PSE | _PAGE_GLOBAL);
 
-        new_prot = canon_pgprot(new_prot);
+        req_prot = canon_pgprot(req_prot);
 
         /*
          * old_pte points to the large page base address. So we need
@@ -1413,6 +1413,8 @@ void kernel_map_pages(struct page *page, int numpages, int enable)
          * but that can deadlock->flush only current cpu:
          */
         __flush_tlb_all();
+
+        arch_flush_lazy_mmu_mode();
 }
 
 #ifdef CONFIG_HIBERNATION
diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index 193350b51f90..17fda6a8b3c2 100644
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -58,6 +58,13 @@ void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte)
 void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd)
 {
         paravirt_release_pmd(__pa(pmd) >> PAGE_SHIFT);
+        /*
+         * NOTE! For PAE, any changes to the top page-directory-pointer-table
+         * entries need a full cr3 reload to flush.
+         */
+#ifdef CONFIG_X86_PAE
+        tlb->need_flush_all = 1;
+#endif
         tlb_remove_page(tlb, virt_to_page(pmd));
 }
 
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 6afbb2ca9a0a..e006c18d288a 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -1748,14 +1748,18 @@ static void *m2v(phys_addr_t maddr)
 }
 
 /* Set the page permissions on an identity-mapped pages */
-static void set_page_prot(void *addr, pgprot_t prot)
+static void set_page_prot_flags(void *addr, pgprot_t prot, unsigned long flags)
 {
         unsigned long pfn = __pa(addr) >> PAGE_SHIFT;
         pte_t pte = pfn_pte(pfn, prot);
 
-        if (HYPERVISOR_update_va_mapping((unsigned long)addr, pte, 0))
+        if (HYPERVISOR_update_va_mapping((unsigned long)addr, pte, flags))
                 BUG();
 }
+static void set_page_prot(void *addr, pgprot_t prot)
+{
+        return set_page_prot_flags(addr, prot, UVMF_NONE);
+}
 #ifdef CONFIG_X86_32
 static void __init xen_map_identity_early(pmd_t *pmd, unsigned long max_pfn)
 {
@@ -1839,12 +1843,12 @@ static void __init check_pt_base(unsigned long *pt_base, unsigned long *pt_end,
                                  unsigned long addr)
 {
         if (*pt_base == PFN_DOWN(__pa(addr))) {
-                set_page_prot((void *)addr, PAGE_KERNEL);
+                set_page_prot_flags((void *)addr, PAGE_KERNEL, UVMF_INVLPG);
                 clear_page((void *)addr);
                 (*pt_base)++;
         }
         if (*pt_end == PFN_DOWN(__pa(addr))) {
-                set_page_prot((void *)addr, PAGE_KERNEL);
+                set_page_prot_flags((void *)addr, PAGE_KERNEL, UVMF_INVLPG);
                 clear_page((void *)addr);
                 (*pt_end)--;
         }
@@ -2196,6 +2200,7 @@ static const struct pv_mmu_ops xen_mmu_ops __initconst = {
         .lazy_mode = {
                 .enter = paravirt_enter_lazy_mmu,
                 .leave = xen_leave_lazy_mmu,
+                .flush = paravirt_flush_lazy_mmu,
         },
 
         .set_fixmap = xen_set_fixmap,