1 files changed, 31 insertions, 7 deletions

diff --git a/mm/memory.c b/mm/memory.c
index 262e3eb6601a..67f0ab9077d9 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -51,6 +51,7 @@
 #include <linux/init.h>
 #include <linux/writeback.h>
 #include <linux/memcontrol.h>
+#include <linux/mmu_notifier.h>
 
 #include <asm/pgalloc.h>
 #include <asm/uaccess.h>
@@ -374,7 +375,8 @@ static inline void add_mm_rss(struct mm_struct *mm, int file_rss, int anon_rss)
  *
  * The calling function must still handle the error.
  */
-void print_bad_pte(struct vm_area_struct *vma, pte_t pte, unsigned long vaddr)
+static void print_bad_pte(struct vm_area_struct *vma, pte_t pte,
+                          unsigned long vaddr)
 {
         printk(KERN_ERR "Bad pte = %08llx, process = %s, "
                         "vm_flags = %lx, vaddr = %lx\n",
@@ -651,6 +653,7 @@ int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
         unsigned long next;
         unsigned long addr = vma->vm_start;
         unsigned long end = vma->vm_end;
+        int ret;
 
         /*
          * Don't copy ptes where a page fault will fill them correctly.
@@ -666,17 +669,33 @@ int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
         if (is_vm_hugetlb_page(vma))
                 return copy_hugetlb_page_range(dst_mm, src_mm, vma);
 
+        /*
+         * We need to invalidate the secondary MMU mappings only when
+         * there could be a permission downgrade on the ptes of the
+         * parent mm. And a permission downgrade will only happen if
+         * is_cow_mapping() returns true.
+         */
+        if (is_cow_mapping(vma->vm_flags))
+                mmu_notifier_invalidate_range_start(src_mm, addr, end);
+
+        ret = 0;
         dst_pgd = pgd_offset(dst_mm, addr);
         src_pgd = pgd_offset(src_mm, addr);
         do {
                 next = pgd_addr_end(addr, end);
                 if (pgd_none_or_clear_bad(src_pgd))
                         continue;
-                if (copy_pud_range(dst_mm, src_mm, dst_pgd, src_pgd,
-                                                vma, addr, next))
-                        return -ENOMEM;
+                if (unlikely(copy_pud_range(dst_mm, src_mm, dst_pgd, src_pgd,
+                                            vma, addr, next))) {
+                        ret = -ENOMEM;
+                        break;
+                }
         } while (dst_pgd++, src_pgd++, addr = next, addr != end);
-        return 0;
+
+        if (is_cow_mapping(vma->vm_flags))
+                mmu_notifier_invalidate_range_end(src_mm,
+                                                  vma->vm_start, end);
+        return ret;
 }
 
 static unsigned long zap_pte_range(struct mmu_gather *tlb,
@@ -880,7 +899,9 @@ unsigned long unmap_vmas(struct mmu_gather **tlbp,
         unsigned long start = start_addr;
         spinlock_t *i_mmap_lock = details? details->i_mmap_lock: NULL;
         int fullmm = (*tlbp)->fullmm;
+        struct mm_struct *mm = vma->vm_mm;
 
+        mmu_notifier_invalidate_range_start(mm, start_addr, end_addr);
         for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next) {
                 unsigned long end;
 
@@ -945,6 +966,7 @@ unsigned long unmap_vmas(struct mmu_gather **tlbp,
                 }
         }
 out:
+        mmu_notifier_invalidate_range_end(mm, start_addr, end_addr);
         return start;   /* which is now the end (or restart) address */
 }
 
@@ -1615,10 +1637,11 @@ int apply_to_page_range(struct mm_struct *mm, unsigned long addr,
 {
         pgd_t *pgd;
         unsigned long next;
-        unsigned long end = addr + size;
+        unsigned long start = addr, end = addr + size;
         int err;
 
         BUG_ON(addr >= end);
+        mmu_notifier_invalidate_range_start(mm, start, end);
         pgd = pgd_offset(mm, addr);
         do {
                 next = pgd_addr_end(addr, end);
@@ -1626,6 +1649,7 @@ int apply_to_page_range(struct mm_struct *mm, unsigned long addr,
                 if (err)
                         break;
         } while (pgd++, addr = next, addr != end);
+        mmu_notifier_invalidate_range_end(mm, start, end);
         return err;
 }
 EXPORT_SYMBOL_GPL(apply_to_page_range);
@@ -1838,7 +1862,7 @@ gotten:
                  * seen in the presence of one thread doing SMC and another
                  * thread doing COW.
                  */
-                ptep_clear_flush(vma, address, page_table);
+                ptep_clear_flush_notify(vma, address, page_table);
                 set_pte_at(mm, address, page_table, entry);
                 update_mmu_cache(vma, address, entry);
                 lru_cache_add_active(new_page);