3 files changed, 61 insertions, 2 deletions

diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h
index b77486d152cd..4e02ee2b071e 100644
--- a/include/linux/mmu_notifier.h
+++ b/include/linux/mmu_notifier.h
@@ -62,6 +62,15 @@ struct mmu_notifier_ops {
                                  unsigned long address);
 
         /*
+         * change_pte is called in cases that pte mapping to page is changed:
+         * for example, when ksm remaps pte to point to a new shared page.
+         */
+        void (*change_pte)(struct mmu_notifier *mn,
+                           struct mm_struct *mm,
+                           unsigned long address,
+                           pte_t pte);
+
+        /*
          * Before this is invoked any secondary MMU is still ok to
          * read/write to the page previously pointed to by the Linux
          * pte because the page hasn't been freed yet and it won't be
@@ -154,6 +163,8 @@ extern void __mmu_notifier_mm_destroy(struct mm_struct *mm);
 extern void __mmu_notifier_release(struct mm_struct *mm);
 extern int __mmu_notifier_clear_flush_young(struct mm_struct *mm,
                                           unsigned long address);
+extern void __mmu_notifier_change_pte(struct mm_struct *mm,
+                                      unsigned long address, pte_t pte);
 extern void __mmu_notifier_invalidate_page(struct mm_struct *mm,
                                           unsigned long address);
 extern void __mmu_notifier_invalidate_range_start(struct mm_struct *mm,
@@ -175,6 +186,13 @@ static inline int mmu_notifier_clear_flush_young(struct mm_struct *mm,
         return 0;
 }
 
+static inline void mmu_notifier_change_pte(struct mm_struct *mm,
+                                           unsigned long address, pte_t pte)
+{
+        if (mm_has_notifiers(mm))
+                __mmu_notifier_change_pte(mm, address, pte);
+}
+
 static inline void mmu_notifier_invalidate_page(struct mm_struct *mm,
                                           unsigned long address)
 {
@@ -236,6 +254,16 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
         __young;                                                        \
 })
 
+#define set_pte_at_notify(__mm, __address, __ptep, __pte)               \
+({                                                                      \
+        struct mm_struct *___mm = __mm;                                 \
+        unsigned long ___address = __address;                           \
+        pte_t ___pte = __pte;                                           \
+                                                                        \
+        set_pte_at(___mm, ___address, __ptep, ___pte);                  \
+        mmu_notifier_change_pte(___mm, ___address, ___pte);             \
+})
+
 #else /* CONFIG_MMU_NOTIFIER */
 
 static inline void mmu_notifier_release(struct mm_struct *mm)
@@ -248,6 +276,11 @@ static inline int mmu_notifier_clear_flush_young(struct mm_struct *mm,
         return 0;
 }
 
+static inline void mmu_notifier_change_pte(struct mm_struct *mm,
+                                           unsigned long address, pte_t pte)
+{
+}
+
 static inline void mmu_notifier_invalidate_page(struct mm_struct *mm,
                                           unsigned long address)
 {
@@ -273,6 +306,7 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
 
 #define ptep_clear_flush_young_notify ptep_clear_flush_young
 #define ptep_clear_flush_notify ptep_clear_flush
+#define set_pte_at_notify set_pte_at
 
 #endif /* CONFIG_MMU_NOTIFIER */
 
diff --git a/mm/memory.c b/mm/memory.c
index e8f63d9961ea..368561f32009 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -2115,9 +2115,14 @@ gotten:
                  * seen in the presence of one thread doing SMC and another
                  * thread doing COW.
                  */
-                ptep_clear_flush_notify(vma, address, page_table);
+                ptep_clear_flush(vma, address, page_table);
                 page_add_new_anon_rmap(new_page, vma, address);
-                set_pte_at(mm, address, page_table, entry);
+                /*
+                 * We call the notify macro here because, when using secondary
+                 * mmu page tables (such as kvm shadow page tables), we want the
+                 * new page to be mapped directly into the secondary page table.
+                 */
+                set_pte_at_notify(mm, address, page_table, entry);
                 update_mmu_cache(vma, address, entry);
                 if (old_page) {
                         /*
diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c
index 5f4ef0250bee..7e33f2cb3c77 100644
--- a/mm/mmu_notifier.c
+++ b/mm/mmu_notifier.c
@@ -99,6 +99,26 @@ int __mmu_notifier_clear_flush_young(struct mm_struct *mm,
         return young;
 }
 
+void __mmu_notifier_change_pte(struct mm_struct *mm, unsigned long address,
+                               pte_t pte)
+{
+        struct mmu_notifier *mn;
+        struct hlist_node *n;
+
+        rcu_read_lock();
+        hlist_for_each_entry_rcu(mn, n, &mm->mmu_notifier_mm->list, hlist) {
+                if (mn->ops->change_pte)
+                        mn->ops->change_pte(mn, mm, address, pte);
+                /*
+                 * Some drivers don't have change_pte,
+                 * so we must call invalidate_page in that case.
+                 */
+                else if (mn->ops->invalidate_page)
+                        mn->ops->invalidate_page(mn, mm, address);
+        }
+        rcu_read_unlock();
+}
+
 void __mmu_notifier_invalidate_page(struct mm_struct *mm,
                                           unsigned long address)
 {