mmu_notifier: call mmu_notifier_invalidate_range() from VMM

Add calls to the new mmu_notifier_invalidate_range() function to all
places in the VMM that need it.

Signed-off-by: Joerg Roedel <jroedel@suse.de>
Reviewed-by: Andrea Arcangeli <aarcange@redhat.com>
Reviewed-by: Jérôme Glisse <jglisse@redhat.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Rik van Riel <riel@redhat.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Johannes Weiner <jweiner@redhat.com>
Cc: Jay Cornwall <Jay.Cornwall@amd.com>
Cc: Oded Gabbay <Oded.Gabbay@amd.com>
Cc: Suravee Suthikulpanit <Suravee.Suthikulpanit@amd.com>
Cc: Jesse Barnes <jbarnes@virtuousgeek.org>
Cc: David Woodhouse <dwmw2@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Oded Gabbay <oded.gabbay@amd.com>

1 files changed, 41 insertions, 0 deletions

diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h
index 17907908d1df..966da2b4b803 100644
--- a/include/linux/mmu_notifier.h
+++ b/include/linux/mmu_notifier.h
@@ -284,6 +284,44 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
         __young;                                                        \
 })
 
+#define ptep_clear_flush_notify(__vma, __address, __ptep)               \
+({                                                                      \
+        unsigned long ___addr = __address & PAGE_MASK;                  \
+        struct mm_struct *___mm = (__vma)->vm_mm;                       \
+        pte_t ___pte;                                                   \
+                                                                        \
+        ___pte = ptep_clear_flush(__vma, __address, __ptep);            \
+        mmu_notifier_invalidate_range(___mm, ___addr,                   \
+                                        ___addr + PAGE_SIZE);           \
+                                                                        \
+        ___pte;                                                         \
+})
+
+#define pmdp_clear_flush_notify(__vma, __haddr, __pmd)                  \
+({                                                                      \
+        unsigned long ___haddr = __haddr & HPAGE_PMD_MASK;              \
+        struct mm_struct *___mm = (__vma)->vm_mm;                       \
+        pmd_t ___pmd;                                                   \
+                                                                        \
+        ___pmd = pmdp_clear_flush(__vma, __haddr, __pmd);               \
+        mmu_notifier_invalidate_range(___mm, ___haddr,                  \
+                                      ___haddr + HPAGE_PMD_SIZE);       \
+                                                                        \
+        ___pmd;                                                         \
+})
+
+#define pmdp_get_and_clear_notify(__mm, __haddr, __pmd)                 \
+({                                                                      \
+        unsigned long ___haddr = __haddr & HPAGE_PMD_MASK;              \
+        pmd_t ___pmd;                                                   \
+                                                                        \
+        ___pmd = pmdp_get_and_clear(__mm, __haddr, __pmd);              \
+        mmu_notifier_invalidate_range(__mm, ___haddr,                   \
+                                      ___haddr + HPAGE_PMD_SIZE);       \
+                                                                        \
+        ___pmd;                                                         \
+})
+
 /*
  * set_pte_at_notify() sets the pte _after_ running the notifier.
  * This is safe to start by updating the secondary MMUs, because the primary MMU
@@ -362,6 +400,9 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
 
 #define ptep_clear_flush_young_notify ptep_clear_flush_young
 #define pmdp_clear_flush_young_notify pmdp_clear_flush_young
+#define ptep_clear_flush_notify ptep_clear_flush
+#define pmdp_clear_flush_notify pmdp_clear_flush
+#define pmdp_get_and_clear_notify pmdp_get_and_clear
 #define set_pte_at_notify set_pte_at
 
 #endif /* CONFIG_MMU_NOTIFIER */