thp: add pmd mmu_notifier helpers

Add mmu notifier helpers to handle pmd huge operations.

Signed-off-by: Andrea Arcangeli <aarcange@redhat.com>
Acked-by: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 40 insertions, 0 deletions

diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h
index 43dcfbdc39de..cbfab1e9957d 100644
--- a/include/linux/mmu_notifier.h
+++ b/include/linux/mmu_notifier.h
@@ -243,6 +243,32 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
         __pte;                                                          \
 })
 
+#define pmdp_clear_flush_notify(__vma, __address, __pmdp)               \
+({                                                                      \
+        pmd_t __pmd;                                                    \
+        struct vm_area_struct *___vma = __vma;                          \
+        unsigned long ___address = __address;                           \
+        VM_BUG_ON(__address & ~HPAGE_PMD_MASK);                         \
+        mmu_notifier_invalidate_range_start(___vma->vm_mm, ___address,  \
+                                            (__address)+HPAGE_PMD_SIZE);\
+        __pmd = pmdp_clear_flush(___vma, ___address, __pmdp);           \
+        mmu_notifier_invalidate_range_end(___vma->vm_mm, ___address,    \
+                                          (__address)+HPAGE_PMD_SIZE);  \
+        __pmd;                                                          \
+})
+
+#define pmdp_splitting_flush_notify(__vma, __address, __pmdp)           \
+({                                                                      \
+        struct vm_area_struct *___vma = __vma;                          \
+        unsigned long ___address = __address;                           \
+        VM_BUG_ON(__address & ~HPAGE_PMD_MASK);                         \
+        mmu_notifier_invalidate_range_start(___vma->vm_mm, ___address,  \
+                                            (__address)+HPAGE_PMD_SIZE);\
+        pmdp_splitting_flush(___vma, ___address, __pmdp);               \
+        mmu_notifier_invalidate_range_end(___vma->vm_mm, ___address,    \
+                                          (__address)+HPAGE_PMD_SIZE);  \
+})
+
 #define ptep_clear_flush_young_notify(__vma, __address, __ptep)         \
 ({                                                                      \
         int __young;                                                    \
@@ -254,6 +280,17 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
         __young;                                                        \
 })
 
+#define pmdp_clear_flush_young_notify(__vma, __address, __pmdp)         \
+({                                                                      \
+        int __young;                                                    \
+        struct vm_area_struct *___vma = __vma;                          \
+        unsigned long ___address = __address;                           \
+        __young = pmdp_clear_flush_young(___vma, ___address, __pmdp);   \
+        __young |= mmu_notifier_clear_flush_young(___vma->vm_mm,        \
+                                                  ___address);          \
+        __young;                                                        \
+})
+
 #define set_pte_at_notify(__mm, __address, __ptep, __pte)               \
 ({                                                                      \
         struct mm_struct *___mm = __mm;                                 \
@@ -305,7 +342,10 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
 }
 
 #define ptep_clear_flush_young_notify ptep_clear_flush_young
+#define pmdp_clear_flush_young_notify pmdp_clear_flush_young
 #define ptep_clear_flush_notify ptep_clear_flush
+#define pmdp_clear_flush_notify pmdp_clear_flush
+#define pmdp_splitting_flush_notify pmdp_splitting_flush
 #define set_pte_at_notify set_pte_at
 
 #endif /* CONFIG_MMU_NOTIFIER */