mmu_notifier: add the callback for mmu_notifier_invalidate_range()

Now that the mmu_notifier_invalidate_range() calls are in place, add the
callback to allow subsystems to register against it.

Signed-off-by: Joerg Roedel <jroedel@suse.de>
Reviewed-by: Andrea Arcangeli <aarcange@redhat.com>
Reviewed-by: Jérôme Glisse <jglisse@redhat.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Rik van Riel <riel@redhat.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Johannes Weiner <jweiner@redhat.com>
Cc: Jay Cornwall <Jay.Cornwall@amd.com>
Cc: Oded Gabbay <Oded.Gabbay@amd.com>
Cc: Suravee Suthikulpanit <Suravee.Suthikulpanit@amd.com>
Cc: Jesse Barnes <jbarnes@virtuousgeek.org>
Cc: David Woodhouse <dwmw2@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Oded Gabbay <oded.gabbay@amd.com>

1 files changed, 25 insertions, 0 deletions

diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c
index 2c8da9825fe3..3b9b3d0741b2 100644
--- a/mm/mmu_notifier.c
+++ b/mm/mmu_notifier.c
@@ -193,6 +193,16 @@ void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,
 
         id = srcu_read_lock(&srcu);
         hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
+                /*
+                 * Call invalidate_range here too to avoid the need for the
+                 * subsystem of having to register an invalidate_range_end
+                 * call-back when there is invalidate_range already. Usually a
+                 * subsystem registers either invalidate_range_start()/end() or
+                 * invalidate_range(), so this will be no additional overhead
+                 * (besides the pointer check).
+                 */
+                if (mn->ops->invalidate_range)
+                        mn->ops->invalidate_range(mn, mm, start, end);
                 if (mn->ops->invalidate_range_end)
                         mn->ops->invalidate_range_end(mn, mm, start, end);
         }
@@ -200,6 +210,21 @@ void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,
 }
 EXPORT_SYMBOL_GPL(__mmu_notifier_invalidate_range_end);
 
+void __mmu_notifier_invalidate_range(struct mm_struct *mm,
+                                  unsigned long start, unsigned long end)
+{
+        struct mmu_notifier *mn;
+        int id;
+
+        id = srcu_read_lock(&srcu);
+        hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
+                if (mn->ops->invalidate_range)
+                        mn->ops->invalidate_range(mn, mm, start, end);
+        }
+        srcu_read_unlock(&srcu, id);
+}
+EXPORT_SYMBOL_GPL(__mmu_notifier_invalidate_range);
+
 static int do_mmu_notifier_register(struct mmu_notifier *mn,
                                     struct mm_struct *mm,
                                     int take_mmap_sem)