1 files changed, 39 insertions, 40 deletions

diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c
index be04122fb277..6725ff183374 100644
--- a/mm/mmu_notifier.c
+++ b/mm/mmu_notifier.c
@@ -40,48 +40,44 @@ void __mmu_notifier_release(struct mm_struct *mm)
         int id;
 
         /*
-         * srcu_read_lock() here will block synchronize_srcu() in
-         * mmu_notifier_unregister() until all registered
-         * ->release() callouts this function makes have
-         * returned.
+         * SRCU here will block mmu_notifier_unregister until
+         * ->release returns.
          */
         id = srcu_read_lock(&srcu);
+        hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist)
+                /*
+                 * If ->release runs before mmu_notifier_unregister it must be
+                 * handled, as it's the only way for the driver to flush all
+                 * existing sptes and stop the driver from establishing any more
+                 * sptes before all the pages in the mm are freed.
+                 */
+                if (mn->ops->release)
+                        mn->ops->release(mn, mm);
+        srcu_read_unlock(&srcu, id);
+
         spin_lock(&mm->mmu_notifier_mm->lock);
         while (unlikely(!hlist_empty(&mm->mmu_notifier_mm->list))) {
                 mn = hlist_entry(mm->mmu_notifier_mm->list.first,
                                  struct mmu_notifier,
                                  hlist);
-
                 /*
-                 * Unlink.  This will prevent mmu_notifier_unregister()
-                 * from also making the ->release() callout.
+                 * We arrived before mmu_notifier_unregister so
+                 * mmu_notifier_unregister will do nothing other than to wait
+                 * for ->release to finish and for mmu_notifier_unregister to
+                 * return.
                  */
                 hlist_del_init_rcu(&mn->hlist);
-                spin_unlock(&mm->mmu_notifier_mm->lock);
-
-                /*
-                 * Clear sptes. (see 'release' description in mmu_notifier.h)
-                 */
-                if (mn->ops->release)
-                        mn->ops->release(mn, mm);
-
-                spin_lock(&mm->mmu_notifier_mm->lock);
         }
         spin_unlock(&mm->mmu_notifier_mm->lock);
 
         /*
-         * All callouts to ->release() which we have done are complete.
-         * Allow synchronize_srcu() in mmu_notifier_unregister() to complete
-         */
-        srcu_read_unlock(&srcu, id);
-
-        /*
-         * mmu_notifier_unregister() may have unlinked a notifier and may
-         * still be calling out to it.  Additionally, other notifiers
-         * may have been active via vmtruncate() et. al. Block here
-         * to ensure that all notifier callouts for this mm have been
-         * completed and the sptes are really cleaned up before returning
-         * to exit_mmap().
+         * synchronize_srcu here prevents mmu_notifier_release from returning to
+         * exit_mmap (which would proceed with freeing all pages in the mm)
+         * until the ->release method returns, if it was invoked by
+         * mmu_notifier_unregister.
+         *
+         * The mmu_notifier_mm can't go away from under us because one mm_count
+         * is held by exit_mmap.
          */
         synchronize_srcu(&srcu);
 }
@@ -292,31 +288,34 @@ void mmu_notifier_unregister(struct mmu_notifier *mn, struct mm_struct *mm)
 {
         BUG_ON(atomic_read(&mm->mm_count) <= 0);
 
-        spin_lock(&mm->mmu_notifier_mm->lock);
         if (!hlist_unhashed(&mn->hlist)) {
+                /*
+                 * SRCU here will force exit_mmap to wait for ->release to
+                 * finish before freeing the pages.
+                 */
                 int id;
 
+                id = srcu_read_lock(&srcu);
                 /*
-                 * Ensure we synchronize up with __mmu_notifier_release().
+                 * exit_mmap will block in mmu_notifier_release to guarantee
+                 * that ->release is called before freeing the pages.
                  */
-                id = srcu_read_lock(&srcu);
-
-                hlist_del_rcu(&mn->hlist);
-                spin_unlock(&mm->mmu_notifier_mm->lock);
-
                 if (mn->ops->release)
                         mn->ops->release(mn, mm);
+                srcu_read_unlock(&srcu, id);
 
+                spin_lock(&mm->mmu_notifier_mm->lock);
                 /*
-                 * Allow __mmu_notifier_release() to complete.
+                 * Can not use list_del_rcu() since __mmu_notifier_release
+                 * can delete it before we hold the lock.
                  */
-                srcu_read_unlock(&srcu, id);
-        } else
+                hlist_del_init_rcu(&mn->hlist);
                 spin_unlock(&mm->mmu_notifier_mm->lock);
+        }
 
         /*
-         * Wait for any running method to finish, including ->release() if it
-         * was run by __mmu_notifier_release() instead of us.
+         * Wait for any running method to finish, of course including
+         * ->release if it was run by mmu_notifier_relase instead of us.
          */
         synchronize_srcu(&srcu);