mm: wrap calls to set_pte_at_notify with invalidate_range_start and invalidate_range_end

In order to allow sleeping during invalidate_page mmu notifier calls, we
need to avoid calling when holding the PT lock.  In addition to its direct
calls, invalidate_page can also be called as a substitute for a change_pte
call, in case the notifier client hasn't implemented change_pte.

This patch drops the invalidate_page call from change_pte, and instead
wraps all calls to change_pte with invalidate_range_start and
invalidate_range_end calls.

Note that change_pte still cannot sleep after this patch, and that clients
implementing change_pte should not take action on it in case the number of
outstanding invalidate_range_start calls is larger than one, otherwise
they might miss a later invalidation.

Signed-off-by: Haggai Eran <haggaie@mellanox.com>
Cc: Andrea Arcangeli <andrea@qumranet.com>
Cc: Sagi Grimberg <sagig@mellanox.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Or Gerlitz <ogerlitz@mellanox.com>
Cc: Haggai Eran <haggaie@mellanox.com>
Cc: Shachar Raindel <raindel@mellanox.com>
Cc: Liran Liss <liranl@mellanox.com>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Avi Kivity <avi@redhat.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 19 insertions, 2 deletions

diff --git a/mm/ksm.c b/mm/ksm.c
index ecbc090cdaad..ae539f0b8aa1 100644
--- a/mm/ksm.c
+++ b/mm/ksm.c
@@ -709,15 +709,22 @@ static int write_protect_page(struct vm_area_struct *vma, struct page *page,
         spinlock_t *ptl;
         int swapped;
         int err = -EFAULT;
+        unsigned long mmun_start;       /* For mmu_notifiers */
+        unsigned long mmun_end;         /* For mmu_notifiers */
 
         addr = page_address_in_vma(page, vma);
         if (addr == -EFAULT)
                 goto out;
 
         BUG_ON(PageTransCompound(page));
+
+        mmun_start = addr;
+        mmun_end   = addr + PAGE_SIZE;
+        mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
+
         ptep = page_check_address(page, mm, addr, &ptl, 0);
         if (!ptep)
-                goto out;
+                goto out_mn;
 
         if (pte_write(*ptep) || pte_dirty(*ptep)) {
                 pte_t entry;
@@ -752,6 +759,8 @@ static int write_protect_page(struct vm_area_struct *vma, struct page *page,
 
 out_unlock:
         pte_unmap_unlock(ptep, ptl);
+out_mn:
+        mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
 out:
         return err;
 }
@@ -776,6 +785,8 @@ static int replace_page(struct vm_area_struct *vma, struct page *page,
         spinlock_t *ptl;
         unsigned long addr;
         int err = -EFAULT;
+        unsigned long mmun_start;       /* For mmu_notifiers */
+        unsigned long mmun_end;         /* For mmu_notifiers */
 
         addr = page_address_in_vma(page, vma);
         if (addr == -EFAULT)
@@ -794,10 +805,14 @@ static int replace_page(struct vm_area_struct *vma, struct page *page,
         if (!pmd_present(*pmd))
                 goto out;
 
+        mmun_start = addr;
+        mmun_end   = addr + PAGE_SIZE;
+        mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
+
         ptep = pte_offset_map_lock(mm, pmd, addr, &ptl);
         if (!pte_same(*ptep, orig_pte)) {
                 pte_unmap_unlock(ptep, ptl);
-                goto out;
+                goto out_mn;
         }
 
         get_page(kpage);
@@ -814,6 +829,8 @@ static int replace_page(struct vm_area_struct *vma, struct page *page,
 
         pte_unmap_unlock(ptep, ptl);
         err = 0;
+out_mn:
+        mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
 out:
         return err;
 }