1 files changed, 62 insertions, 19 deletions

diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index 84fa00497c49..94d38a39d72e 100644
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -26,6 +26,7 @@
 #include <linux/percpu-rwsem.h>
 #include <linux/task_work.h>
 #include <linux/shmem_fs.h>
+#include <linux/khugepaged.h>
 
 #include <linux/uprobes.h>
 
@@ -143,17 +144,19 @@ static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr)
  *
  * @vma:      vma that holds the pte pointing to page
  * @addr:     address the old @page is mapped at
- * @page:     the cowed page we are replacing by kpage
- * @kpage:    the modified page we replace page by
+ * @old_page: the page we are replacing by new_page
+ * @new_page: the modified page we replace page by
  *
- * Returns 0 on success, -EFAULT on failure.
+ * If @new_page is NULL, only unmap @old_page.
+ *
+ * Returns 0 on success, negative error code otherwise.
  */
 static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
                                 struct page *old_page, struct page *new_page)
 {
         struct mm_struct *mm = vma->vm_mm;
         struct page_vma_mapped_walk pvmw = {
-                .page = old_page,
+                .page = compound_head(old_page),
                 .vma = vma,
                 .address = addr,
         };
@@ -164,12 +167,12 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
         mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, addr,
                                 addr + PAGE_SIZE);
 
-        VM_BUG_ON_PAGE(PageTransHuge(old_page), old_page);
-
-        err = mem_cgroup_try_charge(new_page, vma->vm_mm, GFP_KERNEL, &memcg,
-                        false);
-        if (err)
-                return err;
+        if (new_page) {
+                err = mem_cgroup_try_charge(new_page, vma->vm_mm, GFP_KERNEL,
+                                            &memcg, false);
+                if (err)
+                        return err;
+        }
 
         /* For try_to_free_swap() and munlock_vma_page() below */
         lock_page(old_page);
@@ -177,15 +180,20 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
         mmu_notifier_invalidate_range_start(&range);
         err = -EAGAIN;
         if (!page_vma_mapped_walk(&pvmw)) {
-                mem_cgroup_cancel_charge(new_page, memcg, false);
+                if (new_page)
+                        mem_cgroup_cancel_charge(new_page, memcg, false);
                 goto unlock;
         }
         VM_BUG_ON_PAGE(addr != pvmw.address, old_page);
 
-        get_page(new_page);
-        page_add_new_anon_rmap(new_page, vma, addr, false);
-        mem_cgroup_commit_charge(new_page, memcg, false, false);
-        lru_cache_add_active_or_unevictable(new_page, vma);
+        if (new_page) {
+                get_page(new_page);
+                page_add_new_anon_rmap(new_page, vma, addr, false);
+                mem_cgroup_commit_charge(new_page, memcg, false, false);
+                lru_cache_add_active_or_unevictable(new_page, vma);
+        } else
+                /* no new page, just dec_mm_counter for old_page */
+                dec_mm_counter(mm, MM_ANONPAGES);
 
         if (!PageAnon(old_page)) {
                 dec_mm_counter(mm, mm_counter_file(old_page));
@@ -194,8 +202,9 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
 
         flush_cache_page(vma, addr, pte_pfn(*pvmw.pte));
         ptep_clear_flush_notify(vma, addr, pvmw.pte);
-        set_pte_at_notify(mm, addr, pvmw.pte,
-                        mk_pte(new_page, vma->vm_page_prot));
+        if (new_page)
+                set_pte_at_notify(mm, addr, pvmw.pte,
+                                  mk_pte(new_page, vma->vm_page_prot));
 
         page_remove_rmap(old_page, false);
         if (!page_mapped(old_page))
@@ -464,6 +473,7 @@ int uprobe_write_opcode(struct arch_uprobe *auprobe, struct mm_struct *mm,
         struct page *old_page, *new_page;
         struct vm_area_struct *vma;
         int ret, is_register, ref_ctr_updated = 0;
+        bool orig_page_huge = false;
 
         is_register = is_swbp_insn(&opcode);
         uprobe = container_of(auprobe, struct uprobe, arch);
@@ -471,7 +481,7 @@ int uprobe_write_opcode(struct arch_uprobe *auprobe, struct mm_struct *mm,
 retry:
         /* Read the page with vaddr into memory */
         ret = get_user_pages_remote(NULL, mm, vaddr, 1,
-                        FOLL_FORCE | FOLL_SPLIT, &old_page, &vma, NULL);
+                        FOLL_FORCE | FOLL_SPLIT_PMD, &old_page, &vma, NULL);
         if (ret <= 0)
                 return ret;
 
@@ -488,6 +498,10 @@ retry:
                 ref_ctr_updated = 1;
         }
 
+        ret = 0;
+        if (!is_register && !PageAnon(old_page))
+                goto put_old;
+
         ret = anon_vma_prepare(vma);
         if (ret)
                 goto put_old;
@@ -501,8 +515,33 @@ retry:
         copy_highpage(new_page, old_page);
         copy_to_page(new_page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
 
+        if (!is_register) {
+                struct page *orig_page;
+                pgoff_t index;
+
+                VM_BUG_ON_PAGE(!PageAnon(old_page), old_page);
+
+                index = vaddr_to_offset(vma, vaddr & PAGE_MASK) >> PAGE_SHIFT;
+                orig_page = find_get_page(vma->vm_file->f_inode->i_mapping,
+                                          index);
+
+                if (orig_page) {
+                        if (PageUptodate(orig_page) &&
+                            pages_identical(new_page, orig_page)) {
+                                /* let go new_page */
+                                put_page(new_page);
+                                new_page = NULL;
+
+                                if (PageCompound(orig_page))
+                                        orig_page_huge = true;
+                        }
+                        put_page(orig_page);
+                }
+        }
+
         ret = __replace_page(vma, vaddr, old_page, new_page);
-        put_page(new_page);
+        if (new_page)
+                put_page(new_page);
 put_old:
         put_page(old_page);
 
@@ -513,6 +552,10 @@ put_old:
         if (ret && is_register && ref_ctr_updated)
                 update_ref_ctr(uprobe, mm, -1);
 
+        /* try collapse pmd for compound page */
+        if (!ret && orig_page_huge)
+                collapse_pte_mapped_thp(mm, vaddr);
+
         return ret;
 }