3 files changed, 68 insertions, 4 deletions

diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index 48c32ebf65a7..a9ace9c32507 100644
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -22,6 +22,11 @@ extern int zap_huge_pmd(struct mmu_gather *tlb,
 extern int mincore_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
                         unsigned long addr, unsigned long end,
                         unsigned char *vec);
+extern int move_huge_pmd(struct vm_area_struct *vma,
+                         struct vm_area_struct *new_vma,
+                         unsigned long old_addr,
+                         unsigned long new_addr, unsigned long old_end,
+                         pmd_t *old_pmd, pmd_t *new_pmd);
 extern int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
                         unsigned long addr, pgprot_t newprot);
 
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index e2d1587be269..6b072bdccf81 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1052,6 +1052,51 @@ int mincore_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
         return ret;
 }
 
+int move_huge_pmd(struct vm_area_struct *vma, struct vm_area_struct *new_vma,
+                  unsigned long old_addr,
+                  unsigned long new_addr, unsigned long old_end,
+                  pmd_t *old_pmd, pmd_t *new_pmd)
+{
+        int ret = 0;
+        pmd_t pmd;
+
+        struct mm_struct *mm = vma->vm_mm;
+
+        if ((old_addr & ~HPAGE_PMD_MASK) ||
+            (new_addr & ~HPAGE_PMD_MASK) ||
+            old_end - old_addr < HPAGE_PMD_SIZE ||
+            (new_vma->vm_flags & VM_NOHUGEPAGE))
+                goto out;
+
+        /*
+         * The destination pmd shouldn't be established, free_pgtables()
+         * should have release it.
+         */
+        if (WARN_ON(!pmd_none(*new_pmd))) {
+                VM_BUG_ON(pmd_trans_huge(*new_pmd));
+                goto out;
+        }
+
+        spin_lock(&mm->page_table_lock);
+        if (likely(pmd_trans_huge(*old_pmd))) {
+                if (pmd_trans_splitting(*old_pmd)) {
+                        spin_unlock(&mm->page_table_lock);
+                        wait_split_huge_page(vma->anon_vma, old_pmd);
+                        ret = -1;
+                } else {
+                        pmd = pmdp_get_and_clear(mm, old_addr, old_pmd);
+                        VM_BUG_ON(!pmd_none(*new_pmd));
+                        set_pmd_at(mm, new_addr, new_pmd, pmd);
+                        spin_unlock(&mm->page_table_lock);
+                        ret = 1;
+                }
+        } else {
+                spin_unlock(&mm->page_table_lock);
+        }
+out:
+        return ret;
+}
+
 int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
                 unsigned long addr, pgprot_t newprot)
 {
diff --git a/mm/mremap.c b/mm/mremap.c
index a184f3732e1e..d6959cb4df58 100644
--- a/mm/mremap.c
+++ b/mm/mremap.c
@@ -41,8 +41,7 @@ static pmd_t *get_old_pmd(struct mm_struct *mm, unsigned long addr)
                 return NULL;
 
         pmd = pmd_offset(pud, addr);
-        split_huge_page_pmd(mm, pmd);
-        if (pmd_none_or_clear_bad(pmd))
+        if (pmd_none(*pmd))
                 return NULL;
 
         return pmd;
@@ -65,8 +64,6 @@ static pmd_t *alloc_new_pmd(struct mm_struct *mm, struct vm_area_struct *vma,
                 return NULL;
 
         VM_BUG_ON(pmd_trans_huge(*pmd));
-        if (pmd_none(*pmd) && __pte_alloc(mm, vma, pmd, addr))
-                return NULL;
 
         return pmd;
 }
@@ -149,6 +146,23 @@ unsigned long move_page_tables(struct vm_area_struct *vma,
                 new_pmd = alloc_new_pmd(vma->vm_mm, vma, new_addr);
                 if (!new_pmd)
                         break;
+                if (pmd_trans_huge(*old_pmd)) {
+                        int err = 0;
+                        if (extent == HPAGE_PMD_SIZE)
+                                err = move_huge_pmd(vma, new_vma, old_addr,
+                                                    new_addr, old_end,
+                                                    old_pmd, new_pmd);
+                        if (err > 0) {
+                                need_flush = true;
+                                continue;
+                        } else if (!err) {
+                                split_huge_page_pmd(vma->vm_mm, old_pmd);
+                        }
+                        VM_BUG_ON(pmd_trans_huge(*old_pmd));
+                }
+                if (pmd_none(*new_pmd) && __pte_alloc(new_vma->vm_mm, new_vma,
+                                                      new_pmd, new_addr))
+                        break;
                 next = (new_addr + PMD_SIZE) & PMD_MASK;
                 if (extent > next - new_addr)
                         extent = next - new_addr;