5 files changed, 53 insertions, 37 deletions

diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index 7d7856359920..7b5785032049 100644
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -99,9 +99,9 @@ int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep);
 struct page *follow_huge_addr(struct mm_struct *mm, unsigned long address,
                               int write);
 struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
-                                pmd_t *pmd, int write);
+                                pmd_t *pmd, int flags);
 struct page *follow_huge_pud(struct mm_struct *mm, unsigned long address,
-                                pud_t *pud, int write);
+                                pud_t *pud, int flags);
 int pmd_huge(pmd_t pmd);
 int pud_huge(pud_t pmd);
 unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
@@ -133,8 +133,8 @@ static inline void hugetlb_report_meminfo(struct seq_file *m)
 static inline void hugetlb_show_meminfo(void)
 {
 }
-#define follow_huge_pmd(mm, addr, pmd, write)   NULL
-#define follow_huge_pud(mm, addr, pud, write)   NULL
+#define follow_huge_pmd(mm, addr, pmd, flags)   NULL
+#define follow_huge_pud(mm, addr, pud, flags)   NULL
 #define prepare_hugepage_range(file, addr, len) (-EINVAL)
 #define pmd_huge(x)     0
 #define pud_huge(x)     0
diff --git a/include/linux/swapops.h b/include/linux/swapops.h
index 50cbc876be56..831a3168ab35 100644
--- a/include/linux/swapops.h
+++ b/include/linux/swapops.h
@@ -135,6 +135,8 @@ static inline void make_migration_entry_read(swp_entry_t *entry)
         *entry = swp_entry(SWP_MIGRATION_READ, swp_offset(*entry));
 }
 
+extern void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep,
+                                        spinlock_t *ptl);
 extern void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
                                         unsigned long address);
 extern void migration_entry_wait_huge(struct vm_area_struct *vma,
@@ -148,6 +150,8 @@ static inline int is_migration_entry(swp_entry_t swp)
 }
 #define migration_entry_to_page(swp) NULL
 static inline void make_migration_entry_read(swp_entry_t *entryp) { }
+static inline void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep,
+                                        spinlock_t *ptl) { }
 static inline void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
                                          unsigned long address) { }
 static inline void migration_entry_wait_huge(struct vm_area_struct *vma,
diff --git a/mm/gup.c b/mm/gup.c
index 12bc2bc33da7..1a8ab05918e0 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -167,10 +167,10 @@ struct page *follow_page_mask(struct vm_area_struct *vma,
         if (pud_none(*pud))
                 return no_page_table(vma, flags);
         if (pud_huge(*pud) && vma->vm_flags & VM_HUGETLB) {
-                if (flags & FOLL_GET)
-                        return NULL;
-                page = follow_huge_pud(mm, address, pud, flags & FOLL_WRITE);
-                return page;
+                page = follow_huge_pud(mm, address, pud, flags);
+                if (page)
+                        return page;
+                return no_page_table(vma, flags);
         }
         if (unlikely(pud_bad(*pud)))
                 return no_page_table(vma, flags);
@@ -179,19 +179,10 @@ struct page *follow_page_mask(struct vm_area_struct *vma,
         if (pmd_none(*pmd))
                 return no_page_table(vma, flags);
         if (pmd_huge(*pmd) && vma->vm_flags & VM_HUGETLB) {
-                page = follow_huge_pmd(mm, address, pmd, flags & FOLL_WRITE);
-                if (flags & FOLL_GET) {
-                        /*
-                         * Refcount on tail pages are not well-defined and
-                         * shouldn't be taken. The caller should handle a NULL
-                         * return when trying to follow tail pages.
-                         */
-                        if (PageHead(page))
-                                get_page(page);
-                        else
-                                page = NULL;
-                }
-                return page;
+                page = follow_huge_pmd(mm, address, pmd, flags);
+                if (page)
+                        return page;
+                return no_page_table(vma, flags);
         }
         if ((flags & FOLL_NUMA) && pmd_numa(*pmd))
                 return no_page_table(vma, flags);
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index d96b8bfa748f..5aca3707450f 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -3675,28 +3675,48 @@ follow_huge_addr(struct mm_struct *mm, unsigned long address,
 
 struct page * __weak
 follow_huge_pmd(struct mm_struct *mm, unsigned long address,
-                pmd_t *pmd, int write)
+                pmd_t *pmd, int flags)
 {
-        struct page *page;
-
-        if (!pmd_present(*pmd))
-                return NULL;
-        page = pte_page(*(pte_t *)pmd);
-        if (page)
-                page += ((address & ~PMD_MASK) >> PAGE_SHIFT);
+        struct page *page = NULL;
+        spinlock_t *ptl;
+retry:
+        ptl = pmd_lockptr(mm, pmd);
+        spin_lock(ptl);
+        /*
+         * make sure that the address range covered by this pmd is not
+         * unmapped from other threads.
+         */
+        if (!pmd_huge(*pmd))
+                goto out;
+        if (pmd_present(*pmd)) {
+                page = pte_page(*(pte_t *)pmd) +
+                        ((address & ~PMD_MASK) >> PAGE_SHIFT);
+                if (flags & FOLL_GET)
+                        get_page(page);
+        } else {
+                if (is_hugetlb_entry_migration(huge_ptep_get((pte_t *)pmd))) {
+                        spin_unlock(ptl);
+                        __migration_entry_wait(mm, (pte_t *)pmd, ptl);
+                        goto retry;
+                }
+                /*
+                 * hwpoisoned entry is treated as no_page_table in
+                 * follow_page_mask().
+                 */
+        }
+out:
+        spin_unlock(ptl);
         return page;
 }
 
 struct page * __weak
 follow_huge_pud(struct mm_struct *mm, unsigned long address,
-                pud_t *pud, int write)
+                pud_t *pud, int flags)
 {
-        struct page *page;
+        if (flags & FOLL_GET)
+                return NULL;
 
-        page = pte_page(*(pte_t *)pud);
-        if (page)
-                page += ((address & ~PUD_MASK) >> PAGE_SHIFT);
-        return page;
+        return pte_page(*(pte_t *)pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT);
 }
 
 #ifdef CONFIG_MEMORY_FAILURE
diff --git a/mm/migrate.c b/mm/migrate.c
index 6e284bcca8bb..f98067e5d353 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -197,7 +197,7 @@ static void remove_migration_ptes(struct page *old, struct page *new)
  * get to the page and wait until migration is finished.
  * When we return from this function the fault will be retried.
  */
-static void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep,
+void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep,
                                 spinlock_t *ptl)
 {
         pte_t pte;
@@ -1236,7 +1236,8 @@ static int do_move_page_to_node_array(struct mm_struct *mm,
                         goto put_and_set;
 
                 if (PageHuge(page)) {
-                        isolate_huge_page(page, &pagelist);
+                        if (PageHead(page))
+                                isolate_huge_page(page, &pagelist);
                         goto put_and_set;
                 }