1 files changed, 48 insertions, 22 deletions

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 610e3df2768a..cca80d96e509 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1278,64 +1278,90 @@ out:
 int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
                                 unsigned long addr, pmd_t pmd, pmd_t *pmdp)
 {
+        struct anon_vma *anon_vma = NULL;
         struct page *page;
         unsigned long haddr = addr & HPAGE_PMD_MASK;
+        int page_nid = -1, this_nid = numa_node_id();
         int target_nid;
-        int current_nid = -1;
-        bool migrated;
+        bool page_locked;
+        bool migrated = false;
 
         spin_lock(&mm->page_table_lock);
         if (unlikely(!pmd_same(pmd, *pmdp)))
                 goto out_unlock;
 
         page = pmd_page(pmd);
-        get_page(page);
-        current_nid = page_to_nid(page);
+        page_nid = page_to_nid(page);
         count_vm_numa_event(NUMA_HINT_FAULTS);
-        if (current_nid == numa_node_id())
+        if (page_nid == this_nid)
                 count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
 
+        /*
+         * Acquire the page lock to serialise THP migrations but avoid dropping
+         * page_table_lock if at all possible
+         */
+        page_locked = trylock_page(page);
         target_nid = mpol_misplaced(page, vma, haddr);
         if (target_nid == -1) {
-                put_page(page);
-                goto clear_pmdnuma;
+                /* If the page was locked, there are no parallel migrations */
+                if (page_locked)
+                        goto clear_pmdnuma;
+
+                /*
+                 * Otherwise wait for potential migrations and retry. We do
+                 * relock and check_same as the page may no longer be mapped.
+                 * As the fault is being retried, do not account for it.
+                 */
+                spin_unlock(&mm->page_table_lock);
+                wait_on_page_locked(page);
+                page_nid = -1;
+                goto out;
         }
 
-        /* Acquire the page lock to serialise THP migrations */
+        /* Page is misplaced, serialise migrations and parallel THP splits */
+        get_page(page);
         spin_unlock(&mm->page_table_lock);
-        lock_page(page);
+        if (!page_locked)
+                lock_page(page);
+        anon_vma = page_lock_anon_vma_read(page);
 
         /* Confirm the PTE did not while locked */
         spin_lock(&mm->page_table_lock);
         if (unlikely(!pmd_same(pmd, *pmdp))) {
                 unlock_page(page);
                 put_page(page);
+                page_nid = -1;
                 goto out_unlock;
         }
-        spin_unlock(&mm->page_table_lock);
 
-        /* Migrate the THP to the requested node */
+        /*
+         * Migrate the THP to the requested node, returns with page unlocked
+         * and pmd_numa cleared.
+         */
+        spin_unlock(&mm->page_table_lock);
         migrated = migrate_misplaced_transhuge_page(mm, vma,
                                 pmdp, pmd, addr, page, target_nid);
-        if (!migrated)
-                goto check_same;
-
-        task_numa_fault(target_nid, HPAGE_PMD_NR, true);
-        return 0;
+        if (migrated)
+                page_nid = target_nid;
 
-check_same:
-        spin_lock(&mm->page_table_lock);
-        if (unlikely(!pmd_same(pmd, *pmdp)))
-                goto out_unlock;
+        goto out;
 clear_pmdnuma:
+        BUG_ON(!PageLocked(page));
         pmd = pmd_mknonnuma(pmd);
         set_pmd_at(mm, haddr, pmdp, pmd);
         VM_BUG_ON(pmd_numa(*pmdp));
         update_mmu_cache_pmd(vma, addr, pmdp);
+        unlock_page(page);
 out_unlock:
         spin_unlock(&mm->page_table_lock);
-        if (current_nid != -1)
-                task_numa_fault(current_nid, HPAGE_PMD_NR, false);
+
+out:
+        if (anon_vma)
+                page_unlock_anon_vma_read(anon_vma);
+
+        if (page_nid != -1)
+                task_numa_fault(page_nid, HPAGE_PMD_NR, migrated);
+
         return 0;
 }