aboutsummaryrefslogtreecommitdiffstats
path: root/mm
diff options
context:
space:
mode:
Diffstat (limited to 'mm')
-rw-r--r--mm/huge_memory.c22
-rw-r--r--mm/migrate.c12
2 files changed, 28 insertions, 6 deletions
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 70e7429fd8ea..7de1bf85f683 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -882,6 +882,10 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
882 ret = 0; 882 ret = 0;
883 goto out_unlock; 883 goto out_unlock;
884 } 884 }
885
886 /* mmap_sem prevents this happening but warn if that changes */
887 WARN_ON(pmd_trans_migrating(pmd));
888
885 if (unlikely(pmd_trans_splitting(pmd))) { 889 if (unlikely(pmd_trans_splitting(pmd))) {
886 /* split huge page running from under us */ 890 /* split huge page running from under us */
887 spin_unlock(src_ptl); 891 spin_unlock(src_ptl);
@@ -1299,6 +1303,17 @@ int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
1299 if (unlikely(!pmd_same(pmd, *pmdp))) 1303 if (unlikely(!pmd_same(pmd, *pmdp)))
1300 goto out_unlock; 1304 goto out_unlock;
1301 1305
1306 /*
1307 * If there are potential migrations, wait for completion and retry
1308 * without disrupting NUMA hinting information. Do not relock and
1309 * check_same as the page may no longer be mapped.
1310 */
1311 if (unlikely(pmd_trans_migrating(*pmdp))) {
1312 spin_unlock(ptl);
1313 wait_migrate_huge_page(vma->anon_vma, pmdp);
1314 goto out;
1315 }
1316
1302 page = pmd_page(pmd); 1317 page = pmd_page(pmd);
1303 BUG_ON(is_huge_zero_page(page)); 1318 BUG_ON(is_huge_zero_page(page));
1304 page_nid = page_to_nid(page); 1319 page_nid = page_to_nid(page);
@@ -1329,12 +1344,7 @@ int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
1329 goto clear_pmdnuma; 1344 goto clear_pmdnuma;
1330 } 1345 }
1331 1346
1332 /* 1347 /* Migration could have started since the pmd_trans_migrating check */
1333 * If there are potential migrations, wait for completion and retry. We
1334 * do not relock and check_same as the page may no longer be mapped.
1335 * Furtermore, even if the page is currently misplaced, there is no
1336 * guarantee it is still misplaced after the migration completes.
1337 */
1338 if (!page_locked) { 1348 if (!page_locked) {
1339 spin_unlock(ptl); 1349 spin_unlock(ptl);
1340 wait_on_page_locked(page); 1350 wait_on_page_locked(page);
diff --git a/mm/migrate.c b/mm/migrate.c
index a987525810ae..cfb419085261 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -1655,6 +1655,18 @@ int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page)
1655 return 1; 1655 return 1;
1656} 1656}
1657 1657
1658bool pmd_trans_migrating(pmd_t pmd)
1659{
1660 struct page *page = pmd_page(pmd);
1661 return PageLocked(page);
1662}
1663
1664void wait_migrate_huge_page(struct anon_vma *anon_vma, pmd_t *pmd)
1665{
1666 struct page *page = pmd_page(*pmd);
1667 wait_on_page_locked(page);
1668}
1669
1658/* 1670/*
1659 * Attempt to migrate a misplaced page to the specified destination 1671 * Attempt to migrate a misplaced page to the specified destination
1660 * node. Caller is expected to have an elevated reference count on 1672 * node. Caller is expected to have an elevated reference count on