diff options
Diffstat (limited to 'mm')
-rw-r--r-- | mm/huge_memory.c | 22 | ||||
-rw-r--r-- | mm/migrate.c | 12 |
2 files changed, 28 insertions, 6 deletions
diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 70e7429fd8ea..7de1bf85f683 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c | |||
@@ -882,6 +882,10 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, | |||
882 | ret = 0; | 882 | ret = 0; |
883 | goto out_unlock; | 883 | goto out_unlock; |
884 | } | 884 | } |
885 | |||
886 | /* mmap_sem prevents this happening but warn if that changes */ | ||
887 | WARN_ON(pmd_trans_migrating(pmd)); | ||
888 | |||
885 | if (unlikely(pmd_trans_splitting(pmd))) { | 889 | if (unlikely(pmd_trans_splitting(pmd))) { |
886 | /* split huge page running from under us */ | 890 | /* split huge page running from under us */ |
887 | spin_unlock(src_ptl); | 891 | spin_unlock(src_ptl); |
@@ -1299,6 +1303,17 @@ int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma, | |||
1299 | if (unlikely(!pmd_same(pmd, *pmdp))) | 1303 | if (unlikely(!pmd_same(pmd, *pmdp))) |
1300 | goto out_unlock; | 1304 | goto out_unlock; |
1301 | 1305 | ||
1306 | /* | ||
1307 | * If there are potential migrations, wait for completion and retry | ||
1308 | * without disrupting NUMA hinting information. Do not relock and | ||
1309 | * check_same as the page may no longer be mapped. | ||
1310 | */ | ||
1311 | if (unlikely(pmd_trans_migrating(*pmdp))) { | ||
1312 | spin_unlock(ptl); | ||
1313 | wait_migrate_huge_page(vma->anon_vma, pmdp); | ||
1314 | goto out; | ||
1315 | } | ||
1316 | |||
1302 | page = pmd_page(pmd); | 1317 | page = pmd_page(pmd); |
1303 | BUG_ON(is_huge_zero_page(page)); | 1318 | BUG_ON(is_huge_zero_page(page)); |
1304 | page_nid = page_to_nid(page); | 1319 | page_nid = page_to_nid(page); |
@@ -1329,12 +1344,7 @@ int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma, | |||
1329 | goto clear_pmdnuma; | 1344 | goto clear_pmdnuma; |
1330 | } | 1345 | } |
1331 | 1346 | ||
1332 | /* | 1347 | /* Migration could have started since the pmd_trans_migrating check */ |
1333 | * If there are potential migrations, wait for completion and retry. We | ||
1334 | * do not relock and check_same as the page may no longer be mapped. | ||
1335 | * Furtermore, even if the page is currently misplaced, there is no | ||
1336 | * guarantee it is still misplaced after the migration completes. | ||
1337 | */ | ||
1338 | if (!page_locked) { | 1348 | if (!page_locked) { |
1339 | spin_unlock(ptl); | 1349 | spin_unlock(ptl); |
1340 | wait_on_page_locked(page); | 1350 | wait_on_page_locked(page); |
diff --git a/mm/migrate.c b/mm/migrate.c index a987525810ae..cfb419085261 100644 --- a/mm/migrate.c +++ b/mm/migrate.c | |||
@@ -1655,6 +1655,18 @@ int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page) | |||
1655 | return 1; | 1655 | return 1; |
1656 | } | 1656 | } |
1657 | 1657 | ||
1658 | bool pmd_trans_migrating(pmd_t pmd) | ||
1659 | { | ||
1660 | struct page *page = pmd_page(pmd); | ||
1661 | return PageLocked(page); | ||
1662 | } | ||
1663 | |||
1664 | void wait_migrate_huge_page(struct anon_vma *anon_vma, pmd_t *pmd) | ||
1665 | { | ||
1666 | struct page *page = pmd_page(*pmd); | ||
1667 | wait_on_page_locked(page); | ||
1668 | } | ||
1669 | |||
1658 | /* | 1670 | /* |
1659 | * Attempt to migrate a misplaced page to the specified destination | 1671 | * Attempt to migrate a misplaced page to the specified destination |
1660 | * node. Caller is expected to have an elevated reference count on | 1672 | * node. Caller is expected to have an elevated reference count on |