diff options
author | Mel Gorman <mgorman@suse.de> | 2012-11-02 07:33:45 -0400 |
---|---|---|
committer | Mel Gorman <mgorman@suse.de> | 2012-12-11 09:42:42 -0500 |
commit | 4daae3b4b9e49b7e0935499a352f1c59d90287d2 (patch) | |
tree | 2ac600b955c89e3b1b2070110a9b7293a4511b19 /mm/huge_memory.c | |
parent | 149c33e1c98f83050870514f380902dc6d617bd5 (diff) |
mm: mempolicy: Use _PAGE_NUMA to migrate pages
Note: Based on "mm/mpol: Use special PROT_NONE to migrate pages" but
sufficiently different that the signed-off-bys were dropped
Combine our previous _PAGE_NUMA, mpol_misplaced and migrate_misplaced_page()
pieces into an effective migrate on fault scheme.
Note that (on x86) we rely on PROT_NONE pages being !present and avoid
the TLB flush from try_to_unmap(TTU_MIGRATION). This greatly improves the
page-migration performance.
Based-on-work-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Mel Gorman <mgorman@suse.de>
Diffstat (limited to 'mm/huge_memory.c')
-rw-r--r-- | mm/huge_memory.c | 31 |
1 files changed, 28 insertions, 3 deletions
diff --git a/mm/huge_memory.c b/mm/huge_memory.c index f5f37630c54d..5723b551c023 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c | |||
@@ -18,6 +18,7 @@ | |||
18 | #include <linux/freezer.h> | 18 | #include <linux/freezer.h> |
19 | #include <linux/mman.h> | 19 | #include <linux/mman.h> |
20 | #include <linux/pagemap.h> | 20 | #include <linux/pagemap.h> |
21 | #include <linux/migrate.h> | ||
21 | #include <asm/tlb.h> | 22 | #include <asm/tlb.h> |
22 | #include <asm/pgalloc.h> | 23 | #include <asm/pgalloc.h> |
23 | #include "internal.h" | 24 | #include "internal.h" |
@@ -1019,17 +1020,39 @@ out: | |||
1019 | } | 1020 | } |
1020 | 1021 | ||
1021 | /* NUMA hinting page fault entry point for trans huge pmds */ | 1022 | /* NUMA hinting page fault entry point for trans huge pmds */ |
1022 | int do_huge_pmd_numa_page(struct mm_struct *mm, unsigned long addr, | 1023 | int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma, |
1023 | pmd_t pmd, pmd_t *pmdp) | 1024 | unsigned long addr, pmd_t pmd, pmd_t *pmdp) |
1024 | { | 1025 | { |
1025 | struct page *page; | 1026 | struct page *page = NULL; |
1026 | unsigned long haddr = addr & HPAGE_PMD_MASK; | 1027 | unsigned long haddr = addr & HPAGE_PMD_MASK; |
1028 | int target_nid; | ||
1027 | 1029 | ||
1028 | spin_lock(&mm->page_table_lock); | 1030 | spin_lock(&mm->page_table_lock); |
1029 | if (unlikely(!pmd_same(pmd, *pmdp))) | 1031 | if (unlikely(!pmd_same(pmd, *pmdp))) |
1030 | goto out_unlock; | 1032 | goto out_unlock; |
1031 | 1033 | ||
1032 | page = pmd_page(pmd); | 1034 | page = pmd_page(pmd); |
1035 | get_page(page); | ||
1036 | spin_unlock(&mm->page_table_lock); | ||
1037 | |||
1038 | target_nid = mpol_misplaced(page, vma, haddr); | ||
1039 | if (target_nid == -1) | ||
1040 | goto clear_pmdnuma; | ||
1041 | |||
1042 | /* | ||
1043 | * Due to lacking code to migrate thp pages, we'll split | ||
1044 | * (which preserves the special PROT_NONE) and re-take the | ||
1045 | * fault on the normal pages. | ||
1046 | */ | ||
1047 | split_huge_page(page); | ||
1048 | put_page(page); | ||
1049 | return 0; | ||
1050 | |||
1051 | clear_pmdnuma: | ||
1052 | spin_lock(&mm->page_table_lock); | ||
1053 | if (unlikely(!pmd_same(pmd, *pmdp))) | ||
1054 | goto out_unlock; | ||
1055 | |||
1033 | pmd = pmd_mknonnuma(pmd); | 1056 | pmd = pmd_mknonnuma(pmd); |
1034 | set_pmd_at(mm, haddr, pmdp, pmd); | 1057 | set_pmd_at(mm, haddr, pmdp, pmd); |
1035 | VM_BUG_ON(pmd_numa(*pmdp)); | 1058 | VM_BUG_ON(pmd_numa(*pmdp)); |
@@ -1037,6 +1060,8 @@ int do_huge_pmd_numa_page(struct mm_struct *mm, unsigned long addr, | |||
1037 | 1060 | ||
1038 | out_unlock: | 1061 | out_unlock: |
1039 | spin_unlock(&mm->page_table_lock); | 1062 | spin_unlock(&mm->page_table_lock); |
1063 | if (page) | ||
1064 | put_page(page); | ||
1040 | return 0; | 1065 | return 0; |
1041 | } | 1066 | } |
1042 | 1067 | ||