mm: numa: Change page last {nid,pid} into {cpu,pid}

Change the per page last fault tracking to use cpu,pid instead of
nid,pid. This will allow us to try and lookup the alternate task more
easily. Note that even though it is the cpu that is store in the page
flags that the mpol_misplaced decision is still based on the node.

Signed-off-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Mel Gorman <mgorman@suse.de>
Reviewed-by: Rik van Riel <riel@redhat.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Srikar Dronamraju <srikar@linux.vnet.ibm.com>
Link: http://lkml.kernel.org/r/1381141781-10992-43-git-send-email-mgorman@suse.de
[ Fixed build failure on 32-bit systems. ]
Signed-off-by: Ingo Molnar <mingo@kernel.org>

8 files changed, 55 insertions, 53 deletions

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 0baf0e4d5203..becf92ca54f3 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1282,7 +1282,7 @@ int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
         struct page *page;
         unsigned long haddr = addr & HPAGE_PMD_MASK;
         int page_nid = -1, this_nid = numa_node_id();
-        int target_nid, last_nidpid = -1;
+        int target_nid, last_cpupid = -1;
         bool page_locked;
         bool migrated = false;
 
@@ -1293,7 +1293,7 @@ int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
         page = pmd_page(pmd);
         BUG_ON(is_huge_zero_page(page));
         page_nid = page_to_nid(page);
-        last_nidpid = page_nidpid_last(page);
+        last_cpupid = page_cpupid_last(page);
         count_vm_numa_event(NUMA_HINT_FAULTS);
         if (page_nid == this_nid)
                 count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
@@ -1362,7 +1362,7 @@ out:
                 page_unlock_anon_vma_read(anon_vma);
 
         if (page_nid != -1)
-                task_numa_fault(last_nidpid, page_nid, HPAGE_PMD_NR, migrated);
+                task_numa_fault(last_cpupid, page_nid, HPAGE_PMD_NR, migrated);
 
         return 0;
 }
@@ -1682,7 +1682,7 @@ static void __split_huge_page_refcount(struct page *page,
                 page_tail->mapping = page->mapping;
 
                 page_tail->index = page->index + i;
-                page_nidpid_xchg_last(page_tail, page_nidpid_last(page));
+                page_cpupid_xchg_last(page_tail, page_cpupid_last(page));
 
                 BUG_ON(!PageAnon(page_tail));
                 BUG_ON(!PageUptodate(page_tail));
diff --git a/mm/memory.c b/mm/memory.c
index cc7f20691c82..5162e6d0d652 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -69,8 +69,8 @@
 
 #include "internal.h"
 
-#ifdef LAST_NIDPID_NOT_IN_PAGE_FLAGS
-#warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_nidpid.
+#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
+#warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.
 #endif
 
 #ifndef CONFIG_NEED_MULTIPLE_NODES
@@ -3536,7 +3536,7 @@ int do_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
         struct page *page = NULL;
         spinlock_t *ptl;
         int page_nid = -1;
-        int last_nidpid;
+        int last_cpupid;
         int target_nid;
         bool migrated = false;
 
@@ -3567,7 +3567,7 @@ int do_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
         }
         BUG_ON(is_zero_pfn(page_to_pfn(page)));
 
-        last_nidpid = page_nidpid_last(page);
+        last_cpupid = page_cpupid_last(page);
         page_nid = page_to_nid(page);
         target_nid = numa_migrate_prep(page, vma, addr, page_nid);
         pte_unmap_unlock(ptep, ptl);
@@ -3583,7 +3583,7 @@ int do_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
 
 out:
         if (page_nid != -1)
-                task_numa_fault(last_nidpid, page_nid, 1, migrated);
+                task_numa_fault(last_cpupid, page_nid, 1, migrated);
         return 0;
 }
 
@@ -3598,7 +3598,7 @@ static int do_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
         unsigned long offset;
         spinlock_t *ptl;
         bool numa = false;
-        int last_nidpid;
+        int last_cpupid;
 
         spin_lock(&mm->page_table_lock);
         pmd = *pmdp;
@@ -3643,7 +3643,7 @@ static int do_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
                 if (unlikely(!page))
                         continue;
 
-                last_nidpid = page_nidpid_last(page);
+                last_cpupid = page_cpupid_last(page);
                 page_nid = page_to_nid(page);
                 target_nid = numa_migrate_prep(page, vma, addr, page_nid);
                 pte_unmap_unlock(pte, ptl);
@@ -3656,7 +3656,7 @@ static int do_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
                 }
 
                 if (page_nid != -1)
-                        task_numa_fault(last_nidpid, page_nid, 1, migrated);
+                        task_numa_fault(last_cpupid, page_nid, 1, migrated);
 
                 pte = pte_offset_map_lock(mm, pmdp, addr, &ptl);
         }
diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 0e895a2eed5f..a5867ef24bda 100644
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -2324,6 +2324,8 @@ int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long
         struct zone *zone;
         int curnid = page_to_nid(page);
         unsigned long pgoff;
+        int thiscpu = raw_smp_processor_id();
+        int thisnid = cpu_to_node(thiscpu);
         int polnid = -1;
         int ret = -1;
 
@@ -2372,11 +2374,11 @@ int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long
 
         /* Migrate the page towards the node whose CPU is referencing it */
         if (pol->flags & MPOL_F_MORON) {
-                int last_nidpid;
-                int this_nidpid;
+                int last_cpupid;
+                int this_cpupid;
 
-                polnid = numa_node_id();
-                this_nidpid = nid_pid_to_nidpid(polnid, current->pid);
+                polnid = thisnid;
+                this_cpupid = cpu_pid_to_cpupid(thiscpu, current->pid);
 
                 /*
                  * Multi-stage node selection is used in conjunction
@@ -2399,8 +2401,8 @@ int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long
                  * it less likely we act on an unlikely task<->page
                  * relation.
                  */
-                last_nidpid = page_nidpid_xchg_last(page, this_nidpid);
-                if (!nidpid_pid_unset(last_nidpid) && nidpid_to_nid(last_nidpid) != polnid)
+                last_cpupid = page_cpupid_xchg_last(page, this_cpupid);
+                if (!cpupid_pid_unset(last_cpupid) && cpupid_to_nid(last_cpupid) != thisnid)
                         goto out;
 
 #ifdef CONFIG_NUMA_BALANCING
@@ -2410,7 +2412,7 @@ int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long
                  * This way a short and temporary process migration will
                  * not cause excessive memory migration.
                  */
-                if (polnid != current->numa_preferred_nid &&
+                if (thisnid != current->numa_preferred_nid &&
                                 !current->numa_migrate_seq)
                         goto out;
 #endif
diff --git a/mm/migrate.c b/mm/migrate.c
index 025d1e3d2ad2..ff537749d3b4 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -1498,7 +1498,7 @@ static struct page *alloc_misplaced_dst_page(struct page *page,
                                           __GFP_NOWARN) &
                                          ~GFP_IOFS, 0);
         if (newpage)
-                page_nidpid_xchg_last(newpage, page_nidpid_last(page));
+                page_cpupid_xchg_last(newpage, page_cpupid_last(page));
 
         return newpage;
 }
@@ -1675,7 +1675,7 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm,
         if (!new_page)
                 goto out_fail;
 
-        page_nidpid_xchg_last(new_page, page_nidpid_last(page));
+        page_cpupid_xchg_last(new_page, page_cpupid_last(page));
 
         isolated = numamigrate_isolate_page(pgdat, page);
         if (!isolated) {
diff --git a/mm/mm_init.c b/mm/mm_init.c
index 467de579784b..68562e92d50c 100644
--- a/mm/mm_init.c
+++ b/mm/mm_init.c
@@ -71,26 +71,26 @@ void __init mminit_verify_pageflags_layout(void)
         unsigned long or_mask, add_mask;
 
         shift = 8 * sizeof(unsigned long);
-        width = shift - SECTIONS_WIDTH - NODES_WIDTH - ZONES_WIDTH - LAST_NIDPID_SHIFT;
+        width = shift - SECTIONS_WIDTH - NODES_WIDTH - ZONES_WIDTH - LAST_CPUPID_SHIFT;
         mminit_dprintk(MMINIT_TRACE, "pageflags_layout_widths",
-                "Section %d Node %d Zone %d Lastnidpid %d Flags %d\n",
+                "Section %d Node %d Zone %d Lastcpupid %d Flags %d\n",
                 SECTIONS_WIDTH,
                 NODES_WIDTH,
                 ZONES_WIDTH,
-                LAST_NIDPID_WIDTH,
+                LAST_CPUPID_WIDTH,
                 NR_PAGEFLAGS);
         mminit_dprintk(MMINIT_TRACE, "pageflags_layout_shifts",
-                "Section %d Node %d Zone %d Lastnidpid %d\n",
+                "Section %d Node %d Zone %d Lastcpupid %d\n",
                 SECTIONS_SHIFT,
                 NODES_SHIFT,
                 ZONES_SHIFT,
-                LAST_NIDPID_SHIFT);
+                LAST_CPUPID_SHIFT);
         mminit_dprintk(MMINIT_TRACE, "pageflags_layout_pgshifts",
-                "Section %lu Node %lu Zone %lu Lastnidpid %lu\n",
+                "Section %lu Node %lu Zone %lu Lastcpupid %lu\n",
                 (unsigned long)SECTIONS_PGSHIFT,
                 (unsigned long)NODES_PGSHIFT,
                 (unsigned long)ZONES_PGSHIFT,
-                (unsigned long)LAST_NIDPID_PGSHIFT);
+                (unsigned long)LAST_CPUPID_PGSHIFT);
         mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodezoneid",
                 "Node/Zone ID: %lu -> %lu\n",
                 (unsigned long)(ZONEID_PGOFF + ZONEID_SHIFT),
@@ -102,9 +102,9 @@ void __init mminit_verify_pageflags_layout(void)
         mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodeflags",
                 "Node not in page flags");
 #endif
-#ifdef LAST_NIDPID_NOT_IN_PAGE_FLAGS
+#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
         mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodeflags",
-                "Last nidpid not in page flags");
+                "Last cpupid not in page flags");
 #endif
 
         if (SECTIONS_WIDTH) {
diff --git a/mm/mmzone.c b/mm/mmzone.c
index 25bb477deb26..bf34fb8556db 100644
--- a/mm/mmzone.c
+++ b/mm/mmzone.c
@@ -97,20 +97,20 @@ void lruvec_init(struct lruvec *lruvec)
                 INIT_LIST_HEAD(&lruvec->lists[lru]);
 }
 
-#if defined(CONFIG_NUMA_BALANCING) && !defined(LAST_NIDPID_NOT_IN_PAGE_FLAGS)
-int page_nidpid_xchg_last(struct page *page, int nidpid)
+#if defined(CONFIG_NUMA_BALANCING) && !defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS)
+int page_cpupid_xchg_last(struct page *page, int cpupid)
 {
         unsigned long old_flags, flags;
-        int last_nidpid;
+        int last_cpupid;
 
         do {
                 old_flags = flags = page->flags;
-                last_nidpid = page_nidpid_last(page);
+                last_cpupid = page_cpupid_last(page);
 
-                flags &= ~(LAST_NIDPID_MASK << LAST_NIDPID_PGSHIFT);
-                flags |= (nidpid & LAST_NIDPID_MASK) << LAST_NIDPID_PGSHIFT;
+                flags &= ~(LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT);
+                flags |= (cpupid & LAST_CPUPID_MASK) << LAST_CPUPID_PGSHIFT;
         } while (unlikely(cmpxchg(&page->flags, old_flags, flags) != old_flags));
 
-        return last_nidpid;
+        return last_cpupid;
 }
 #endif
diff --git a/mm/mprotect.c b/mm/mprotect.c
index 5aae39017d6d..9a74855f1241 100644
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -37,14 +37,14 @@ static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot)
 
 static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
                 unsigned long addr, unsigned long end, pgprot_t newprot,
-                int dirty_accountable, int prot_numa, bool *ret_all_same_nidpid)
+                int dirty_accountable, int prot_numa, bool *ret_all_same_cpupid)
 {
         struct mm_struct *mm = vma->vm_mm;
         pte_t *pte, oldpte;
         spinlock_t *ptl;
         unsigned long pages = 0;
-        bool all_same_nidpid = true;
-        int last_nid = -1;
+        bool all_same_cpupid = true;
+        int last_cpu = -1;
         int last_pid = -1;
 
         pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
@@ -64,17 +64,17 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
 
                                 page = vm_normal_page(vma, addr, oldpte);
                                 if (page) {
-                                        int nidpid = page_nidpid_last(page);
-                                        int this_nid = nidpid_to_nid(nidpid);
-                                        int this_pid = nidpid_to_pid(nidpid);
+                                        int cpupid = page_cpupid_last(page);
+                                        int this_cpu = cpupid_to_cpu(cpupid);
+                                        int this_pid = cpupid_to_pid(cpupid);
 
-                                        if (last_nid == -1)
-                                                last_nid = this_nid;
+                                        if (last_cpu == -1)
+                                                last_cpu = this_cpu;
                                         if (last_pid == -1)
                                                 last_pid = this_pid;
-                                        if (last_nid != this_nid ||
+                                        if (last_cpu != this_cpu ||
                                             last_pid != this_pid) {
-                                                all_same_nidpid = false;
+                                                all_same_cpupid = false;
                                         }
 
                                         if (!pte_numa(oldpte)) {
@@ -115,7 +115,7 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
         arch_leave_lazy_mmu_mode();
         pte_unmap_unlock(pte - 1, ptl);
 
-        *ret_all_same_nidpid = all_same_nidpid;
+        *ret_all_same_cpupid = all_same_cpupid;
         return pages;
 }
 
@@ -142,7 +142,7 @@ static inline unsigned long change_pmd_range(struct vm_area_struct *vma,
         pmd_t *pmd;
         unsigned long next;
         unsigned long pages = 0;
-        bool all_same_nidpid;
+        bool all_same_cpupid;
 
         pmd = pmd_offset(pud, addr);
         do {
@@ -168,7 +168,7 @@ static inline unsigned long change_pmd_range(struct vm_area_struct *vma,
                 if (pmd_none_or_clear_bad(pmd))
                         continue;
                 this_pages = change_pte_range(vma, pmd, addr, next, newprot,
-                                 dirty_accountable, prot_numa, &all_same_nidpid);
+                                 dirty_accountable, prot_numa, &all_same_cpupid);
                 pages += this_pages;
 
                 /*
@@ -177,7 +177,7 @@ static inline unsigned long change_pmd_range(struct vm_area_struct *vma,
                  * node. This allows a regular PMD to be handled as one fault
                  * and effectively batches the taking of the PTL
                  */
-                if (prot_numa && this_pages && all_same_nidpid)
+                if (prot_numa && this_pages && all_same_cpupid)
                         change_pmd_protnuma(vma->vm_mm, addr, pmd);
         } while (pmd++, addr = next, addr != end);
 
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 89bedd0e4cad..73d812f16dde 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -626,7 +626,7 @@ static inline int free_pages_check(struct page *page)
                 bad_page(page);
                 return 1;
         }
-        page_nidpid_reset_last(page);
+        page_cpupid_reset_last(page);
         if (page->flags & PAGE_FLAGS_CHECK_AT_PREP)
                 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
         return 0;
@@ -4015,7 +4015,7 @@ void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
                 mminit_verify_page_links(page, zone, nid, pfn);
                 init_page_count(page);
                 page_mapcount_reset(page);
-                page_nidpid_reset_last(page);
+                page_cpupid_reset_last(page);
                 SetPageReserved(page);
                 /*
                  * Mark the block movable so that blocks are reserved for