mm: numa: Migrate pages handled during a pmd_numa hinting fault

To say that the PMD handling code was incorrectly transferred from autonuma
is an understatement. The intention was to handle a PMDs worth of pages
in the same fault and effectively batch the taking of the PTL and page
migration. The copied version instead has the impact of clearing a number
of pte_numa PTE entries and whether any page migration takes place depends
on racing. This just happens to work in some cases.

This patch handles pte_numa faults in batch when a pmd_numa fault is
handled. The pages are migrated if they are currently misplaced.
Essentially this is making an assumption that NUMA locality is
on a PMD boundary but that could be addressed by only setting
pmd_numa if all the pages within that PMD are on the same node
if necessary.

Signed-off-by: Mel Gorman <mgorman@suse.de>

1 files changed, 20 insertions, 5 deletions

diff --git a/mm/mprotect.c b/mm/mprotect.c
index 7ef6ae964e8f..dce6fb48edc4 100644
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -37,12 +37,14 @@ static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot)
 
 static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
                 unsigned long addr, unsigned long end, pgprot_t newprot,
-                int dirty_accountable, int prot_numa)
+                int dirty_accountable, int prot_numa, bool *ret_all_same_node)
 {
         struct mm_struct *mm = vma->vm_mm;
         pte_t *pte, oldpte;
         spinlock_t *ptl;
         unsigned long pages = 0;
+        bool all_same_node = true;
+        int last_nid = -1;
 
         pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
         arch_enter_lazy_mmu_mode();
@@ -61,6 +63,12 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
 
                                 page = vm_normal_page(vma, addr, oldpte);
                                 if (page) {
+                                        int this_nid = page_to_nid(page);
+                                        if (last_nid == -1)
+                                                last_nid = this_nid;
+                                        if (last_nid != this_nid)
+                                                all_same_node = false;
+
                                         /* only check non-shared pages */
                                         if (!pte_numa(oldpte) &&
                                             page_mapcount(page) == 1) {
@@ -81,7 +89,6 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
 
                         if (updated)
                                 pages++;
-
                         ptep_modify_prot_commit(mm, addr, pte, ptent);
                 } else if (IS_ENABLED(CONFIG_MIGRATION) && !pte_file(oldpte)) {
                         swp_entry_t entry = pte_to_swp_entry(oldpte);
@@ -101,6 +108,7 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
         arch_leave_lazy_mmu_mode();
         pte_unmap_unlock(pte - 1, ptl);
 
+        *ret_all_same_node = all_same_node;
         return pages;
 }
 
@@ -127,6 +135,7 @@ static inline unsigned long change_pmd_range(struct vm_area_struct *vma, pud_t *
         pmd_t *pmd;
         unsigned long next;
         unsigned long pages = 0;
+        bool all_same_node;
 
         pmd = pmd_offset(pud, addr);
         do {
@@ -143,9 +152,15 @@ static inline unsigned long change_pmd_range(struct vm_area_struct *vma, pud_t *
                 if (pmd_none_or_clear_bad(pmd))
                         continue;
                 pages += change_pte_range(vma, pmd, addr, next, newprot,
-                                 dirty_accountable, prot_numa);
-
-                if (prot_numa)
+                                 dirty_accountable, prot_numa, &all_same_node);
+
+                /*
+                 * If we are changing protections for NUMA hinting faults then
+                 * set pmd_numa if the examined pages were all on the same
+                 * node. This allows a regular PMD to be handled as one fault
+                 * and effectively batches the taking of the PTL
+                 */
+                if (prot_numa && all_same_node)
                         change_pmd_protnuma(vma->vm_mm, addr, pmd);
         } while (pmd++, addr = next, addr != end);