Merge tag 'balancenuma-v11' of git://git.kernel.org/pub/scm/linux/kernel/git/mel/linux-balancenuma

Pull Automatic NUMA Balancing bare-bones from Mel Gorman:
 "There are three implementations for NUMA balancing, this tree
  (balancenuma), numacore which has been developed in tip/master and
  autonuma which is in aa.git.

  In almost all respects balancenuma is the dumbest of the three because
  its main impact is on the VM side with no attempt to be smart about
  scheduling.  In the interest of getting the ball rolling, it would be
  desirable to see this much merged for 3.8 with the view to building
  scheduler smarts on top and adapting the VM where required for 3.9.

  The most recent set of comparisons available from different people are

    mel:    https://lkml.org/lkml/2012/12/9/108
    mingo:  https://lkml.org/lkml/2012/12/7/331
    tglx:   https://lkml.org/lkml/2012/12/10/437
    srikar: https://lkml.org/lkml/2012/12/10/397

  The results are a mixed bag.  In my own tests, balancenuma does
  reasonably well.  It's dumb as rocks and does not regress against
  mainline.  On the other hand, Ingo's tests shows that balancenuma is
  incapable of converging for this workloads driven by perf which is bad
  but is potentially explained by the lack of scheduler smarts.  Thomas'
  results show balancenuma improves on mainline but falls far short of
  numacore or autonuma.  Srikar's results indicate we all suffer on a
  large machine with imbalanced node sizes.

  My own testing showed that recent numacore results have improved
  dramatically, particularly in the last week but not universally.
  We've butted heads heavily on system CPU usage and high levels of
  migration even when it shows that overall performance is better.
  There are also cases where it regresses.  Of interest is that for
  specjbb in some configurations it will regress for lower numbers of
  warehouses and show gains for higher numbers which is not reported by
  the tool by default and sometimes missed in treports.  Recently I
  reported for numacore that the JVM was crashing with
  NullPointerExceptions but currently it's unclear what the source of
  this problem is.  Initially I thought it was in how numacore batch
  handles PTEs but I'm no longer think this is the case.  It's possible
  numacore is just able to trigger it due to higher rates of migration.

  These reports were quite late in the cycle so I/we would like to start
  with this tree as it contains much of the code we can agree on and has
  not changed significantly over the last 2-3 weeks."

* tag 'balancenuma-v11' of git://git.kernel.org/pub/scm/linux/kernel/git/mel/linux-balancenuma: (50 commits)
  mm/rmap, migration: Make rmap_walk_anon() and try_to_unmap_anon() more scalable
  mm/rmap: Convert the struct anon_vma::mutex to an rwsem
  mm: migrate: Account a transhuge page properly when rate limiting
  mm: numa: Account for failed allocations and isolations as migration failures
  mm: numa: Add THP migration for the NUMA working set scanning fault case build fix
  mm: numa: Add THP migration for the NUMA working set scanning fault case.
  mm: sched: numa: Delay PTE scanning until a task is scheduled on a new node
  mm: sched: numa: Control enabling and disabling of NUMA balancing if !SCHED_DEBUG
  mm: sched: numa: Control enabling and disabling of NUMA balancing
  mm: sched: Adapt the scanning rate if a NUMA hinting fault does not migrate
  mm: numa: Use a two-stage filter to restrict pages being migrated for unlikely task<->node relationships
  mm: numa: migrate: Set last_nid on newly allocated page
  mm: numa: split_huge_page: Transfer last_nid on tail page
  mm: numa: Introduce last_nid to the page frame
  sched: numa: Slowly increase the scanning period as NUMA faults are handled
  mm: numa: Rate limit setting of pte_numa if node is saturated
  mm: numa: Rate limit the amount of memory that is migrated between nodes
  mm: numa: Structures for Migrate On Fault per NUMA migration rate limiting
  mm: numa: Migrate pages handled during a pmd_numa hinting fault
  mm: numa: Migrate on reference policy
  ...

1 files changed, 111 insertions, 24 deletions

diff --git a/mm/mprotect.c b/mm/mprotect.c
index e8c3938db6fa..3dca970367db 100644
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -35,12 +35,16 @@ static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot)
 }
 #endif
 
-static void change_pte_range(struct mm_struct *mm, pmd_t *pmd,
+static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
                 unsigned long addr, unsigned long end, pgprot_t newprot,
-                int dirty_accountable)
+                int dirty_accountable, int prot_numa, bool *ret_all_same_node)
 {
+        struct mm_struct *mm = vma->vm_mm;
         pte_t *pte, oldpte;
         spinlock_t *ptl;
+        unsigned long pages = 0;
+        bool all_same_node = true;
+        int last_nid = -1;
 
         pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
         arch_enter_lazy_mmu_mode();
@@ -48,17 +52,43 @@ static void change_pte_range(struct mm_struct *mm, pmd_t *pmd,
                 oldpte = *pte;
                 if (pte_present(oldpte)) {
                         pte_t ptent;
+                        bool updated = false;
 
                         ptent = ptep_modify_prot_start(mm, addr, pte);
-                        ptent = pte_modify(ptent, newprot);
+                        if (!prot_numa) {
+                                ptent = pte_modify(ptent, newprot);
+                                updated = true;
+                        } else {
+                                struct page *page;
+
+                                page = vm_normal_page(vma, addr, oldpte);
+                                if (page) {
+                                        int this_nid = page_to_nid(page);
+                                        if (last_nid == -1)
+                                                last_nid = this_nid;
+                                        if (last_nid != this_nid)
+                                                all_same_node = false;
+
+                                        /* only check non-shared pages */
+                                        if (!pte_numa(oldpte) &&
+                                            page_mapcount(page) == 1) {
+                                                ptent = pte_mknuma(ptent);
+                                                updated = true;
+                                        }
+                                }
+                        }
 
                         /*
                          * Avoid taking write faults for pages we know to be
                          * dirty.
                          */
-                        if (dirty_accountable && pte_dirty(ptent))
+                        if (dirty_accountable && pte_dirty(ptent)) {
                                 ptent = pte_mkwrite(ptent);
+                                updated = true;
+                        }
 
+                        if (updated)
+                                pages++;
                         ptep_modify_prot_commit(mm, addr, pte, ptent);
                 } else if (IS_ENABLED(CONFIG_MIGRATION) && !pte_file(oldpte)) {
                         swp_entry_t entry = pte_to_swp_entry(oldpte);
@@ -72,18 +102,40 @@ static void change_pte_range(struct mm_struct *mm, pmd_t *pmd,
                                 set_pte_at(mm, addr, pte,
                                         swp_entry_to_pte(entry));
                         }
+                        pages++;
                 }
         } while (pte++, addr += PAGE_SIZE, addr != end);
         arch_leave_lazy_mmu_mode();
         pte_unmap_unlock(pte - 1, ptl);
+
+        *ret_all_same_node = all_same_node;
+        return pages;
 }
 
-static inline void change_pmd_range(struct vm_area_struct *vma, pud_t *pud,
+#ifdef CONFIG_NUMA_BALANCING
+static inline void change_pmd_protnuma(struct mm_struct *mm, unsigned long addr,
+                pmd_t *pmd)
+{
+        spin_lock(&mm->page_table_lock);
+        set_pmd_at(mm, addr & PMD_MASK, pmd, pmd_mknuma(*pmd));
+        spin_unlock(&mm->page_table_lock);
+}
+#else
+static inline void change_pmd_protnuma(struct mm_struct *mm, unsigned long addr,
+                pmd_t *pmd)
+{
+        BUG();
+}
+#endif /* CONFIG_NUMA_BALANCING */
+
+static inline unsigned long change_pmd_range(struct vm_area_struct *vma, pud_t *pud,
                 unsigned long addr, unsigned long end, pgprot_t newprot,
-                int dirty_accountable)
+                int dirty_accountable, int prot_numa)
 {
         pmd_t *pmd;
         unsigned long next;
+        unsigned long pages = 0;
+        bool all_same_node;
 
         pmd = pmd_offset(pud, addr);
         do {
@@ -91,42 +143,59 @@ static inline void change_pmd_range(struct vm_area_struct *vma, pud_t *pud,
                 if (pmd_trans_huge(*pmd)) {
                         if (next - addr != HPAGE_PMD_SIZE)
                                 split_huge_page_pmd(vma, addr, pmd);
-                        else if (change_huge_pmd(vma, pmd, addr, newprot))
+                        else if (change_huge_pmd(vma, pmd, addr, newprot, prot_numa)) {
+                                pages += HPAGE_PMD_NR;
                                 continue;
+                        }
                         /* fall through */
                 }
                 if (pmd_none_or_clear_bad(pmd))
                         continue;
-                change_pte_range(vma->vm_mm, pmd, addr, next, newprot,
-                                 dirty_accountable);
+                pages += change_pte_range(vma, pmd, addr, next, newprot,
+                                 dirty_accountable, prot_numa, &all_same_node);
+
+                /*
+                 * If we are changing protections for NUMA hinting faults then
+                 * set pmd_numa if the examined pages were all on the same
+                 * node. This allows a regular PMD to be handled as one fault
+                 * and effectively batches the taking of the PTL
+                 */
+                if (prot_numa && all_same_node)
+                        change_pmd_protnuma(vma->vm_mm, addr, pmd);
         } while (pmd++, addr = next, addr != end);
+
+        return pages;
 }
 
-static inline void change_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
+static inline unsigned long change_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
                 unsigned long addr, unsigned long end, pgprot_t newprot,
-                int dirty_accountable)
+                int dirty_accountable, int prot_numa)
 {
         pud_t *pud;
         unsigned long next;
+        unsigned long pages = 0;
 
         pud = pud_offset(pgd, addr);
         do {
                 next = pud_addr_end(addr, end);
                 if (pud_none_or_clear_bad(pud))
                         continue;
-                change_pmd_range(vma, pud, addr, next, newprot,
-                                 dirty_accountable);
+                pages += change_pmd_range(vma, pud, addr, next, newprot,
+                                 dirty_accountable, prot_numa);
         } while (pud++, addr = next, addr != end);
+
+        return pages;
 }
 
-static void change_protection(struct vm_area_struct *vma,
+static unsigned long change_protection_range(struct vm_area_struct *vma,
                 unsigned long addr, unsigned long end, pgprot_t newprot,
-                int dirty_accountable)
+                int dirty_accountable, int prot_numa)
 {
         struct mm_struct *mm = vma->vm_mm;
         pgd_t *pgd;
         unsigned long next;
         unsigned long start = addr;
+        unsigned long pages = 0;
 
         BUG_ON(addr >= end);
         pgd = pgd_offset(mm, addr);
@@ -135,10 +204,32 @@ static void change_protection(struct vm_area_struct *vma,
                 next = pgd_addr_end(addr, end);
                 if (pgd_none_or_clear_bad(pgd))
                         continue;
-                change_pud_range(vma, pgd, addr, next, newprot,
-                                 dirty_accountable);
+                pages += change_pud_range(vma, pgd, addr, next, newprot,
+                                 dirty_accountable, prot_numa);
         } while (pgd++, addr = next, addr != end);
-        flush_tlb_range(vma, start, end);
+
+        /* Only flush the TLB if we actually modified any entries: */
+        if (pages)
+                flush_tlb_range(vma, start, end);
+
+        return pages;
+}
+
+unsigned long change_protection(struct vm_area_struct *vma, unsigned long start,
+                       unsigned long end, pgprot_t newprot,
+                       int dirty_accountable, int prot_numa)
+{
+        struct mm_struct *mm = vma->vm_mm;
+        unsigned long pages;
+
+        mmu_notifier_invalidate_range_start(mm, start, end);
+        if (is_vm_hugetlb_page(vma))
+                pages = hugetlb_change_protection(vma, start, end, newprot);
+        else
+                pages = change_protection_range(vma, start, end, newprot, dirty_accountable, prot_numa);
+        mmu_notifier_invalidate_range_end(mm, start, end);
+
+        return pages;
 }
 
 int
@@ -213,12 +304,8 @@ success:
                 dirty_accountable = 1;
         }
 
-        mmu_notifier_invalidate_range_start(mm, start, end);
-        if (is_vm_hugetlb_page(vma))
-                hugetlb_change_protection(vma, start, end, vma->vm_page_prot);
-        else
-                change_protection(vma, start, end, vma->vm_page_prot, dirty_accountable);
-        mmu_notifier_invalidate_range_end(mm, start, end);
+        change_protection(vma, start, end, vma->vm_page_prot, dirty_accountable, 0);
+
         vm_stat_account(mm, oldflags, vma->vm_file, -nrpages);
         vm_stat_account(mm, newflags, vma->vm_file, nrpages);
         perf_event_mmap(vma);