1 files changed, 78 insertions, 4 deletions

diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 04729647f359..71cb253368cb 100644
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -1679,6 +1679,30 @@ struct mempolicy *get_vma_policy(struct task_struct *task,
         return pol;
 }
 
+bool vma_policy_mof(struct task_struct *task, struct vm_area_struct *vma)
+{
+        struct mempolicy *pol = get_task_policy(task);
+        if (vma) {
+                if (vma->vm_ops && vma->vm_ops->get_policy) {
+                        bool ret = false;
+
+                        pol = vma->vm_ops->get_policy(vma, vma->vm_start);
+                        if (pol && (pol->flags & MPOL_F_MOF))
+                                ret = true;
+                        mpol_cond_put(pol);
+
+                        return ret;
+                } else if (vma->vm_policy) {
+                        pol = vma->vm_policy;
+                }
+        }
+
+        if (!pol)
+                return default_policy.flags & MPOL_F_MOF;
+
+        return pol->flags & MPOL_F_MOF;
+}
+
 static int apply_policy_zone(struct mempolicy *policy, enum zone_type zone)
 {
         enum zone_type dynamic_policy_zone = policy_zone;
@@ -2277,6 +2301,35 @@ static void sp_free(struct sp_node *n)
         kmem_cache_free(sn_cache, n);
 }
 
+#ifdef CONFIG_NUMA_BALANCING
+static bool numa_migrate_deferred(struct task_struct *p, int last_cpupid)
+{
+        /* Never defer a private fault */
+        if (cpupid_match_pid(p, last_cpupid))
+                return false;
+
+        if (p->numa_migrate_deferred) {
+                p->numa_migrate_deferred--;
+                return true;
+        }
+        return false;
+}
+
+static inline void defer_numa_migrate(struct task_struct *p)
+{
+        p->numa_migrate_deferred = sysctl_numa_balancing_migrate_deferred;
+}
+#else
+static inline bool numa_migrate_deferred(struct task_struct *p, int last_cpupid)
+{
+        return false;
+}
+
+static inline void defer_numa_migrate(struct task_struct *p)
+{
+}
+#endif /* CONFIG_NUMA_BALANCING */
+
 /**
  * mpol_misplaced - check whether current page node is valid in policy
  *
@@ -2300,6 +2353,8 @@ int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long
         struct zone *zone;
         int curnid = page_to_nid(page);
         unsigned long pgoff;
+        int thiscpu = raw_smp_processor_id();
+        int thisnid = cpu_to_node(thiscpu);
         int polnid = -1;
         int ret = -1;
 
@@ -2348,9 +2403,11 @@ int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long
 
         /* Migrate the page towards the node whose CPU is referencing it */
         if (pol->flags & MPOL_F_MORON) {
-                int last_nid;
+                int last_cpupid;
+                int this_cpupid;
 
-                polnid = numa_node_id();
+                polnid = thisnid;
+                this_cpupid = cpu_pid_to_cpupid(thiscpu, current->pid);
 
                 /*
                  * Multi-stage node selection is used in conjunction
@@ -2373,8 +2430,25 @@ int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long
                  * it less likely we act on an unlikely task<->page
                  * relation.
                  */
-                last_nid = page_nid_xchg_last(page, polnid);
-                if (last_nid != polnid)
+                last_cpupid = page_cpupid_xchg_last(page, this_cpupid);
+                if (!cpupid_pid_unset(last_cpupid) && cpupid_to_nid(last_cpupid) != thisnid) {
+
+                        /* See sysctl_numa_balancing_migrate_deferred comment */
+                        if (!cpupid_match_pid(current, last_cpupid))
+                                defer_numa_migrate(current);
+
+                        goto out;
+                }
+
+                /*
+                 * The quadratic filter above reduces extraneous migration
+                 * of shared pages somewhat. This code reduces it even more,
+                 * reducing the overhead of page migrations of shared pages.
+                 * This makes workloads with shared pages rely more on
+                 * "move task near its memory", and less on "move memory
+                 * towards its task", which is exactly what we want.
+                 */
+                if (numa_migrate_deferred(current, last_cpupid))
                         goto out;
         }