3 files changed, 71 insertions, 28 deletions

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 5fb0cfb43ecf..5ab3b89fc33e 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1589,6 +1589,8 @@ extern void task_numa_fault(int last_node, int node, int pages, int flags);
 extern pid_t task_numa_group_id(struct task_struct *p);
 extern void set_numabalancing_state(bool enabled);
 extern void task_numa_free(struct task_struct *p);
+extern bool should_numa_migrate_memory(struct task_struct *p, struct page *page,
+                                        int src_nid, int dst_cpu);
 #else
 static inline void task_numa_fault(int last_node, int node, int pages,
                                    int flags)
@@ -1604,6 +1606,11 @@ static inline void set_numabalancing_state(bool enabled)
 static inline void task_numa_free(struct task_struct *p)
 {
 }
+static inline bool should_numa_migrate_memory(struct task_struct *p,
+                                struct page *page, int src_nid, int dst_cpu)
+{
+        return true;
+}
 #endif
 
 static inline struct pid *task_pid(struct task_struct *task)
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 1ee921f1ec35..eeabb33f349e 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -954,6 +954,69 @@ static inline unsigned long group_weight(struct task_struct *p, int nid)
         return 1000 * group_faults(p, nid) / p->numa_group->total_faults;
 }
 
+bool should_numa_migrate_memory(struct task_struct *p, struct page * page,
+                                int src_nid, int dst_cpu)
+{
+        struct numa_group *ng = p->numa_group;
+        int dst_nid = cpu_to_node(dst_cpu);
+        int last_cpupid, this_cpupid;
+
+        this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid);
+
+        /*
+         * Multi-stage node selection is used in conjunction with a periodic
+         * migration fault to build a temporal task<->page relation. By using
+         * a two-stage filter we remove short/unlikely relations.
+         *
+         * Using P(p) ~ n_p / n_t as per frequentist probability, we can equate
+         * a task's usage of a particular page (n_p) per total usage of this
+         * page (n_t) (in a given time-span) to a probability.
+         *
+         * Our periodic faults will sample this probability and getting the
+         * same result twice in a row, given these samples are fully
+         * independent, is then given by P(n)^2, provided our sample period
+         * is sufficiently short compared to the usage pattern.
+         *
+         * This quadric squishes small probabilities, making it less likely we
+         * act on an unlikely task<->page relation.
+         */
+        last_cpupid = page_cpupid_xchg_last(page, this_cpupid);
+        if (!cpupid_pid_unset(last_cpupid) &&
+                                cpupid_to_nid(last_cpupid) != dst_nid)
+                return false;
+
+        /* Always allow migrate on private faults */
+        if (cpupid_match_pid(p, last_cpupid))
+                return true;
+
+        /* A shared fault, but p->numa_group has not been set up yet. */
+        if (!ng)
+                return true;
+
+        /*
+         * Do not migrate if the destination is not a node that
+         * is actively used by this numa group.
+         */
+        if (!node_isset(dst_nid, ng->active_nodes))
+                return false;
+
+        /*
+         * Source is a node that is not actively used by this
+         * numa group, while the destination is. Migrate.
+         */
+        if (!node_isset(src_nid, ng->active_nodes))
+                return true;
+
+        /*
+         * Both source and destination are nodes in active
+         * use by this numa group. Maximize memory bandwidth
+         * by migrating from more heavily used groups, to less
+         * heavily used ones, spreading the load around.
+         * Use a 1/4 hysteresis to avoid spurious page movement.
+         */
+        return group_faults(p, dst_nid) < (group_faults(p, src_nid) * 3 / 4);
+}
+
 static unsigned long weighted_cpuload(const int cpu);
 static unsigned long source_load(int cpu, int type);
 static unsigned long target_load(int cpu, int type);
diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 68d5c7f7164e..784c11ef7719 100644
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -2377,37 +2377,10 @@ int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long
 
         /* Migrate the page towards the node whose CPU is referencing it */
         if (pol->flags & MPOL_F_MORON) {
-                int last_cpupid;
-                int this_cpupid;
-
                 polnid = thisnid;
-                this_cpupid = cpu_pid_to_cpupid(thiscpu, current->pid);
 
-                /*
-                 * Multi-stage node selection is used in conjunction
-                 * with a periodic migration fault to build a temporal
-                 * task<->page relation. By using a two-stage filter we
-                 * remove short/unlikely relations.
-                 *
-                 * Using P(p) ~ n_p / n_t as per frequentist
-                 * probability, we can equate a task's usage of a
-                 * particular page (n_p) per total usage of this
-                 * page (n_t) (in a given time-span) to a probability.
-                 *
-                 * Our periodic faults will sample this probability and
-                 * getting the same result twice in a row, given these
-                 * samples are fully independent, is then given by
-                 * P(n)^2, provided our sample period is sufficiently
-                 * short compared to the usage pattern.
-                 *
-                 * This quadric squishes small probabilities, making
-                 * it less likely we act on an unlikely task<->page
-                 * relation.
-                 */
-                last_cpupid = page_cpupid_xchg_last(page, this_cpupid);
-                if (!cpupid_pid_unset(last_cpupid) && cpupid_to_nid(last_cpupid) != thisnid) {
+                if (!should_numa_migrate_memory(current, page, curnid, thiscpu))
                         goto out;
-                }
         }
 
         if (curnid != polnid)