1 files changed, 239 insertions, 37 deletions

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index ffeaa4105e48..0d4632f7799b 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -141,9 +141,9 @@ static inline void update_load_set(struct load_weight *lw, unsigned long w)
  *
  * This idea comes from the SD scheduler of Con Kolivas:
  */
-static int get_update_sysctl_factor(void)
+static unsigned int get_update_sysctl_factor(void)
 {
-        unsigned int cpus = min_t(int, num_online_cpus(), 8);
+        unsigned int cpus = min_t(unsigned int, num_online_cpus(), 8);
         unsigned int factor;
 
         switch (sysctl_sched_tunable_scaling) {
@@ -576,7 +576,7 @@ int sched_proc_update_handler(struct ctl_table *table, int write,
                 loff_t *ppos)
 {
         int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
-        int factor = get_update_sysctl_factor();
+        unsigned int factor = get_update_sysctl_factor();
 
         if (ret || !write)
                 return ret;
@@ -834,7 +834,7 @@ static unsigned int task_nr_scan_windows(struct task_struct *p)
 
 static unsigned int task_scan_min(struct task_struct *p)
 {
-        unsigned int scan_size = ACCESS_ONCE(sysctl_numa_balancing_scan_size);
+        unsigned int scan_size = READ_ONCE(sysctl_numa_balancing_scan_size);
         unsigned int scan, floor;
         unsigned int windows = 1;
 
@@ -1794,7 +1794,12 @@ static void task_numa_placement(struct task_struct *p)
         u64 runtime, period;
         spinlock_t *group_lock = NULL;
 
-        seq = ACCESS_ONCE(p->mm->numa_scan_seq);
+        /*
+         * The p->mm->numa_scan_seq field gets updated without
+         * exclusive access. Use READ_ONCE() here to ensure
+         * that the field is read in a single access:
+         */
+        seq = READ_ONCE(p->mm->numa_scan_seq);
         if (p->numa_scan_seq == seq)
                 return;
         p->numa_scan_seq = seq;
@@ -1938,7 +1943,7 @@ static void task_numa_group(struct task_struct *p, int cpupid, int flags,
         }
 
         rcu_read_lock();
-        tsk = ACCESS_ONCE(cpu_rq(cpu)->curr);
+        tsk = READ_ONCE(cpu_rq(cpu)->curr);
 
         if (!cpupid_match_pid(tsk, cpupid))
                 goto no_join;
@@ -2107,7 +2112,15 @@ void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags)
 
 static void reset_ptenuma_scan(struct task_struct *p)
 {
-        ACCESS_ONCE(p->mm->numa_scan_seq)++;
+        /*
+         * We only did a read acquisition of the mmap sem, so
+         * p->mm->numa_scan_seq is written to without exclusive access
+         * and the update is not guaranteed to be atomic. That's not
+         * much of an issue though, since this is just used for
+         * statistical sampling. Use READ_ONCE/WRITE_ONCE, which are not
+         * expensive, to avoid any form of compiler optimizations:
+         */
+        WRITE_ONCE(p->mm->numa_scan_seq, READ_ONCE(p->mm->numa_scan_seq) + 1);
         p->mm->numa_scan_offset = 0;
 }
 
@@ -4323,6 +4336,189 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
 }
 
 #ifdef CONFIG_SMP
+
+/*
+ * per rq 'load' arrray crap; XXX kill this.
+ */
+
+/*
+ * The exact cpuload at various idx values, calculated at every tick would be
+ * load = (2^idx - 1) / 2^idx * load + 1 / 2^idx * cur_load
+ *
+ * If a cpu misses updates for n-1 ticks (as it was idle) and update gets called
+ * on nth tick when cpu may be busy, then we have:
+ * load = ((2^idx - 1) / 2^idx)^(n-1) * load
+ * load = (2^idx - 1) / 2^idx) * load + 1 / 2^idx * cur_load
+ *
+ * decay_load_missed() below does efficient calculation of
+ * load = ((2^idx - 1) / 2^idx)^(n-1) * load
+ * avoiding 0..n-1 loop doing load = ((2^idx - 1) / 2^idx) * load
+ *
+ * The calculation is approximated on a 128 point scale.
+ * degrade_zero_ticks is the number of ticks after which load at any
+ * particular idx is approximated to be zero.
+ * degrade_factor is a precomputed table, a row for each load idx.
+ * Each column corresponds to degradation factor for a power of two ticks,
+ * based on 128 point scale.
+ * Example:
+ * row 2, col 3 (=12) says that the degradation at load idx 2 after
+ * 8 ticks is 12/128 (which is an approximation of exact factor 3^8/4^8).
+ *
+ * With this power of 2 load factors, we can degrade the load n times
+ * by looking at 1 bits in n and doing as many mult/shift instead of
+ * n mult/shifts needed by the exact degradation.
+ */
+#define DEGRADE_SHIFT           7
+static const unsigned char
+                degrade_zero_ticks[CPU_LOAD_IDX_MAX] = {0, 8, 32, 64, 128};
+static const unsigned char
+                degrade_factor[CPU_LOAD_IDX_MAX][DEGRADE_SHIFT + 1] = {
+                                        {0, 0, 0, 0, 0, 0, 0, 0},
+                                        {64, 32, 8, 0, 0, 0, 0, 0},
+                                        {96, 72, 40, 12, 1, 0, 0},
+                                        {112, 98, 75, 43, 15, 1, 0},
+                                        {120, 112, 98, 76, 45, 16, 2} };
+
+/*
+ * Update cpu_load for any missed ticks, due to tickless idle. The backlog
+ * would be when CPU is idle and so we just decay the old load without
+ * adding any new load.
+ */
+static unsigned long
+decay_load_missed(unsigned long load, unsigned long missed_updates, int idx)
+{
+        int j = 0;
+
+        if (!missed_updates)
+                return load;
+
+        if (missed_updates >= degrade_zero_ticks[idx])
+                return 0;
+
+        if (idx == 1)
+                return load >> missed_updates;
+
+        while (missed_updates) {
+                if (missed_updates % 2)
+                        load = (load * degrade_factor[idx][j]) >> DEGRADE_SHIFT;
+
+                missed_updates >>= 1;
+                j++;
+        }
+        return load;
+}
+
+/*
+ * Update rq->cpu_load[] statistics. This function is usually called every
+ * scheduler tick (TICK_NSEC). With tickless idle this will not be called
+ * every tick. We fix it up based on jiffies.
+ */
+static void __update_cpu_load(struct rq *this_rq, unsigned long this_load,
+                              unsigned long pending_updates)
+{
+        int i, scale;
+
+        this_rq->nr_load_updates++;
+
+        /* Update our load: */
+        this_rq->cpu_load[0] = this_load; /* Fasttrack for idx 0 */
+        for (i = 1, scale = 2; i < CPU_LOAD_IDX_MAX; i++, scale += scale) {
+                unsigned long old_load, new_load;
+
+                /* scale is effectively 1 << i now, and >> i divides by scale */
+
+                old_load = this_rq->cpu_load[i];
+                old_load = decay_load_missed(old_load, pending_updates - 1, i);
+                new_load = this_load;
+                /*
+                 * Round up the averaging division if load is increasing. This
+                 * prevents us from getting stuck on 9 if the load is 10, for
+                 * example.
+                 */
+                if (new_load > old_load)
+                        new_load += scale - 1;
+
+                this_rq->cpu_load[i] = (old_load * (scale - 1) + new_load) >> i;
+        }
+
+        sched_avg_update(this_rq);
+}
+
+#ifdef CONFIG_NO_HZ_COMMON
+/*
+ * There is no sane way to deal with nohz on smp when using jiffies because the
+ * cpu doing the jiffies update might drift wrt the cpu doing the jiffy reading
+ * causing off-by-one errors in observed deltas; {0,2} instead of {1,1}.
+ *
+ * Therefore we cannot use the delta approach from the regular tick since that
+ * would seriously skew the load calculation. However we'll make do for those
+ * updates happening while idle (nohz_idle_balance) or coming out of idle
+ * (tick_nohz_idle_exit).
+ *
+ * This means we might still be one tick off for nohz periods.
+ */
+
+/*
+ * Called from nohz_idle_balance() to update the load ratings before doing the
+ * idle balance.
+ */
+static void update_idle_cpu_load(struct rq *this_rq)
+{
+        unsigned long curr_jiffies = READ_ONCE(jiffies);
+        unsigned long load = this_rq->cfs.runnable_load_avg;
+        unsigned long pending_updates;
+
+        /*
+         * bail if there's load or we're actually up-to-date.
+         */
+        if (load || curr_jiffies == this_rq->last_load_update_tick)
+                return;
+
+        pending_updates = curr_jiffies - this_rq->last_load_update_tick;
+        this_rq->last_load_update_tick = curr_jiffies;
+
+        __update_cpu_load(this_rq, load, pending_updates);
+}
+
+/*
+ * Called from tick_nohz_idle_exit() -- try and fix up the ticks we missed.
+ */
+void update_cpu_load_nohz(void)
+{
+        struct rq *this_rq = this_rq();
+        unsigned long curr_jiffies = READ_ONCE(jiffies);
+        unsigned long pending_updates;
+
+        if (curr_jiffies == this_rq->last_load_update_tick)
+                return;
+
+        raw_spin_lock(&this_rq->lock);
+        pending_updates = curr_jiffies - this_rq->last_load_update_tick;
+        if (pending_updates) {
+                this_rq->last_load_update_tick = curr_jiffies;
+                /*
+                 * We were idle, this means load 0, the current load might be
+                 * !0 due to remote wakeups and the sort.
+                 */
+                __update_cpu_load(this_rq, 0, pending_updates);
+        }
+        raw_spin_unlock(&this_rq->lock);
+}
+#endif /* CONFIG_NO_HZ */
+
+/*
+ * Called from scheduler_tick()
+ */
+void update_cpu_load_active(struct rq *this_rq)
+{
+        unsigned long load = this_rq->cfs.runnable_load_avg;
+        /*
+         * See the mess around update_idle_cpu_load() / update_cpu_load_nohz().
+         */
+        this_rq->last_load_update_tick = jiffies;
+        __update_cpu_load(this_rq, load, 1);
+}
+
 /* Used instead of source_load when we know the type == 0 */
 static unsigned long weighted_cpuload(const int cpu)
 {
@@ -4375,7 +4571,7 @@ static unsigned long capacity_orig_of(int cpu)
 static unsigned long cpu_avg_load_per_task(int cpu)
 {
         struct rq *rq = cpu_rq(cpu);
-        unsigned long nr_running = ACCESS_ONCE(rq->cfs.h_nr_running);
+        unsigned long nr_running = READ_ONCE(rq->cfs.h_nr_running);
         unsigned long load_avg = rq->cfs.runnable_load_avg;
 
         if (nr_running)
@@ -5467,10 +5663,15 @@ static int task_hot(struct task_struct *p, struct lb_env *env)
 }
 
 #ifdef CONFIG_NUMA_BALANCING
-/* Returns true if the destination node has incurred more faults */
+/*
+ * Returns true if the destination node is the preferred node.
+ * Needs to match fbq_classify_rq(): if there is a runnable task
+ * that is not on its preferred node, we should identify it.
+ */
 static bool migrate_improves_locality(struct task_struct *p, struct lb_env *env)
 {
         struct numa_group *numa_group = rcu_dereference(p->numa_group);
+        unsigned long src_faults, dst_faults;
         int src_nid, dst_nid;
 
         if (!sched_feat(NUMA_FAVOUR_HIGHER) || !p->numa_faults ||
@@ -5484,29 +5685,30 @@ static bool migrate_improves_locality(struct task_struct *p, struct lb_env *env)
         if (src_nid == dst_nid)
                 return false;
 
-        if (numa_group) {
-                /* Task is already in the group's interleave set. */
-                if (node_isset(src_nid, numa_group->active_nodes))
-                        return false;
-
-                /* Task is moving into the group's interleave set. */
-                if (node_isset(dst_nid, numa_group->active_nodes))
-                        return true;
-
-                return group_faults(p, dst_nid) > group_faults(p, src_nid);
-        }
-
         /* Encourage migration to the preferred node. */
         if (dst_nid == p->numa_preferred_nid)
                 return true;
 
-        return task_faults(p, dst_nid) > task_faults(p, src_nid);
+        /* Migrating away from the preferred node is bad. */
+        if (src_nid == p->numa_preferred_nid)
+                return false;
+
+        if (numa_group) {
+                src_faults = group_faults(p, src_nid);
+                dst_faults = group_faults(p, dst_nid);
+        } else {
+                src_faults = task_faults(p, src_nid);
+                dst_faults = task_faults(p, dst_nid);
+        }
+
+        return dst_faults > src_faults;
 }
 
 
 static bool migrate_degrades_locality(struct task_struct *p, struct lb_env *env)
 {
         struct numa_group *numa_group = rcu_dereference(p->numa_group);
+        unsigned long src_faults, dst_faults;
         int src_nid, dst_nid;
 
         if (!sched_feat(NUMA) || !sched_feat(NUMA_RESIST_LOWER))
@@ -5521,23 +5723,23 @@ static bool migrate_degrades_locality(struct task_struct *p, struct lb_env *env)
         if (src_nid == dst_nid)
                 return false;
 
-        if (numa_group) {
-                /* Task is moving within/into the group's interleave set. */
-                if (node_isset(dst_nid, numa_group->active_nodes))
-                        return false;
+        /* Migrating away from the preferred node is bad. */
+        if (src_nid == p->numa_preferred_nid)
+                return true;
 
-                /* Task is moving out of the group's interleave set. */
-                if (node_isset(src_nid, numa_group->active_nodes))
-                        return true;
+        /* Encourage migration to the preferred node. */
+        if (dst_nid == p->numa_preferred_nid)
+                return false;
 
-                return group_faults(p, dst_nid) < group_faults(p, src_nid);
+        if (numa_group) {
+                src_faults = group_faults(p, src_nid);
+                dst_faults = group_faults(p, dst_nid);
+        } else {
+                src_faults = task_faults(p, src_nid);
+                dst_faults = task_faults(p, dst_nid);
         }
 
-        /* Migrating away from the preferred node is always bad. */
-        if (src_nid == p->numa_preferred_nid)
-                return true;
-
-        return task_faults(p, dst_nid) < task_faults(p, src_nid);
+        return dst_faults < src_faults;
 }
 
 #else
@@ -6037,8 +6239,8 @@ static unsigned long scale_rt_capacity(int cpu)
          * Since we're reading these variables without serialization make sure
          * we read them once before doing sanity checks on them.
          */
-        age_stamp = ACCESS_ONCE(rq->age_stamp);
-        avg = ACCESS_ONCE(rq->rt_avg);
+        age_stamp = READ_ONCE(rq->age_stamp);
+        avg = READ_ONCE(rq->rt_avg);
         delta = __rq_clock_broken(rq) - age_stamp;
 
         if (unlikely(delta < 0))