1 files changed, 50 insertions, 68 deletions

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index c219bf8d704c..6b800a14b990 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -597,7 +597,7 @@ calc_delta_fair(unsigned long delta, struct sched_entity *se)
 /*
  * The idea is to set a period in which each task runs once.
  *
- * When there are too many tasks (sysctl_sched_nr_latency) we have to stretch
+ * When there are too many tasks (sched_nr_latency) we have to stretch
  * this period because otherwise the slices get too small.
  *
  * p = (nr <= nl) ? l : l*nr/nl
@@ -2052,7 +2052,7 @@ static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
         hrtimer_cancel(&cfs_b->slack_timer);
 }
 
-void unthrottle_offline_cfs_rqs(struct rq *rq)
+static void unthrottle_offline_cfs_rqs(struct rq *rq)
 {
         struct cfs_rq *cfs_rq;
 
@@ -2106,7 +2106,7 @@ static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
         return NULL;
 }
 static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
-void unthrottle_offline_cfs_rqs(struct rq *rq) {}
+static inline void unthrottle_offline_cfs_rqs(struct rq *rq) {}
 
 #endif /* CONFIG_CFS_BANDWIDTH */
 
@@ -2637,6 +2637,8 @@ static int select_idle_sibling(struct task_struct *p, int target)
         int cpu = smp_processor_id();
         int prev_cpu = task_cpu(p);
         struct sched_domain *sd;
+        struct sched_group *sg;
+        int i;
 
         /*
          * If the task is going to be woken-up on this cpu and if it is
@@ -2653,17 +2655,29 @@ static int select_idle_sibling(struct task_struct *p, int target)
                 return prev_cpu;
 
         /*
-         * Otherwise, check assigned siblings to find an elegible idle cpu.
+         * Otherwise, iterate the domains and find an elegible idle cpu.
          */
         sd = rcu_dereference(per_cpu(sd_llc, target));
-
         for_each_lower_domain(sd) {
-                if (!cpumask_test_cpu(sd->idle_buddy, tsk_cpus_allowed(p)))
-                        continue;
-                if (idle_cpu(sd->idle_buddy))
-                        return sd->idle_buddy;
-        }
+                sg = sd->groups;
+                do {
+                        if (!cpumask_intersects(sched_group_cpus(sg),
+                                                tsk_cpus_allowed(p)))
+                                goto next;
 
+                        for_each_cpu(i, sched_group_cpus(sg)) {
+                                if (!idle_cpu(i))
+                                        goto next;
+                        }
+
+                        target = cpumask_first_and(sched_group_cpus(sg),
+                                        tsk_cpus_allowed(p));
+                        goto done;
+next:
+                        sg = sg->next;
+                } while (sg != sd->groups);
+        }
+done:
         return target;
 }
 
@@ -2686,7 +2700,6 @@ select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flags)
         int prev_cpu = task_cpu(p);
         int new_cpu = cpu;
         int want_affine = 0;
-        int want_sd = 1;
         int sync = wake_flags & WF_SYNC;
 
         if (p->nr_cpus_allowed == 1)
@@ -2704,48 +2717,21 @@ select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flags)
                         continue;
 
                 /*
-                 * If power savings logic is enabled for a domain, see if we
-                 * are not overloaded, if so, don't balance wider.
-                 */
-                if (tmp->flags & (SD_PREFER_LOCAL)) {
-                        unsigned long power = 0;
-                        unsigned long nr_running = 0;
-                        unsigned long capacity;
-                        int i;
-
-                        for_each_cpu(i, sched_domain_span(tmp)) {
-                                power += power_of(i);
-                                nr_running += cpu_rq(i)->cfs.nr_running;
-                        }
-
-                        capacity = DIV_ROUND_CLOSEST(power, SCHED_POWER_SCALE);
-
-                        if (nr_running < capacity)
-                                want_sd = 0;
-                }
-
-                /*
                  * If both cpu and prev_cpu are part of this domain,
                  * cpu is a valid SD_WAKE_AFFINE target.
                  */
                 if (want_affine && (tmp->flags & SD_WAKE_AFFINE) &&
                     cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) {
                         affine_sd = tmp;
-                        want_affine = 0;
-                }
-
-                if (!want_sd && !want_affine)
                         break;
+                }
 
-                if (!(tmp->flags & sd_flag))
-                        continue;
-
-                if (want_sd)
+                if (tmp->flags & sd_flag)
                         sd = tmp;
         }
 
         if (affine_sd) {
-                if (cpu == prev_cpu || wake_affine(affine_sd, p, sync))
+                if (cpu != prev_cpu && wake_affine(affine_sd, p, sync))
                         prev_cpu = cpu;
 
                 new_cpu = select_idle_sibling(p, prev_cpu);
@@ -3658,7 +3644,6 @@ fix_small_capacity(struct sched_domain *sd, struct sched_group *group)
  * @group: sched_group whose statistics are to be updated.
  * @load_idx: Load index of sched_domain of this_cpu for load calc.
  * @local_group: Does group contain this_cpu.
- * @cpus: Set of cpus considered for load balancing.
  * @balance: Should we balance.
  * @sgs: variable to hold the statistics for this group.
  */
@@ -3805,7 +3790,6 @@ static bool update_sd_pick_busiest(struct lb_env *env,
 /**
  * update_sd_lb_stats - Update sched_domain's statistics for load balancing.
  * @env: The load balancing environment.
- * @cpus: Set of cpus considered for load balancing.
  * @balance: Should we balance.
  * @sds: variable to hold the statistics for this sched_domain.
  */
@@ -4283,7 +4267,7 @@ redo:
                 goto out_balanced;
         }
 
-        BUG_ON(busiest == this_rq);
+        BUG_ON(busiest == env.dst_rq);
 
         schedstat_add(sd, lb_imbalance[idle], env.imbalance);
 
@@ -4304,7 +4288,7 @@ redo:
                 update_h_load(env.src_cpu);
 more_balance:
                 local_irq_save(flags);
-                double_rq_lock(this_rq, busiest);
+                double_rq_lock(env.dst_rq, busiest);
 
                 /*
                  * cur_ld_moved - load moved in current iteration
@@ -4312,7 +4296,7 @@ more_balance:
                  */
                 cur_ld_moved = move_tasks(&env);
                 ld_moved += cur_ld_moved;
-                double_rq_unlock(this_rq, busiest);
+                double_rq_unlock(env.dst_rq, busiest);
                 local_irq_restore(flags);
 
                 if (env.flags & LBF_NEED_BREAK) {
@@ -4348,8 +4332,7 @@ more_balance:
                 if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0 &&
                                 lb_iterations++ < max_lb_iterations) {
 
-                        this_rq          = cpu_rq(env.new_dst_cpu);
-                        env.dst_rq       = this_rq;
+                        env.dst_rq       = cpu_rq(env.new_dst_cpu);
                         env.dst_cpu      = env.new_dst_cpu;
                         env.flags       &= ~LBF_SOME_PINNED;
                         env.loop         = 0;
@@ -4634,7 +4617,7 @@ static void nohz_balancer_kick(int cpu)
         return;
 }
 
-static inline void clear_nohz_tick_stopped(int cpu)
+static inline void nohz_balance_exit_idle(int cpu)
 {
         if (unlikely(test_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)))) {
                 cpumask_clear_cpu(cpu, nohz.idle_cpus_mask);
@@ -4674,28 +4657,23 @@ void set_cpu_sd_state_idle(void)
 }
 
 /*
- * This routine will record that this cpu is going idle with tick stopped.
+ * This routine will record that the cpu is going idle with tick stopped.
  * This info will be used in performing idle load balancing in the future.
  */
-void select_nohz_load_balancer(int stop_tick)
+void nohz_balance_enter_idle(int cpu)
 {
-        int cpu = smp_processor_id();
-
         /*
          * If this cpu is going down, then nothing needs to be done.
          */
         if (!cpu_active(cpu))
                 return;
 
-        if (stop_tick) {
-                if (test_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)))
-                        return;
+        if (test_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)))
+                return;
 
-                cpumask_set_cpu(cpu, nohz.idle_cpus_mask);
-                atomic_inc(&nohz.nr_cpus);
-                set_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu));
-        }
-        return;
+        cpumask_set_cpu(cpu, nohz.idle_cpus_mask);
+        atomic_inc(&nohz.nr_cpus);
+        set_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu));
 }
 
 static int __cpuinit sched_ilb_notifier(struct notifier_block *nfb,
@@ -4703,7 +4681,7 @@ static int __cpuinit sched_ilb_notifier(struct notifier_block *nfb,
 {
         switch (action & ~CPU_TASKS_FROZEN) {
         case CPU_DYING:
-                clear_nohz_tick_stopped(smp_processor_id());
+                nohz_balance_exit_idle(smp_processor_id());
                 return NOTIFY_OK;
         default:
                 return NOTIFY_DONE;
@@ -4825,14 +4803,15 @@ static void nohz_idle_balance(int this_cpu, enum cpu_idle_type idle)
                 if (need_resched())
                         break;
 
-                raw_spin_lock_irq(&this_rq->lock);
-                update_rq_clock(this_rq);
-                update_idle_cpu_load(this_rq);
-                raw_spin_unlock_irq(&this_rq->lock);
+                rq = cpu_rq(balance_cpu);
+
+                raw_spin_lock_irq(&rq->lock);
+                update_rq_clock(rq);
+                update_idle_cpu_load(rq);
+                raw_spin_unlock_irq(&rq->lock);
 
                 rebalance_domains(balance_cpu, CPU_IDLE);
 
-                rq = cpu_rq(balance_cpu);
                 if (time_after(this_rq->next_balance, rq->next_balance))
                         this_rq->next_balance = rq->next_balance;
         }
@@ -4863,7 +4842,7 @@ static inline int nohz_kick_needed(struct rq *rq, int cpu)
         * busy tick after returning from idle, we will update the busy stats.
         */
         set_cpu_sd_state_busy();
-        clear_nohz_tick_stopped(cpu);
+        nohz_balance_exit_idle(cpu);
 
         /*
          * None are in tickless mode and hence no need for NOHZ idle load
@@ -4956,6 +4935,9 @@ static void rq_online_fair(struct rq *rq)
 static void rq_offline_fair(struct rq *rq)
 {
         update_sysctl();
+
+        /* Ensure any throttled groups are reachable by pick_next_task */
+        unthrottle_offline_cfs_rqs(rq);
 }
 
 #endif /* CONFIG_SMP */