4 files changed, 38 insertions, 74 deletions

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index fbf1fd098dc6..649c9f876cb1 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5304,27 +5304,17 @@ void idle_task_exit(void)
 }
 
 /*
- * While a dead CPU has no uninterruptible tasks queued at this point,
- * it might still have a nonzero ->nr_uninterruptible counter, because
- * for performance reasons the counter is not stricly tracking tasks to
- * their home CPUs. So we just add the counter to another CPU's counter,
- * to keep the global sum constant after CPU-down:
- */
-static void migrate_nr_uninterruptible(struct rq *rq_src)
-{
-        struct rq *rq_dest = cpu_rq(cpumask_any(cpu_active_mask));
-
-        rq_dest->nr_uninterruptible += rq_src->nr_uninterruptible;
-        rq_src->nr_uninterruptible = 0;
-}
-
-/*
- * remove the tasks which were accounted by rq from calc_load_tasks.
+ * Since this CPU is going 'away' for a while, fold any nr_active delta
+ * we might have. Assumes we're called after migrate_tasks() so that the
+ * nr_active count is stable.
+ *
+ * Also see the comment "Global load-average calculations".
  */
-static void calc_global_load_remove(struct rq *rq)
+static void calc_load_migrate(struct rq *rq)
 {
-        atomic_long_sub(rq->calc_load_active, &calc_load_tasks);
-        rq->calc_load_active = 0;
+        long delta = calc_load_fold_active(rq);
+        if (delta)
+                atomic_long_add(delta, &calc_load_tasks);
 }
 
 /*
@@ -5352,9 +5342,6 @@ static void migrate_tasks(unsigned int dead_cpu)
          */
         rq->stop = NULL;
 
-        /* Ensure any throttled groups are reachable by pick_next_task */
-        unthrottle_offline_cfs_rqs(rq);
-
         for ( ; ; ) {
                 /*
                  * There's this thread running, bail when that's the only
@@ -5618,8 +5605,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
                 BUG_ON(rq->nr_running != 1); /* the migration thread */
                 raw_spin_unlock_irqrestore(&rq->lock, flags);
 
-                migrate_nr_uninterruptible(rq);
-                calc_global_load_remove(rq);
+                calc_load_migrate(rq);
                 break;
 #endif
         }
@@ -6028,11 +6014,6 @@ static void destroy_sched_domains(struct sched_domain *sd, int cpu)
  * SD_SHARE_PKG_RESOURCE set (Last Level Cache Domain) for this
  * allows us to avoid some pointer chasing select_idle_sibling().
  *
- * Iterate domains and sched_groups downward, assigning CPUs to be
- * select_idle_sibling() hw buddy.  Cross-wiring hw makes bouncing
- * due to random perturbation self canceling, ie sw buddies pull
- * their counterpart to their CPU's hw counterpart.
- *
  * Also keep a unique ID per domain (we use the first cpu number in
  * the cpumask of the domain), this allows us to quickly tell if
  * two cpus are in the same cache domain, see cpus_share_cache().
@@ -6046,40 +6027,8 @@ static void update_top_cache_domain(int cpu)
         int id = cpu;
 
         sd = highest_flag_domain(cpu, SD_SHARE_PKG_RESOURCES);
-        if (sd) {
-                struct sched_domain *tmp = sd;
-                struct sched_group *sg, *prev;
-                bool right;
-
-                /*
-                 * Traverse to first CPU in group, and count hops
-                 * to cpu from there, switching direction on each
-                 * hop, never ever pointing the last CPU rightward.
-                 */
-                do {
-                        id = cpumask_first(sched_domain_span(tmp));
-                        prev = sg = tmp->groups;
-                        right = 1;
-
-                        while (cpumask_first(sched_group_cpus(sg)) != id)
-                                sg = sg->next;
-
-                        while (!cpumask_test_cpu(cpu, sched_group_cpus(sg))) {
-                                prev = sg;
-                                sg = sg->next;
-                                right = !right;
-                        }
-
-                        /* A CPU went down, never point back to domain start. */
-                        if (right && cpumask_first(sched_group_cpus(sg->next)) == id)
-                                right = false;
-
-                        sg = right ? sg->next : prev;
-                        tmp->idle_buddy = cpumask_first(sched_group_cpus(sg));
-                } while ((tmp = tmp->child));
-
+        if (sd)
                 id = cpumask_first(sched_domain_span(sd));
-        }
 
         rcu_assign_pointer(per_cpu(sd_llc, cpu), sd);
         per_cpu(sd_llc_id, cpu) = id;
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index c219bf8d704c..96e2b18b6283 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -2052,7 +2052,7 @@ static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
         hrtimer_cancel(&cfs_b->slack_timer);
 }
 
-void unthrottle_offline_cfs_rqs(struct rq *rq)
+static void unthrottle_offline_cfs_rqs(struct rq *rq)
 {
         struct cfs_rq *cfs_rq;
 
@@ -2106,7 +2106,7 @@ static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
         return NULL;
 }
 static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
-void unthrottle_offline_cfs_rqs(struct rq *rq) {}
+static inline void unthrottle_offline_cfs_rqs(struct rq *rq) {}
 
 #endif /* CONFIG_CFS_BANDWIDTH */
 
@@ -2637,6 +2637,8 @@ static int select_idle_sibling(struct task_struct *p, int target)
         int cpu = smp_processor_id();
         int prev_cpu = task_cpu(p);
         struct sched_domain *sd;
+        struct sched_group *sg;
+        int i;
 
         /*
          * If the task is going to be woken-up on this cpu and if it is
@@ -2653,17 +2655,29 @@ static int select_idle_sibling(struct task_struct *p, int target)
                 return prev_cpu;
 
         /*
-         * Otherwise, check assigned siblings to find an elegible idle cpu.
+         * Otherwise, iterate the domains and find an elegible idle cpu.
          */
         sd = rcu_dereference(per_cpu(sd_llc, target));
-
         for_each_lower_domain(sd) {
-                if (!cpumask_test_cpu(sd->idle_buddy, tsk_cpus_allowed(p)))
-                        continue;
-                if (idle_cpu(sd->idle_buddy))
-                        return sd->idle_buddy;
-        }
+                sg = sd->groups;
+                do {
+                        if (!cpumask_intersects(sched_group_cpus(sg),
+                                                tsk_cpus_allowed(p)))
+                                goto next;
 
+                        for_each_cpu(i, sched_group_cpus(sg)) {
+                                if (!idle_cpu(i))
+                                        goto next;
+                        }
+
+                        target = cpumask_first_and(sched_group_cpus(sg),
+                                        tsk_cpus_allowed(p));
+                        goto done;
+next:
+                        sg = sg->next;
+                } while (sg != sd->groups);
+        }
+done:
         return target;
 }
 
@@ -3658,7 +3672,6 @@ fix_small_capacity(struct sched_domain *sd, struct sched_group *group)
  * @group: sched_group whose statistics are to be updated.
  * @load_idx: Load index of sched_domain of this_cpu for load calc.
  * @local_group: Does group contain this_cpu.
- * @cpus: Set of cpus considered for load balancing.
  * @balance: Should we balance.
  * @sgs: variable to hold the statistics for this group.
  */
@@ -3805,7 +3818,6 @@ static bool update_sd_pick_busiest(struct lb_env *env,
 /**
  * update_sd_lb_stats - Update sched_domain's statistics for load balancing.
  * @env: The load balancing environment.
- * @cpus: Set of cpus considered for load balancing.
  * @balance: Should we balance.
  * @sds: variable to hold the statistics for this sched_domain.
  */
@@ -4956,6 +4968,9 @@ static void rq_online_fair(struct rq *rq)
 static void rq_offline_fair(struct rq *rq)
 {
         update_sysctl();
+
+        /* Ensure any throttled groups are reachable by pick_next_task */
+        unthrottle_offline_cfs_rqs(rq);
 }
 
 #endif /* CONFIG_SMP */
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index 944cb68420e9..e0b7ba9c040f 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -691,6 +691,7 @@ balanced:
                  * runtime - in which case borrowing doesn't make sense.
                  */
                 rt_rq->rt_runtime = RUNTIME_INF;
+                rt_rq->rt_throttled = 0;
                 raw_spin_unlock(&rt_rq->rt_runtime_lock);
                 raw_spin_unlock(&rt_b->rt_runtime_lock);
         }
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index f6714d009e77..0848fa36c383 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1144,7 +1144,6 @@ extern void print_rt_stats(struct seq_file *m, int cpu);
 
 extern void init_cfs_rq(struct cfs_rq *cfs_rq);
 extern void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq);
-extern void unthrottle_offline_cfs_rqs(struct rq *rq);
 
 extern void account_cfs_bandwidth_used(int enabled, int was_enabled);