1 files changed, 23 insertions, 17 deletions

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 22321db64952..c219bf8d704c 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -3069,6 +3069,9 @@ struct lb_env {
         int                     new_dst_cpu;
         enum cpu_idle_type      idle;
         long                    imbalance;
+        /* The set of CPUs under consideration for load-balancing */
+        struct cpumask          *cpus;
+
         unsigned int            flags;
 
         unsigned int            loop;
@@ -3384,6 +3387,14 @@ static int tg_load_down(struct task_group *tg, void *data)
 
 static void update_h_load(long cpu)
 {
+        struct rq *rq = cpu_rq(cpu);
+        unsigned long now = jiffies;
+
+        if (rq->h_load_throttle == now)
+                return;
+
+        rq->h_load_throttle = now;
+
         rcu_read_lock();
         walk_tg_tree(tg_load_down, tg_nop, (void *)cpu);
         rcu_read_unlock();
@@ -3653,8 +3664,7 @@ fix_small_capacity(struct sched_domain *sd, struct sched_group *group)
  */
 static inline void update_sg_lb_stats(struct lb_env *env,
                         struct sched_group *group, int load_idx,
-                        int local_group, const struct cpumask *cpus,
-                        int *balance, struct sg_lb_stats *sgs)
+                        int local_group, int *balance, struct sg_lb_stats *sgs)
 {
         unsigned long nr_running, max_nr_running, min_nr_running;
         unsigned long load, max_cpu_load, min_cpu_load;
@@ -3671,7 +3681,7 @@ static inline void update_sg_lb_stats(struct lb_env *env,
         max_nr_running = 0;
         min_nr_running = ~0UL;
 
-        for_each_cpu_and(i, sched_group_cpus(group), cpus) {
+        for_each_cpu_and(i, sched_group_cpus(group), env->cpus) {
                 struct rq *rq = cpu_rq(i);
 
                 nr_running = rq->nr_running;
@@ -3800,8 +3810,7 @@ static bool update_sd_pick_busiest(struct lb_env *env,
  * @sds: variable to hold the statistics for this sched_domain.
  */
 static inline void update_sd_lb_stats(struct lb_env *env,
-                                      const struct cpumask *cpus,
-                                      int *balance, struct sd_lb_stats *sds)
+                                        int *balance, struct sd_lb_stats *sds)
 {
         struct sched_domain *child = env->sd->child;
         struct sched_group *sg = env->sd->groups;
@@ -3818,8 +3827,7 @@ static inline void update_sd_lb_stats(struct lb_env *env,
 
                 local_group = cpumask_test_cpu(env->dst_cpu, sched_group_cpus(sg));
                 memset(&sgs, 0, sizeof(sgs));
-                update_sg_lb_stats(env, sg, load_idx, local_group,
-                                   cpus, balance, &sgs);
+                update_sg_lb_stats(env, sg, load_idx, local_group, balance, &sgs);
 
                 if (local_group && !(*balance))
                         return;
@@ -4055,7 +4063,6 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s
  * to restore balance.
  *
  * @env: The load balancing environment.
- * @cpus: The set of CPUs under consideration for load-balancing.
  * @balance: Pointer to a variable indicating if this_cpu
  *      is the appropriate cpu to perform load balancing at this_level.
  *
@@ -4065,7 +4072,7 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s
  *                 put to idle by rebalancing its tasks onto our group.
  */
 static struct sched_group *
-find_busiest_group(struct lb_env *env, const struct cpumask *cpus, int *balance)
+find_busiest_group(struct lb_env *env, int *balance)
 {
         struct sd_lb_stats sds;
 
@@ -4075,7 +4082,7 @@ find_busiest_group(struct lb_env *env, const struct cpumask *cpus, int *balance)
          * Compute the various statistics relavent for load balancing at
          * this level.
          */
-        update_sd_lb_stats(env, cpus, balance, &sds);
+        update_sd_lb_stats(env, balance, &sds);
 
         /*
          * this_cpu is not the appropriate cpu to perform load balancing at
@@ -4155,8 +4162,7 @@ ret:
  * find_busiest_queue - find the busiest runqueue among the cpus in group.
  */
 static struct rq *find_busiest_queue(struct lb_env *env,
-                                     struct sched_group *group,
-                                     const struct cpumask *cpus)
+                                     struct sched_group *group)
 {
         struct rq *busiest = NULL, *rq;
         unsigned long max_load = 0;
@@ -4171,7 +4177,7 @@ static struct rq *find_busiest_queue(struct lb_env *env,
                 if (!capacity)
                         capacity = fix_small_capacity(env->sd, group);
 
-                if (!cpumask_test_cpu(i, cpus))
+                if (!cpumask_test_cpu(i, env->cpus))
                         continue;
 
                 rq = cpu_rq(i);
@@ -4252,6 +4258,7 @@ static int load_balance(int this_cpu, struct rq *this_rq,
                 .dst_grpmask    = sched_group_cpus(sd->groups),
                 .idle           = idle,
                 .loop_break     = sched_nr_migrate_break,
+                .cpus           = cpus,
         };
 
         cpumask_copy(cpus, cpu_active_mask);
@@ -4260,7 +4267,7 @@ static int load_balance(int this_cpu, struct rq *this_rq,
         schedstat_inc(sd, lb_count[idle]);
 
 redo:
-        group = find_busiest_group(&env, cpus, balance);
+        group = find_busiest_group(&env, balance);
 
         if (*balance == 0)
                 goto out_balanced;
@@ -4270,7 +4277,7 @@ redo:
                 goto out_balanced;
         }
 
-        busiest = find_busiest_queue(&env, group, cpus);
+        busiest = find_busiest_queue(&env, group);
         if (!busiest) {
                 schedstat_inc(sd, lb_nobusyq[idle]);
                 goto out_balanced;
@@ -4294,11 +4301,10 @@ redo:
                 env.src_rq    = busiest;
                 env.loop_max  = min(sysctl_sched_nr_migrate, busiest->nr_running);
 
+                update_h_load(env.src_cpu);
 more_balance:
                 local_irq_save(flags);
                 double_rq_lock(this_rq, busiest);
-                if (!env.loop)
-                        update_h_load(env.src_cpu);
 
                 /*
                  * cur_ld_moved - load moved in current iteration