3 files changed, 28 insertions, 18 deletions

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index aa066f306be2..1deccd78be98 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -4883,6 +4883,8 @@ DEFINE_PER_CPU(struct sched_domain *, sd_llc);
 DEFINE_PER_CPU(int, sd_llc_size);
 DEFINE_PER_CPU(int, sd_llc_id);
 DEFINE_PER_CPU(struct sched_domain *, sd_numa);
+DEFINE_PER_CPU(struct sched_domain *, sd_busy);
+DEFINE_PER_CPU(struct sched_domain *, sd_asym);
 
 static void update_top_cache_domain(int cpu)
 {
@@ -4894,6 +4896,7 @@ static void update_top_cache_domain(int cpu)
         if (sd) {
                 id = cpumask_first(sched_domain_span(sd));
                 size = cpumask_weight(sched_domain_span(sd));
+                rcu_assign_pointer(per_cpu(sd_busy, cpu), sd->parent);
         }
 
         rcu_assign_pointer(per_cpu(sd_llc, cpu), sd);
@@ -4902,6 +4905,9 @@ static void update_top_cache_domain(int cpu)
 
         sd = lowest_flag_domain(cpu, SD_NUMA);
         rcu_assign_pointer(per_cpu(sd_numa, cpu), sd);
+
+        sd = highest_flag_domain(cpu, SD_ASYM_PACKING);
+        rcu_assign_pointer(per_cpu(sd_asym, cpu), sd);
 }
 
 /*
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 074551a792f7..df77c605c7a6 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -6534,16 +6534,16 @@ static inline void nohz_balance_exit_idle(int cpu)
 static inline void set_cpu_sd_state_busy(void)
 {
         struct sched_domain *sd;
+        int cpu = smp_processor_id();
 
         rcu_read_lock();
-        sd = rcu_dereference_check_sched_domain(this_rq()->sd);
+        sd = rcu_dereference(per_cpu(sd_busy, cpu));
 
         if (!sd || !sd->nohz_idle)
                 goto unlock;
         sd->nohz_idle = 0;
 
-        for (; sd; sd = sd->parent)
-                atomic_inc(&sd->groups->sgp->nr_busy_cpus);
+        atomic_inc(&sd->groups->sgp->nr_busy_cpus);
 unlock:
         rcu_read_unlock();
 }
@@ -6551,16 +6551,16 @@ unlock:
 void set_cpu_sd_state_idle(void)
 {
         struct sched_domain *sd;
+        int cpu = smp_processor_id();
 
         rcu_read_lock();
-        sd = rcu_dereference_check_sched_domain(this_rq()->sd);
+        sd = rcu_dereference(per_cpu(sd_busy, cpu));
 
         if (!sd || sd->nohz_idle)
                 goto unlock;
         sd->nohz_idle = 1;
 
-        for (; sd; sd = sd->parent)
-                atomic_dec(&sd->groups->sgp->nr_busy_cpus);
+        atomic_dec(&sd->groups->sgp->nr_busy_cpus);
 unlock:
         rcu_read_unlock();
 }
@@ -6767,6 +6767,8 @@ static inline int nohz_kick_needed(struct rq *rq, int cpu)
 {
         unsigned long now = jiffies;
         struct sched_domain *sd;
+        struct sched_group_power *sgp;
+        int nr_busy;
 
         if (unlikely(idle_cpu(cpu)))
                 return 0;
@@ -6792,22 +6794,22 @@ static inline int nohz_kick_needed(struct rq *rq, int cpu)
                 goto need_kick;
 
         rcu_read_lock();
-        for_each_domain(cpu, sd) {
-                struct sched_group *sg = sd->groups;
-                struct sched_group_power *sgp = sg->sgp;
-                int nr_busy = atomic_read(&sgp->nr_busy_cpus);
+        sd = rcu_dereference(per_cpu(sd_busy, cpu));
 
-                if (sd->flags & SD_SHARE_PKG_RESOURCES && nr_busy > 1)
-                        goto need_kick_unlock;
+        if (sd) {
+                sgp = sd->groups->sgp;
+                nr_busy = atomic_read(&sgp->nr_busy_cpus);
 
-                if (sd->flags & SD_ASYM_PACKING
-                    && (cpumask_first_and(nohz.idle_cpus_mask,
-                                          sched_domain_span(sd)) < cpu))
+                if (nr_busy > 1)
                         goto need_kick_unlock;
-
-                if (!(sd->flags & (SD_SHARE_PKG_RESOURCES | SD_ASYM_PACKING)))
-                        break;
         }
+
+        sd = rcu_dereference(per_cpu(sd_asym, cpu));
+
+        if (sd && (cpumask_first_and(nohz.idle_cpus_mask,
+                                  sched_domain_span(sd)) < cpu))
+                goto need_kick_unlock;
+
         rcu_read_unlock();
         return 0;
 
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 4e650acffed7..88c85b21d633 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -623,6 +623,8 @@ DECLARE_PER_CPU(struct sched_domain *, sd_llc);
 DECLARE_PER_CPU(int, sd_llc_size);
 DECLARE_PER_CPU(int, sd_llc_id);
 DECLARE_PER_CPU(struct sched_domain *, sd_numa);
+DECLARE_PER_CPU(struct sched_domain *, sd_busy);
+DECLARE_PER_CPU(struct sched_domain *, sd_asym);
 
 struct sched_group_power {
         atomic_t ref;