3 files changed, 70 insertions, 30 deletions

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index cdf51a2adc26..dba878c73a08 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -1511,6 +1511,11 @@ static int ttwu_activate_remote(struct task_struct *p, int wake_flags)
 
 }
 #endif /* __ARCH_WANT_INTERRUPTS_ON_CTXSW */
+
+static inline int ttwu_share_cache(int this_cpu, int that_cpu)
+{
+        return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);
+}
 #endif /* CONFIG_SMP */
 
 static void ttwu_queue(struct task_struct *p, int cpu)
@@ -1518,7 +1523,7 @@ static void ttwu_queue(struct task_struct *p, int cpu)
         struct rq *rq = cpu_rq(cpu);
 
 #if defined(CONFIG_SMP)
-        if (sched_feat(TTWU_QUEUE) && cpu != smp_processor_id()) {
+        if (sched_feat(TTWU_QUEUE) && !ttwu_share_cache(smp_processor_id(), cpu)) {
                 sched_clock_cpu(cpu); /* sync clocks x-cpu */
                 ttwu_queue_remote(p, cpu);
                 return;
@@ -5744,6 +5749,31 @@ static void destroy_sched_domains(struct sched_domain *sd, int cpu)
 }
 
 /*
+ * Keep a special pointer to the highest sched_domain that has
+ * SD_SHARE_PKG_RESOURCE set (Last Level Cache Domain) for this
+ * allows us to avoid some pointer chasing select_idle_sibling().
+ *
+ * Also keep a unique ID per domain (we use the first cpu number in
+ * the cpumask of the domain), this allows us to quickly tell if
+ * two cpus are in the same cache domain, see ttwu_share_cache().
+ */
+DEFINE_PER_CPU(struct sched_domain *, sd_llc);
+DEFINE_PER_CPU(int, sd_llc_id);
+
+static void update_top_cache_domain(int cpu)
+{
+        struct sched_domain *sd;
+        int id = cpu;
+
+        sd = highest_flag_domain(cpu, SD_SHARE_PKG_RESOURCES);
+        if (sd)
+                id = cpumask_first(sched_domain_span(sd));
+
+        rcu_assign_pointer(per_cpu(sd_llc, cpu), sd);
+        per_cpu(sd_llc_id, cpu) = id;
+}
+
+/*
  * Attach the domain 'sd' to 'cpu' as its base domain. Callers must
  * hold the hotplug lock.
  */
@@ -5782,6 +5812,8 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
         tmp = rq->sd;
         rcu_assign_pointer(rq->sd, sd);
         destroy_sched_domains(tmp, cpu);
+
+        update_top_cache_domain(cpu);
 }
 
 /* cpus with isolated domains */
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index a4d2b7abc3cd..2237ffefdbce 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -2644,28 +2644,6 @@ find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
         return idlest;
 }
 
-/**
- * highest_flag_domain - Return highest sched_domain containing flag.
- * @cpu:        The cpu whose highest level of sched domain is to
- *              be returned.
- * @flag:       The flag to check for the highest sched_domain
- *              for the given cpu.
- *
- * Returns the highest sched_domain of a cpu which contains the given flag.
- */
-static inline struct sched_domain *highest_flag_domain(int cpu, int flag)
-{
-        struct sched_domain *sd, *hsd = NULL;
-
-        for_each_domain(cpu, sd) {
-                if (!(sd->flags & flag))
-                        break;
-                hsd = sd;
-        }
-
-        return hsd;
-}
-
 /*
  * Try and locate an idle CPU in the sched_domain.
  */
@@ -2696,7 +2674,7 @@ static int select_idle_sibling(struct task_struct *p, int target)
          */
         rcu_read_lock();
 
-        sd = highest_flag_domain(target, SD_SHARE_PKG_RESOURCES);
+        sd = rcu_dereference(per_cpu(sd_llc, target));
         for_each_lower_domain(sd) {
                 sg = sd->groups;
                 do {
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index d8d3613a4055..98c0c2623db8 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -487,6 +487,14 @@ static inline int cpu_of(struct rq *rq)
 
 DECLARE_PER_CPU(struct rq, runqueues);
 
+#define cpu_rq(cpu)             (&per_cpu(runqueues, (cpu)))
+#define this_rq()               (&__get_cpu_var(runqueues))
+#define task_rq(p)              cpu_rq(task_cpu(p))
+#define cpu_curr(cpu)           (cpu_rq(cpu)->curr)
+#define raw_rq()                (&__raw_get_cpu_var(runqueues))
+
+#ifdef CONFIG_SMP
+
 #define rcu_dereference_check_sched_domain(p) \
         rcu_dereference_check((p), \
                               lockdep_is_held(&sched_domains_mutex))
@@ -499,15 +507,37 @@ DECLARE_PER_CPU(struct rq, runqueues);
  * preempt-disabled sections.
  */
 #define for_each_domain(cpu, __sd) \
-        for (__sd = rcu_dereference_check_sched_domain(cpu_rq(cpu)->sd); __sd; __sd = __sd->parent)
+        for (__sd = rcu_dereference_check_sched_domain(cpu_rq(cpu)->sd); \
+                        __sd; __sd = __sd->parent)
 
 #define for_each_lower_domain(sd) for (; sd; sd = sd->child)
 
-#define cpu_rq(cpu)             (&per_cpu(runqueues, (cpu)))
-#define this_rq()               (&__get_cpu_var(runqueues))
-#define task_rq(p)              cpu_rq(task_cpu(p))
-#define cpu_curr(cpu)           (cpu_rq(cpu)->curr)
-#define raw_rq()                (&__raw_get_cpu_var(runqueues))
+/**
+ * highest_flag_domain - Return highest sched_domain containing flag.
+ * @cpu:        The cpu whose highest level of sched domain is to
+ *              be returned.
+ * @flag:       The flag to check for the highest sched_domain
+ *              for the given cpu.
+ *
+ * Returns the highest sched_domain of a cpu which contains the given flag.
+ */
+static inline struct sched_domain *highest_flag_domain(int cpu, int flag)
+{
+        struct sched_domain *sd, *hsd = NULL;
+
+        for_each_domain(cpu, sd) {
+                if (!(sd->flags & flag))
+                        break;
+                hsd = sd;
+        }
+
+        return hsd;
+}
+
+DECLARE_PER_CPU(struct sched_domain *, sd_llc);
+DECLARE_PER_CPU(int, sd_llc_id);
+
+#endif /* CONFIG_SMP */
 
 #include "stats.h"
 #include "auto_group.h"