sched: Pre-compute cpumask_weight(sched_domain_span(sd))

Dave reported that his large SPARC machines spend lots of time in
hweight64(), try and optimize some of those needless cpumask_weight()
invocations (esp. with the large offstack cpumasks these are very
expensive indeed).

Reported-by: David Miller <davem@davemloft.net>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

3 files changed, 9 insertions, 7 deletions

diff --git a/include/linux/sched.h b/include/linux/sched.h
index e3e900f318d7..dfea40574b2a 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -960,6 +960,7 @@ struct sched_domain {
         char *name;
 #endif
 
+        unsigned int span_weight;
         /*
          * Span of all CPUs in this domain.
          *
diff --git a/kernel/sched.c b/kernel/sched.c
index 0cc913a8554f..4956ed092838 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -6271,6 +6271,9 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
         struct rq *rq = cpu_rq(cpu);
         struct sched_domain *tmp;
 
+        for (tmp = sd; tmp; tmp = tmp->parent)
+                tmp->span_weight = cpumask_weight(sched_domain_span(tmp));
+
         /* Remove the sched domains which do not contribute to scheduling. */
         for (tmp = sd; tmp; ) {
                 struct sched_domain *parent = tmp->parent;
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index 88d3053ac7c2..0a413c7e3ab8 100644
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -1508,9 +1508,7 @@ select_task_rq_fair(struct rq *rq, struct task_struct *p, int sd_flag, int wake_
                  * Pick the largest domain to update shares over
                  */
                 tmp = sd;
-                if (affine_sd && (!tmp ||
-                                  cpumask_weight(sched_domain_span(affine_sd)) >
-                                  cpumask_weight(sched_domain_span(sd))))
+                if (affine_sd && (!tmp || affine_sd->span_weight > sd->span_weight))
                         tmp = affine_sd;
 
                 if (tmp) {
@@ -1554,10 +1552,10 @@ select_task_rq_fair(struct rq *rq, struct task_struct *p, int sd_flag, int wake_
 
                 /* Now try balancing at a lower domain level of new_cpu */
                 cpu = new_cpu;
-                weight = cpumask_weight(sched_domain_span(sd));
+                weight = sd->span_weight;
                 sd = NULL;
                 for_each_domain(cpu, tmp) {
-                        if (weight <= cpumask_weight(sched_domain_span(tmp)))
+                        if (weight <= tmp->span_weight)
                                 break;
                         if (tmp->flags & sd_flag)
                                 sd = tmp;
@@ -2243,7 +2241,7 @@ unsigned long __weak arch_scale_freq_power(struct sched_domain *sd, int cpu)
 
 unsigned long default_scale_smt_power(struct sched_domain *sd, int cpu)
 {
-        unsigned long weight = cpumask_weight(sched_domain_span(sd));
+        unsigned long weight = sd->span_weight;
         unsigned long smt_gain = sd->smt_gain;
 
         smt_gain /= weight;
@@ -2276,7 +2274,7 @@ unsigned long scale_rt_power(int cpu)
 
 static void update_cpu_power(struct sched_domain *sd, int cpu)
 {
-        unsigned long weight = cpumask_weight(sched_domain_span(sd));
+        unsigned long weight = sd->span_weight;
         unsigned long power = SCHED_LOAD_SCALE;
         struct sched_group *sdg = sd->groups;