sched: Merge select_task_rq_fair() and sched_balance_self()

The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.

To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().

Modify sched_balance_self() to:

  - update_shares() when walking up the domain tree,
    (it only called it for the top domain, but it should
     have done this anyway), which allows us to remove
    this ugly bit from try_to_wake_up().

  - do wake_affine() on the smallest domain that contains
    both this (the waking) and the prev (the wakee) cpu for
    WAKE invocations.

Then use the top-down balance steps it had to replace wake_idle().

This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.

SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.

Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

6 files changed, 11 insertions, 13 deletions

diff --git a/arch/ia64/include/asm/topology.h b/arch/ia64/include/asm/topology.h
index 7b4c8c70b2d1..cf6053b226c3 100644
--- a/arch/ia64/include/asm/topology.h
+++ b/arch/ia64/include/asm/topology.h
@@ -67,6 +67,7 @@ void build_cpu_to_node_map(void);
         .flags                  = SD_LOAD_BALANCE       \
                                 | SD_BALANCE_NEWIDLE    \
                                 | SD_BALANCE_EXEC       \
+                                | SD_BALANCE_WAKE       \
                                 | SD_WAKE_AFFINE,       \
         .last_balance           = jiffies,              \
         .balance_interval       = 1,                    \
@@ -91,8 +92,8 @@ void build_cpu_to_node_map(void);
         .flags                  = SD_LOAD_BALANCE       \
                                 | SD_BALANCE_EXEC       \
                                 | SD_BALANCE_FORK       \
-                                | SD_SERIALIZE          \
-                                | SD_WAKE_BALANCE,      \
+                                | SD_BALANCE_WAKE       \
+                                | SD_SERIALIZE,         \
         .last_balance           = jiffies,              \
         .balance_interval       = 64,                   \
         .nr_balance_failed      = 0,                    \
diff --git a/arch/mips/include/asm/mach-ip27/topology.h b/arch/mips/include/asm/mach-ip27/topology.h
index 07547231e078..d8332398f5be 100644
--- a/arch/mips/include/asm/mach-ip27/topology.h
+++ b/arch/mips/include/asm/mach-ip27/topology.h
@@ -48,7 +48,7 @@ extern unsigned char __node_distances[MAX_COMPACT_NODES][MAX_COMPACT_NODES];
         .cache_nice_tries       = 1,                    \
         .flags                  = SD_LOAD_BALANCE       \
                                 | SD_BALANCE_EXEC       \
-                                | SD_WAKE_BALANCE,      \
+                                | SD_BALANCE_WAKE,      \
         .last_balance           = jiffies,              \
         .balance_interval       = 1,                    \
         .nr_balance_failed      = 0,                    \
diff --git a/arch/powerpc/include/asm/topology.h b/arch/powerpc/include/asm/topology.h
index 054a16d68082..c6343313ff59 100644
--- a/arch/powerpc/include/asm/topology.h
+++ b/arch/powerpc/include/asm/topology.h
@@ -62,9 +62,8 @@ static inline int pcibus_to_node(struct pci_bus *bus)
         .flags                  = SD_LOAD_BALANCE       \
                                 | SD_BALANCE_EXEC       \
                                 | SD_BALANCE_NEWIDLE    \
-                                | SD_WAKE_IDLE          \
-                                | SD_SERIALIZE          \
-                                | SD_WAKE_BALANCE,      \
+                                | SD_BALANCE_WAKE       \
+                                | SD_SERIALIZE,         \
         .last_balance           = jiffies,              \
         .balance_interval       = 1,                    \
         .nr_balance_failed      = 0,                    \
diff --git a/arch/sh/include/asm/topology.h b/arch/sh/include/asm/topology.h
index b69ee850906d..dc1531e2f25f 100644
--- a/arch/sh/include/asm/topology.h
+++ b/arch/sh/include/asm/topology.h
@@ -21,8 +21,8 @@
         .flags                  = SD_LOAD_BALANCE       \
                                 | SD_BALANCE_FORK       \
                                 | SD_BALANCE_EXEC       \
-                                | SD_SERIALIZE          \
-                                | SD_WAKE_BALANCE,      \
+                                | SD_BALANCE_WAKE       \
+                                | SD_SERIALIZE,         \
         .last_balance           = jiffies,              \
         .balance_interval       = 1,                    \
         .nr_balance_failed      = 0,                    \
diff --git a/arch/sparc/include/asm/topology_64.h b/arch/sparc/include/asm/topology_64.h
index e5ea8d332421..1d091abd2d13 100644
--- a/arch/sparc/include/asm/topology_64.h
+++ b/arch/sparc/include/asm/topology_64.h
@@ -57,8 +57,8 @@ static inline int pcibus_to_node(struct pci_bus *pbus)
         .flags                  = SD_LOAD_BALANCE       \
                                 | SD_BALANCE_FORK       \
                                 | SD_BALANCE_EXEC       \
-                                | SD_SERIALIZE          \
-                                | SD_WAKE_BALANCE,      \
+                                | SD_BALANCE_WAKE       \
+                                | SD_SERIALIZE,         \
         .last_balance           = jiffies,              \
         .balance_interval       = 1,                    \
 }
diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h
index 26d06e052a18..966d58dc6274 100644
--- a/arch/x86/include/asm/topology.h
+++ b/arch/x86/include/asm/topology.h
@@ -145,14 +145,12 @@ extern unsigned long node_remap_size[];
                                 | 1*SD_BALANCE_NEWIDLE                  \
                                 | 1*SD_BALANCE_EXEC                     \
                                 | 1*SD_BALANCE_FORK                     \
-                                | 0*SD_WAKE_IDLE                        \
+                                | 1*SD_BALANCE_WAKE                     \
                                 | 1*SD_WAKE_AFFINE                      \
-                                | 1*SD_WAKE_BALANCE                     \
                                 | 0*SD_SHARE_CPUPOWER                   \
                                 | 0*SD_POWERSAVINGS_BALANCE             \
                                 | 0*SD_SHARE_PKG_RESOURCES              \
                                 | 1*SD_SERIALIZE                        \
-                                | 1*SD_WAKE_IDLE_FAR                    \
                                 | 0*SD_PREFER_SIBLING                   \
                                 ,                                       \
         .last_balance           = jiffies,                              \