[PATCH] sched: balance on fork

Reimplement the balance on exec balancing to be sched-domains aware.  Use this
to also do balance on fork balancing.  Make x86_64 do balance on fork over the
NUMA domain.

The problem that the non sched domains aware blancing became apparent on dual
core, multi socket opterons.  What we want is for the new tasks to be sent to
a different socket, but more often than not, we would first load up our
sibling core, or fill two cores of a single remote socket before selecting a
new one.

This gives large improvements to STREAM on such systems.

Signed-off-by: Nick Piggin <nickpiggin@yahoo.com.au>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

3 files changed, 10 insertions, 4 deletions

diff --git a/include/asm-x86_64/topology.h b/include/asm-x86_64/topology.h
index 9cb7459ce722..802d09b9c99f 100644
--- a/include/asm-x86_64/topology.h
+++ b/include/asm-x86_64/topology.h
@@ -44,9 +44,11 @@ extern int __node_distance(int, int);
         .idle_idx               = 2,                    \
         .newidle_idx            = 1,                    \
         .wake_idx               = 1,                    \
+        .forkexec_idx           = 1,                    \
         .per_cpu_gain           = 100,                  \
         .flags                  = SD_LOAD_BALANCE       \
                                 | SD_BALANCE_NEWIDLE    \
+                                | SD_BALANCE_FORK       \
                                 | SD_BALANCE_EXEC       \
                                 | SD_WAKE_BALANCE,      \
         .last_balance           = jiffies,              \
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 664981ac1fb6..613491d3a875 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -460,10 +460,11 @@ enum idle_type
 #define SD_LOAD_BALANCE         1       /* Do load balancing on this domain. */
 #define SD_BALANCE_NEWIDLE      2       /* Balance when about to become idle */
 #define SD_BALANCE_EXEC         4       /* Balance on exec */
-#define SD_WAKE_IDLE            8       /* Wake to idle CPU on task wakeup */
-#define SD_WAKE_AFFINE          16      /* Wake task to waking CPU */
-#define SD_WAKE_BALANCE         32      /* Perform balancing at task wakeup */
-#define SD_SHARE_CPUPOWER       64      /* Domain members share cpu power */
+#define SD_BALANCE_FORK         8       /* Balance on fork, clone */
+#define SD_WAKE_IDLE            16      /* Wake to idle CPU on task wakeup */
+#define SD_WAKE_AFFINE          32      /* Wake task to waking CPU */
+#define SD_WAKE_BALANCE         64      /* Perform balancing at task wakeup */
+#define SD_SHARE_CPUPOWER       128     /* Domain members share cpu power */
 
 struct sched_group {
         struct sched_group *next;       /* Must be a circular list */
@@ -492,6 +493,7 @@ struct sched_domain {
         unsigned int idle_idx;
         unsigned int newidle_idx;
         unsigned int wake_idx;
+        unsigned int forkexec_idx;
         int flags;                      /* See SD_* */
 
         /* Runtime fields. */
diff --git a/include/linux/topology.h b/include/linux/topology.h
index b23ec64df7f1..665597207def 100644
--- a/include/linux/topology.h
+++ b/include/linux/topology.h
@@ -93,6 +93,7 @@
         .idle_idx               = 0,                    \
         .newidle_idx            = 0,                    \
         .wake_idx               = 0,                    \
+        .forkexec_idx           = 0,                    \
         .flags                  = SD_LOAD_BALANCE       \
                                 | SD_BALANCE_NEWIDLE    \
                                 | SD_BALANCE_EXEC       \
@@ -123,6 +124,7 @@
         .idle_idx               = 0,                    \
         .newidle_idx            = 1,                    \
         .wake_idx               = 1,                    \
+        .forkexec_idx           = 0,                    \
         .flags                  = SD_LOAD_BALANCE       \
                                 | SD_BALANCE_NEWIDLE    \
                                 | SD_BALANCE_EXEC       \