3 files changed, 126 insertions, 18 deletions

diff --git a/include/linux/sched.h b/include/linux/sched.h
index c731296e5e93..ff154e10752b 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -801,7 +801,7 @@ enum cpu_idle_type {
 #define SD_POWERSAVINGS_BALANCE 0x0100  /* Balance for power savings */
 #define SD_SHARE_PKG_RESOURCES  0x0200  /* Domain members share cpu pkg resources */
 #define SD_SERIALIZE            0x0400  /* Only a single load balancing instance */
-
+#define SD_ASYM_PACKING         0x0800  /* Place busy groups earlier in the domain */
 #define SD_PREFER_SIBLING       0x1000  /* Prefer to place tasks in a sibling domain */
 
 enum powersavings_balance_level {
@@ -836,6 +836,8 @@ static inline int sd_balance_for_package_power(void)
         return SD_PREFER_SIBLING;
 }
 
+extern int __weak arch_sd_sibiling_asym_packing(void);
+
 /*
  * Optimise SD flags for power savings:
  * SD_BALANCE_NEWIDLE helps agressive task consolidation and power savings.
diff --git a/include/linux/topology.h b/include/linux/topology.h
index c44df50a05ab..cf57f30d0dcb 100644
--- a/include/linux/topology.h
+++ b/include/linux/topology.h
@@ -103,6 +103,7 @@ int arch_update_cpu_topology(void);
                                 | 1*SD_SHARE_PKG_RESOURCES              \
                                 | 0*SD_SERIALIZE                        \
                                 | 0*SD_PREFER_SIBLING                   \
+                                | arch_sd_sibiling_asym_packing()       \
                                 ,                                       \
         .last_balance           = jiffies,                              \
         .balance_interval       = 1,                                    \
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index b9b3462483b7..593424f91a8a 100644
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -2458,11 +2458,53 @@ static inline void update_sg_lb_stats(struct sched_domain *sd,
 }
 
 /**
+ * update_sd_pick_busiest - return 1 on busiest group
+ * @sd: sched_domain whose statistics are to be checked
+ * @sds: sched_domain statistics
+ * @sg: sched_group candidate to be checked for being the busiest
+ * @sds: sched_group statistics
+ *
+ * Determine if @sg is a busier group than the previously selected
+ * busiest group.
+ */
+static bool update_sd_pick_busiest(struct sched_domain *sd,
+                                   struct sd_lb_stats *sds,
+                                   struct sched_group *sg,
+                                   struct sg_lb_stats *sgs,
+                                   int this_cpu)
+{
+        if (sgs->avg_load <= sds->max_load)
+                return false;
+
+        if (sgs->sum_nr_running > sgs->group_capacity)
+                return true;
+
+        if (sgs->group_imb)
+                return true;
+
+        /*
+         * ASYM_PACKING needs to move all the work to the lowest
+         * numbered CPUs in the group, therefore mark all groups
+         * higher than ourself as busy.
+         */
+        if ((sd->flags & SD_ASYM_PACKING) && sgs->sum_nr_running &&
+            this_cpu < group_first_cpu(sg)) {
+                if (!sds->busiest)
+                        return true;
+
+                if (group_first_cpu(sds->busiest) > group_first_cpu(sg))
+                        return true;
+        }
+
+        return false;
+}
+
+/**
  * update_sd_lb_stats - Update sched_group's statistics for load balancing.
  * @sd: sched_domain whose statistics are to be updated.
  * @this_cpu: Cpu for which load balance is currently performed.
  * @idle: Idle status of this_cpu
- * @sd_idle: Idle status of the sched_domain containing group.
+ * @sd_idle: Idle status of the sched_domain containing sg.
  * @cpus: Set of cpus considered for load balancing.
  * @balance: Should we balance.
  * @sds: variable to hold the statistics for this sched_domain.
@@ -2473,7 +2515,7 @@ static inline void update_sd_lb_stats(struct sched_domain *sd, int this_cpu,
                         struct sd_lb_stats *sds)
 {
         struct sched_domain *child = sd->child;
-        struct sched_group *group = sd->groups;
+        struct sched_group *sg = sd->groups;
         struct sg_lb_stats sgs;
         int load_idx, prefer_sibling = 0;
 
@@ -2486,21 +2528,20 @@ static inline void update_sd_lb_stats(struct sched_domain *sd, int this_cpu,
         do {
                 int local_group;
 
-                local_group = cpumask_test_cpu(this_cpu,
-                                               sched_group_cpus(group));
+                local_group = cpumask_test_cpu(this_cpu, sched_group_cpus(sg));
                 memset(&sgs, 0, sizeof(sgs));
-                update_sg_lb_stats(sd, group, this_cpu, idle, load_idx, sd_idle,
+                update_sg_lb_stats(sd, sg, this_cpu, idle, load_idx, sd_idle,
                                 local_group, cpus, balance, &sgs);
 
                 if (local_group && !(*balance))
                         return;
 
                 sds->total_load += sgs.group_load;
-                sds->total_pwr += group->cpu_power;
+                sds->total_pwr += sg->cpu_power;
 
                 /*
                  * In case the child domain prefers tasks go to siblings
-                 * first, lower the group capacity to one so that we'll try
+                 * first, lower the sg capacity to one so that we'll try
                  * and move all the excess tasks away.
                  */
                 if (prefer_sibling)
@@ -2508,23 +2549,72 @@ static inline void update_sd_lb_stats(struct sched_domain *sd, int this_cpu,
 
                 if (local_group) {
                         sds->this_load = sgs.avg_load;
-                        sds->this = group;
+                        sds->this = sg;
                         sds->this_nr_running = sgs.sum_nr_running;
                         sds->this_load_per_task = sgs.sum_weighted_load;
-                } else if (sgs.avg_load > sds->max_load &&
-                           (sgs.sum_nr_running > sgs.group_capacity ||
-                                sgs.group_imb)) {
+                } else if (update_sd_pick_busiest(sd, sds, sg, &sgs, this_cpu)) {
                         sds->max_load = sgs.avg_load;
-                        sds->busiest = group;
+                        sds->busiest = sg;
                         sds->busiest_nr_running = sgs.sum_nr_running;
                         sds->busiest_group_capacity = sgs.group_capacity;
                         sds->busiest_load_per_task = sgs.sum_weighted_load;
                         sds->group_imb = sgs.group_imb;
                 }
 
-                update_sd_power_savings_stats(group, sds, local_group, &sgs);
-                group = group->next;
-        } while (group != sd->groups);
+                update_sd_power_savings_stats(sg, sds, local_group, &sgs);
+                sg = sg->next;
+        } while (sg != sd->groups);
+}
+
+int __weak arch_sd_sibiling_asym_packing(void)
+{
+       return 0*SD_ASYM_PACKING;
+}
+
+/**
+ * check_asym_packing - Check to see if the group is packed into the
+ *                      sched doman.
+ *
+ * This is primarily intended to used at the sibling level.  Some
+ * cores like POWER7 prefer to use lower numbered SMT threads.  In the
+ * case of POWER7, it can move to lower SMT modes only when higher
+ * threads are idle.  When in lower SMT modes, the threads will
+ * perform better since they share less core resources.  Hence when we
+ * have idle threads, we want them to be the higher ones.
+ *
+ * This packing function is run on idle threads.  It checks to see if
+ * the busiest CPU in this domain (core in the P7 case) has a higher
+ * CPU number than the packing function is being run on.  Here we are
+ * assuming lower CPU number will be equivalent to lower a SMT thread
+ * number.
+ *
+ * @sd: The sched_domain whose packing is to be checked.
+ * @sds: Statistics of the sched_domain which is to be packed
+ * @this_cpu: The cpu at whose sched_domain we're performing load-balance.
+ * @imbalance: returns amount of imbalanced due to packing.
+ *
+ * Returns 1 when packing is required and a task should be moved to
+ * this CPU.  The amount of the imbalance is returned in *imbalance.
+ */
+static int check_asym_packing(struct sched_domain *sd,
+                              struct sd_lb_stats *sds,
+                              int this_cpu, unsigned long *imbalance)
+{
+        int busiest_cpu;
+
+        if (!(sd->flags & SD_ASYM_PACKING))
+                return 0;
+
+        if (!sds->busiest)
+                return 0;
+
+        busiest_cpu = group_first_cpu(sds->busiest);
+        if (this_cpu > busiest_cpu)
+                return 0;
+
+        *imbalance = DIV_ROUND_CLOSEST(sds->max_load * sds->busiest->cpu_power,
+                                       SCHED_LOAD_SCALE);
+        return 1;
 }
 
 /**
@@ -2719,6 +2809,10 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
         if (!(*balance))
                 goto ret;
 
+        if ((idle == CPU_IDLE || idle == CPU_NEWLY_IDLE) &&
+            check_asym_packing(sd, &sds, this_cpu, imbalance))
+                return sds.busiest;
+
         if (!sds.busiest || sds.busiest_nr_running == 0)
                 goto out_balanced;
 
@@ -2808,9 +2902,19 @@ find_busiest_queue(struct sched_domain *sd, struct sched_group *group,
 /* Working cpumask for load_balance and load_balance_newidle. */
 static DEFINE_PER_CPU(cpumask_var_t, load_balance_tmpmask);
 
-static int need_active_balance(struct sched_domain *sd, int sd_idle, int idle)
+static int need_active_balance(struct sched_domain *sd, int sd_idle, int idle,
+                               int busiest_cpu, int this_cpu)
 {
         if (idle == CPU_NEWLY_IDLE) {
+
+                /*
+                 * ASYM_PACKING needs to force migrate tasks from busy but
+                 * higher numbered CPUs in order to pack all tasks in the
+                 * lowest numbered CPUs.
+                 */
+                if ((sd->flags & SD_ASYM_PACKING) && busiest_cpu > this_cpu)
+                        return 1;
+
                 /*
                  * The only task running in a non-idle cpu can be moved to this
                  * cpu in an attempt to completely freeup the other CPU
@@ -2929,7 +3033,8 @@ redo:
                 schedstat_inc(sd, lb_failed[idle]);
                 sd->nr_balance_failed++;
 
-                if (need_active_balance(sd, sd_idle, idle)) {
+                if (need_active_balance(sd, sd_idle, idle, cpu_of(busiest),
+                                        this_cpu)) {
                         raw_spin_lock_irqsave(&busiest->lock, flags);
 
                         /* don't kick the active_load_balance_cpu_stop,