sched: fix bug in balance_tasks()

There are two problems with balance_tasks() and how it used:

1. The variables best_prio and best_prio_seen (inherited from the old
move_tasks()) were only required to handle problems caused by the
active/expired arrays, the order in which they were processed and the
possibility that the task with the highest priority could be on either.
  These issues are no longer present and the extra overhead associated
with their use is unnecessary (and possibly wrong).

2. In the absence of CONFIG_FAIR_GROUP_SCHED being set, the same
this_best_prio variable needs to be used by all scheduling classes or
there is a risk of moving too much load.  E.g. if the highest priority
task on this at the beginning is a fairly low priority task and the rt
class migrates a task (during its turn) then that moved task becomes the
new highest priority task on this_rq but when the sched_fair class
initializes its copy of this_best_prio it will get the priority of the
original highest priority task as, due to the run queue locks being
held, the reschedule triggered by pull_task() will not have taken place.
  This could result in inappropriate overriding of skip_for_load and
excessive load being moved.

The attached patch addresses these problems by deleting all reference to
best_prio and best_prio_seen and making this_best_prio a reference
parameter to the various functions involved.

load_balance_fair() has also been modified so that this_best_prio is
only reset (in the loop) if CONFIG_FAIR_GROUP_SCHED is set.  This should
preserve the effect of helping spread groups' higher priority tasks
around the available CPUs while improving system performance when
CONFIG_FAIR_GROUP_SCHED isn't set.

Signed-off-by: Peter Williams <pwil3058@bigpond.net.au>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

1 files changed, 11 insertions, 15 deletions

diff --git a/kernel/sched.c b/kernel/sched.c
index 85b93118d244..1fa07c14624e 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -745,8 +745,7 @@ static int balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
                       unsigned long max_nr_move, unsigned long max_load_move,
                       struct sched_domain *sd, enum cpu_idle_type idle,
                       int *all_pinned, unsigned long *load_moved,
-                      int this_best_prio, int best_prio, int best_prio_seen,
-                      struct rq_iterator *iterator);
+                      int *this_best_prio, struct rq_iterator *iterator);
 
 #include "sched_stats.h"
 #include "sched_rt.c"
@@ -2165,8 +2164,7 @@ static int balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
                       unsigned long max_nr_move, unsigned long max_load_move,
                       struct sched_domain *sd, enum cpu_idle_type idle,
                       int *all_pinned, unsigned long *load_moved,
-                      int this_best_prio, int best_prio, int best_prio_seen,
-                      struct rq_iterator *iterator)
+                      int *this_best_prio, struct rq_iterator *iterator)
 {
         int pulled = 0, pinned = 0, skip_for_load;
         struct task_struct *p;
@@ -2191,12 +2189,8 @@ next:
          */
         skip_for_load = (p->se.load.weight >> 1) > rem_load_move +
                                                          SCHED_LOAD_SCALE_FUZZ;
-        if (skip_for_load && p->prio < this_best_prio)
-                skip_for_load = !best_prio_seen && p->prio == best_prio;
-        if (skip_for_load ||
+        if ((skip_for_load && p->prio >= *this_best_prio) ||
             !can_migrate_task(p, busiest, this_cpu, sd, idle, &pinned)) {
-
-                best_prio_seen |= p->prio == best_prio;
                 p = iterator->next(iterator->arg);
                 goto next;
         }
@@ -2210,8 +2204,8 @@ next:
          * and the prescribed amount of weighted load.
          */
         if (pulled < max_nr_move && rem_load_move > 0) {
-                if (p->prio < this_best_prio)
-                        this_best_prio = p->prio;
+                if (p->prio < *this_best_prio)
+                        *this_best_prio = p->prio;
                 p = iterator->next(iterator->arg);
                 goto next;
         }
@@ -2243,12 +2237,13 @@ static int move_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
 {
         struct sched_class *class = sched_class_highest;
         unsigned long total_load_moved = 0;
+        int this_best_prio = this_rq->curr->prio;
 
         do {
                 total_load_moved +=
                         class->load_balance(this_rq, this_cpu, busiest,
                                 ULONG_MAX, max_load_move - total_load_moved,
-                                sd, idle, all_pinned);
+                                sd, idle, all_pinned, &this_best_prio);
                 class = class->next;
         } while (class && max_load_move > total_load_moved);
 
@@ -2266,10 +2261,12 @@ static int move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
                          struct sched_domain *sd, enum cpu_idle_type idle)
 {
         struct sched_class *class;
+        int this_best_prio = MAX_PRIO;
 
         for (class = sched_class_highest; class; class = class->next)
                 if (class->load_balance(this_rq, this_cpu, busiest,
-                                        1, ULONG_MAX, sd, idle, NULL))
+                                        1, ULONG_MAX, sd, idle, NULL,
+                                        &this_best_prio))
                         return 1;
 
         return 0;
@@ -3184,8 +3181,7 @@ static int balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
                       unsigned long max_nr_move, unsigned long max_load_move,
                       struct sched_domain *sd, enum cpu_idle_type idle,
                       int *all_pinned, unsigned long *load_moved,
-                      int this_best_prio, int best_prio, int best_prio_seen,
-                      struct rq_iterator *iterator)
+                      int *this_best_prio, struct rq_iterator *iterator)
 {
         *load_moved = 0;