1 files changed, 48 insertions, 21 deletions

diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index bdf64346b4d1..bc1563e7a248 100644
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -283,7 +283,7 @@ static void update_min_vruntime(struct cfs_rq *cfs_rq)
                                                    struct sched_entity,
                                                    run_node);
 
-                if (vruntime == cfs_rq->min_vruntime)
+                if (!cfs_rq->curr)
                         vruntime = se->vruntime;
                 else
                         vruntime = min_vruntime(vruntime, se->vruntime);
@@ -429,7 +429,10 @@ static u64 sched_slice(struct cfs_rq *cfs_rq, struct sched_entity *se)
         u64 slice = __sched_period(cfs_rq->nr_running + !se->on_rq);
 
         for_each_sched_entity(se) {
-                struct load_weight *load = &cfs_rq->load;
+                struct load_weight *load;
+
+                cfs_rq = cfs_rq_of(se);
+                load = &cfs_rq->load;
 
                 if (unlikely(!se->on_rq)) {
                         struct load_weight lw = cfs_rq->load;
@@ -677,9 +680,13 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial)
                         unsigned long thresh = sysctl_sched_latency;
 
                         /*
-                         * convert the sleeper threshold into virtual time
+                         * Convert the sleeper threshold into virtual time.
+                         * SCHED_IDLE is a special sub-class.  We care about
+                         * fairness only relative to other SCHED_IDLE tasks,
+                         * all of which have the same weight.
                          */
-                        if (sched_feat(NORMALIZED_SLEEPER))
+                        if (sched_feat(NORMALIZED_SLEEPER) &&
+                                        task_of(se)->policy != SCHED_IDLE)
                                 thresh = calc_delta_fair(thresh, se);
 
                         vruntime -= thresh;
@@ -712,7 +719,7 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int wakeup)
                 __enqueue_entity(cfs_rq, se);
 }
 
-static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se)
+static void __clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
         if (cfs_rq->last == se)
                 cfs_rq->last = NULL;
@@ -721,6 +728,12 @@ static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se)
                 cfs_rq->next = NULL;
 }
 
+static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se)
+{
+        for_each_sched_entity(se)
+                __clear_buddies(cfs_rq_of(se), se);
+}
+
 static void
 dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep)
 {
@@ -761,8 +774,14 @@ check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr)
 
         ideal_runtime = sched_slice(cfs_rq, curr);
         delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime;
-        if (delta_exec > ideal_runtime)
+        if (delta_exec > ideal_runtime) {
                 resched_task(rq_of(cfs_rq)->curr);
+                /*
+                 * The current task ran long enough, ensure it doesn't get
+                 * re-elected due to buddy favours.
+                 */
+                clear_buddies(cfs_rq, curr);
+        }
 }
 
 static void
@@ -1172,20 +1191,15 @@ wake_affine(struct sched_domain *this_sd, struct rq *this_rq,
             int idx, unsigned long load, unsigned long this_load,
             unsigned int imbalance)
 {
-        struct task_struct *curr = this_rq->curr;
-        struct task_group *tg;
         unsigned long tl = this_load;
         unsigned long tl_per_task;
+        struct task_group *tg;
         unsigned long weight;
         int balanced;
 
         if (!(this_sd->flags & SD_WAKE_AFFINE) || !sched_feat(AFFINE_WAKEUPS))
                 return 0;
 
-        if (sync && (curr->se.avg_overlap > sysctl_sched_migration_cost ||
-                        p->se.avg_overlap > sysctl_sched_migration_cost))
-                sync = 0;
-
         /*
          * If sync wakeup then subtract the (maximum possible)
          * effect of the currently running task from the load
@@ -1387,14 +1401,18 @@ wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se)
 
 static void set_last_buddy(struct sched_entity *se)
 {
-        for_each_sched_entity(se)
-                cfs_rq_of(se)->last = se;
+        if (likely(task_of(se)->policy != SCHED_IDLE)) {
+                for_each_sched_entity(se)
+                        cfs_rq_of(se)->last = se;
+        }
 }
 
 static void set_next_buddy(struct sched_entity *se)
 {
-        for_each_sched_entity(se)
-                cfs_rq_of(se)->next = se;
+        if (likely(task_of(se)->policy != SCHED_IDLE)) {
+                for_each_sched_entity(se)
+                        cfs_rq_of(se)->next = se;
+        }
 }
 
 /*
@@ -1440,18 +1458,22 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int sync)
                 return;
 
         /*
-         * Batch tasks do not preempt (their preemption is driven by
+         * Batch and idle tasks do not preempt (their preemption is driven by
          * the tick):
          */
-        if (unlikely(p->policy == SCHED_BATCH))
+        if (unlikely(p->policy != SCHED_NORMAL))
                 return;
 
+        /* Idle tasks are by definition preempted by everybody. */
+        if (unlikely(curr->policy == SCHED_IDLE)) {
+                resched_task(curr);
+                return;
+        }
+
         if (!sched_feat(WAKEUP_PREEMPT))
                 return;
 
-        if (sched_feat(WAKEUP_OVERLAP) && (sync ||
-                        (se->avg_overlap < sysctl_sched_migration_cost &&
-                         pse->avg_overlap < sysctl_sched_migration_cost))) {
+        if (sched_feat(WAKEUP_OVERLAP) && sync) {
                 resched_task(curr);
                 return;
         }
@@ -1482,6 +1504,11 @@ static struct task_struct *pick_next_task_fair(struct rq *rq)
 
         do {
                 se = pick_next_entity(cfs_rq);
+                /*
+                 * If se was a buddy, clear it so that it will have to earn
+                 * the favour again.
+                 */
+                __clear_buddies(cfs_rq, se);
                 set_next_entity(cfs_rq, se);
                 cfs_rq = group_cfs_rq(se);
         } while (cfs_rq);