1 files changed, 32 insertions, 26 deletions

diff --git a/kernel/sched.c b/kernel/sched.c
index 52b98675acb2..3f7c5eb254e2 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -301,7 +301,7 @@ struct cfs_rq {
         /* 'curr' points to currently running entity on this cfs_rq.
          * It is set to NULL otherwise (i.e when none are currently running).
          */
-        struct sched_entity *curr;
+        struct sched_entity *curr, *next;
 
         unsigned long nr_spread_over;
 
@@ -1084,7 +1084,7 @@ calc_delta_mine(unsigned long delta_exec, unsigned long weight,
         u64 tmp;
 
         if (unlikely(!lw->inv_weight))
-                lw->inv_weight = (WMULT_CONST - lw->weight/2) / lw->weight + 1;
+                lw->inv_weight = (WMULT_CONST-lw->weight/2) / (lw->weight+1);
 
         tmp = (u64)delta_exec * weight;
         /*
@@ -1108,11 +1108,13 @@ calc_delta_fair(unsigned long delta_exec, struct load_weight *lw)
 static inline void update_load_add(struct load_weight *lw, unsigned long inc)
 {
         lw->weight += inc;
+        lw->inv_weight = 0;
 }
 
 static inline void update_load_sub(struct load_weight *lw, unsigned long dec)
 {
         lw->weight -= dec;
+        lw->inv_weight = 0;
 }
 
 /*
@@ -1394,6 +1396,12 @@ task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
 {
         s64 delta;
 
+        /*
+         * Buddy candidates are cache hot:
+         */
+        if (&p->se == cfs_rq_of(&p->se)->next)
+                return 1;
+
         if (p->sched_class != &fair_sched_class)
                 return 0;
 
@@ -1853,10 +1861,11 @@ out_activate:
                 schedstat_inc(p, se.nr_wakeups_remote);
         update_rq_clock(rq);
         activate_task(rq, p, 1);
-        check_preempt_curr(rq, p);
         success = 1;
 
 out_running:
+        check_preempt_curr(rq, p);
+
         p->state = TASK_RUNNING;
 #ifdef CONFIG_SMP
         if (p->sched_class->task_wake_up)
@@ -1890,6 +1899,8 @@ static void __sched_fork(struct task_struct *p)
         p->se.exec_start                = 0;
         p->se.sum_exec_runtime          = 0;
         p->se.prev_sum_exec_runtime     = 0;
+        p->se.last_wakeup               = 0;
+        p->se.avg_overlap               = 0;
 
 #ifdef CONFIG_SCHEDSTATS
         p->se.wait_start                = 0;
@@ -4268,11 +4279,10 @@ void rt_mutex_setprio(struct task_struct *p, int prio)
         oldprio = p->prio;
         on_rq = p->se.on_rq;
         running = task_current(rq, p);
-        if (on_rq) {
+        if (on_rq)
                 dequeue_task(rq, p, 0);
-                if (running)
-                        p->sched_class->put_prev_task(rq, p);
-        }
+        if (running)
+                p->sched_class->put_prev_task(rq, p);
 
         if (rt_prio(prio))
                 p->sched_class = &rt_sched_class;
@@ -4281,10 +4291,9 @@ void rt_mutex_setprio(struct task_struct *p, int prio)
 
         p->prio = prio;
 
+        if (running)
+                p->sched_class->set_curr_task(rq);
         if (on_rq) {
-                if (running)
-                        p->sched_class->set_curr_task(rq);
-
                 enqueue_task(rq, p, 0);
 
                 check_class_changed(rq, p, prev_class, oldprio, running);
@@ -4581,19 +4590,17 @@ recheck:
         update_rq_clock(rq);
         on_rq = p->se.on_rq;
         running = task_current(rq, p);
-        if (on_rq) {
+        if (on_rq)
                 deactivate_task(rq, p, 0);
-                if (running)
-                        p->sched_class->put_prev_task(rq, p);
-        }
+        if (running)
+                p->sched_class->put_prev_task(rq, p);
 
         oldprio = p->prio;
         __setscheduler(rq, p, policy, param->sched_priority);
 
+        if (running)
+                p->sched_class->set_curr_task(rq);
         if (on_rq) {
-                if (running)
-                        p->sched_class->set_curr_task(rq);
-
                 activate_task(rq, p, 0);
 
                 check_class_changed(rq, p, prev_class, oldprio, running);
@@ -5881,7 +5888,8 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
                 spin_unlock_irq(&rq->lock);
                 break;
 
-        case CPU_DOWN_PREPARE:
+        case CPU_DYING:
+        case CPU_DYING_FROZEN:
                 /* Update our root-domain */
                 rq = cpu_rq(cpu);
                 spin_lock_irqsave(&rq->lock, flags);
@@ -7617,11 +7625,10 @@ void sched_move_task(struct task_struct *tsk)
         running = task_current(rq, tsk);
         on_rq = tsk->se.on_rq;
 
-        if (on_rq) {
+        if (on_rq)
                 dequeue_task(rq, tsk, 0);
-                if (unlikely(running))
-                        tsk->sched_class->put_prev_task(rq, tsk);
-        }
+        if (unlikely(running))
+                tsk->sched_class->put_prev_task(rq, tsk);
 
         set_task_rq(tsk, task_cpu(tsk));
 
@@ -7630,11 +7637,10 @@ void sched_move_task(struct task_struct *tsk)
                 tsk->sched_class->moved_group(tsk);
 #endif
 
-        if (on_rq) {
-                if (unlikely(running))
-                        tsk->sched_class->set_curr_task(rq);
+        if (unlikely(running))
+                tsk->sched_class->set_curr_task(rq);
+        if (on_rq)
                 enqueue_task(rq, tsk, 0);
-        }
 
         task_rq_unlock(rq, &flags);
 }