1 files changed, 647 insertions, 679 deletions

diff --git a/kernel/sched.c b/kernel/sched.c
index 3c2a54f70ffe..41541d79e3c8 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -55,9 +55,9 @@
 #include <linux/cpu.h>
 #include <linux/cpuset.h>
 #include <linux/percpu.h>
-#include <linux/kthread.h>
 #include <linux/proc_fs.h>
 #include <linux/seq_file.h>
+#include <linux/stop_machine.h>
 #include <linux/sysctl.h>
 #include <linux/syscalls.h>
 #include <linux/times.h>
@@ -77,6 +77,7 @@
 #include <asm/irq_regs.h>
 
 #include "sched_cpupri.h"
+#include "workqueue_sched.h"
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/sched.h>
@@ -306,52 +307,6 @@ static int init_task_group_load = INIT_TASK_GROUP_LOAD;
  */
 struct task_group init_task_group;
 
-/* return group to which a task belongs */
-static inline struct task_group *task_group(struct task_struct *p)
-{
-        struct task_group *tg;
-
-#ifdef CONFIG_CGROUP_SCHED
-        tg = container_of(task_subsys_state(p, cpu_cgroup_subsys_id),
-                                struct task_group, css);
-#else
-        tg = &init_task_group;
-#endif
-        return tg;
-}
-
-/* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */
-static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
-{
-        /*
-         * Strictly speaking this rcu_read_lock() is not needed since the
-         * task_group is tied to the cgroup, which in turn can never go away
-         * as long as there are tasks attached to it.
-         *
-         * However since task_group() uses task_subsys_state() which is an
-         * rcu_dereference() user, this quiets CONFIG_PROVE_RCU.
-         */
-        rcu_read_lock();
-#ifdef CONFIG_FAIR_GROUP_SCHED
-        p->se.cfs_rq = task_group(p)->cfs_rq[cpu];
-        p->se.parent = task_group(p)->se[cpu];
-#endif
-
-#ifdef CONFIG_RT_GROUP_SCHED
-        p->rt.rt_rq  = task_group(p)->rt_rq[cpu];
-        p->rt.parent = task_group(p)->rt_se[cpu];
-#endif
-        rcu_read_unlock();
-}
-
-#else
-
-static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { }
-static inline struct task_group *task_group(struct task_struct *p)
-{
-        return NULL;
-}
-
 #endif  /* CONFIG_CGROUP_SCHED */
 
 /* CFS-related fields in a runqueue */
@@ -502,9 +457,13 @@ struct rq {
         unsigned long nr_running;
         #define CPU_LOAD_IDX_MAX 5
         unsigned long cpu_load[CPU_LOAD_IDX_MAX];
+        unsigned long last_load_update_tick;
 #ifdef CONFIG_NO_HZ
-        unsigned char in_nohz_recently;
+        u64 nohz_stamp;
+        unsigned char nohz_balance_kick;
 #endif
+        unsigned int skip_clock_update;
+
         /* capture load from *all* tasks on this cpu: */
         struct load_weight load;
         unsigned long nr_load_updates;
@@ -541,20 +500,20 @@ struct rq {
         struct root_domain *rd;
         struct sched_domain *sd;
 
+        unsigned long cpu_power;
+
         unsigned char idle_at_tick;
         /* For active balancing */
         int post_schedule;
         int active_balance;
         int push_cpu;
+        struct cpu_stop_work active_balance_work;
         /* cpu of this runqueue: */
         int cpu;
         int online;
 
         unsigned long avg_load_per_task;
 
-        struct task_struct *migration_thread;
-        struct list_head migration_queue;
-
         u64 rt_avg;
         u64 age_stamp;
         u64 idle_stamp;
@@ -602,6 +561,13 @@ static inline
 void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
 {
         rq->curr->sched_class->check_preempt_curr(rq, p, flags);
+
+        /*
+         * A queue event has occurred, and we're going to schedule.  In
+         * this case, we can save a useless back to back clock update.
+         */
+        if (test_tsk_need_resched(p))
+                rq->skip_clock_update = 1;
 }
 
 static inline int cpu_of(struct rq *rq)
@@ -634,9 +600,53 @@ static inline int cpu_of(struct rq *rq)
 #define cpu_curr(cpu)           (cpu_rq(cpu)->curr)
 #define raw_rq()                (&__raw_get_cpu_var(runqueues))
 
+#ifdef CONFIG_CGROUP_SCHED
+
+/*
+ * Return the group to which this tasks belongs.
+ *
+ * We use task_subsys_state_check() and extend the RCU verification
+ * with lockdep_is_held(&task_rq(p)->lock) because cpu_cgroup_attach()
+ * holds that lock for each task it moves into the cgroup. Therefore
+ * by holding that lock, we pin the task to the current cgroup.
+ */
+static inline struct task_group *task_group(struct task_struct *p)
+{
+        struct cgroup_subsys_state *css;
+
+        css = task_subsys_state_check(p, cpu_cgroup_subsys_id,
+                        lockdep_is_held(&task_rq(p)->lock));
+        return container_of(css, struct task_group, css);
+}
+
+/* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */
+static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
+{
+#ifdef CONFIG_FAIR_GROUP_SCHED
+        p->se.cfs_rq = task_group(p)->cfs_rq[cpu];
+        p->se.parent = task_group(p)->se[cpu];
+#endif
+
+#ifdef CONFIG_RT_GROUP_SCHED
+        p->rt.rt_rq  = task_group(p)->rt_rq[cpu];
+        p->rt.parent = task_group(p)->rt_se[cpu];
+#endif
+}
+
+#else /* CONFIG_CGROUP_SCHED */
+
+static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { }
+static inline struct task_group *task_group(struct task_struct *p)
+{
+        return NULL;
+}
+
+#endif /* CONFIG_CGROUP_SCHED */
+
 inline void update_rq_clock(struct rq *rq)
 {
-        rq->clock = sched_clock_cpu(cpu_of(rq));
+        if (!rq->skip_clock_update)
+                rq->clock = sched_clock_cpu(cpu_of(rq));
 }
 
 /*
@@ -914,16 +924,12 @@ static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
 #endif /* __ARCH_WANT_UNLOCKED_CTXSW */
 
 /*
- * Check whether the task is waking, we use this to synchronize against
- * ttwu() so that task_cpu() reports a stable number.
- *
- * We need to make an exception for PF_STARTING tasks because the fork
- * path might require task_rq_lock() to work, eg. it can call
- * set_cpus_allowed_ptr() from the cpuset clone_ns code.
+ * Check whether the task is waking, we use this to synchronize ->cpus_allowed
+ * against ttwu().
  */
 static inline int task_is_waking(struct task_struct *p)
 {
-        return unlikely((p->state == TASK_WAKING) && !(p->flags & PF_STARTING));
+        return unlikely(p->state == TASK_WAKING);
 }
 
 /*
@@ -936,11 +942,9 @@ static inline struct rq *__task_rq_lock(struct task_struct *p)
         struct rq *rq;
 
         for (;;) {
-                while (task_is_waking(p))
-                        cpu_relax();
                 rq = task_rq(p);
                 raw_spin_lock(&rq->lock);
-                if (likely(rq == task_rq(p) && !task_is_waking(p)))
+                if (likely(rq == task_rq(p)))
                         return rq;
                 raw_spin_unlock(&rq->lock);
         }
@@ -957,25 +961,15 @@ static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)
         struct rq *rq;
 
         for (;;) {
-                while (task_is_waking(p))
-                        cpu_relax();
                 local_irq_save(*flags);
                 rq = task_rq(p);
                 raw_spin_lock(&rq->lock);
-                if (likely(rq == task_rq(p) && !task_is_waking(p)))
+                if (likely(rq == task_rq(p)))
                         return rq;
                 raw_spin_unlock_irqrestore(&rq->lock, *flags);
         }
 }
 
-void task_rq_unlock_wait(struct task_struct *p)
-{
-        struct rq *rq = task_rq(p);
-
-        smp_mb(); /* spin-unlock-wait is not a full memory barrier */
-        raw_spin_unlock_wait(&rq->lock);
-}
-
 static void __task_rq_unlock(struct rq *rq)
         __releases(rq->lock)
 {
@@ -1201,6 +1195,27 @@ static void resched_cpu(int cpu)
 
 #ifdef CONFIG_NO_HZ
 /*
+ * In the semi idle case, use the nearest busy cpu for migrating timers
+ * from an idle cpu.  This is good for power-savings.
+ *
+ * We don't do similar optimization for completely idle system, as
+ * selecting an idle cpu will add more delays to the timers than intended
+ * (as that cpu's timer base may not be uptodate wrt jiffies etc).
+ */
+int get_nohz_timer_target(void)
+{
+        int cpu = smp_processor_id();
+        int i;
+        struct sched_domain *sd;
+
+        for_each_domain(cpu, sd) {
+                for_each_cpu(i, sched_domain_span(sd))
+                        if (!idle_cpu(i))
+                                return i;
+        }
+        return cpu;
+}
+/*
  * When add_timer_on() enqueues a timer into the timer wheel of an
  * idle CPU then this timer might expire before the next timer event
  * which is scheduled to wake up that CPU. In case of a completely
@@ -1239,6 +1254,7 @@ void wake_up_idle_cpu(int cpu)
         if (!tsk_is_polling(rq->idle))
                 smp_send_reschedule(cpu);
 }
+
 #endif /* CONFIG_NO_HZ */
 
 static u64 sched_avg_period(void)
@@ -1251,6 +1267,12 @@ static void sched_avg_update(struct rq *rq)
         s64 period = sched_avg_period();
 
         while ((s64)(rq->clock - rq->age_stamp) > period) {
+                /*
+                 * Inline assembly required to prevent the compiler
+                 * optimising this loop into a divmod call.
+                 * See __iter_div_u64_rem() for another example of this.
+                 */
+                asm("" : "+rm" (rq->age_stamp));
                 rq->age_stamp += period;
                 rq->rt_avg /= 2;
         }
@@ -1495,24 +1517,9 @@ static unsigned long target_load(int cpu, int type)
         return max(rq->cpu_load[type-1], total);
 }
 
-static struct sched_group *group_of(int cpu)
-{
-        struct sched_domain *sd = rcu_dereference_sched(cpu_rq(cpu)->sd);
-
-        if (!sd)
-                return NULL;
-
-        return sd->groups;
-}
-
 static unsigned long power_of(int cpu)
 {
-        struct sched_group *group = group_of(cpu);
-
-        if (!group)
-                return SCHED_LOAD_SCALE;
-
-        return group->cpu_power;
+        return cpu_rq(cpu)->cpu_power;
 }
 
 static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd);
@@ -1658,7 +1665,7 @@ static void update_shares(struct sched_domain *sd)
         if (root_task_group_empty())
                 return;
 
-        now = cpu_clock(raw_smp_processor_id());
+        now = local_clock();
         elapsed = now - sd->last_update;
 
         if (elapsed >= (s64)(u64)sysctl_sched_shares_ratelimit) {
@@ -1669,9 +1676,6 @@ static void update_shares(struct sched_domain *sd)
 
 static void update_h_load(long cpu)
 {
-        if (root_task_group_empty())
-                return;
-
         walk_tg_tree(tg_load_down, tg_nop, (void *)cpu);
 }
 
@@ -1781,8 +1785,6 @@ static void double_rq_lock(struct rq *rq1, struct rq *rq2)
                         raw_spin_lock_nested(&rq1->lock, SINGLE_DEPTH_NESTING);
                 }
         }
-        update_rq_clock(rq1);
-        update_rq_clock(rq2);
 }
 
 /*
@@ -1813,9 +1815,10 @@ static void cfs_rq_set_shares(struct cfs_rq *cfs_rq, unsigned long shares)
 }
 #endif
 
-static void calc_load_account_active(struct rq *this_rq);
+static void calc_load_account_idle(struct rq *this_rq);
 static void update_sysctl(void);
 static int get_update_sysctl_factor(void);
+static void update_cpu_load(struct rq *this_rq);
 
 static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
 {
@@ -1852,8 +1855,8 @@ static void dec_nr_running(struct rq *rq)
 static void set_load_weight(struct task_struct *p)
 {
         if (task_has_rt_policy(p)) {
-                p->se.load.weight = prio_to_weight[0] * 2;
-                p->se.load.inv_weight = prio_to_wmult[0] >> 1;
+                p->se.load.weight = 0;
+                p->se.load.inv_weight = WMULT_CONST;
                 return;
         }
 
@@ -1870,62 +1873,43 @@ static void set_load_weight(struct task_struct *p)
         p->se.load.inv_weight = prio_to_wmult[p->static_prio - MAX_RT_PRIO];
 }
 
-static void update_avg(u64 *avg, u64 sample)
-{
-        s64 diff = sample - *avg;
-        *avg += diff >> 3;
-}
-
-static void
-enqueue_task(struct rq *rq, struct task_struct *p, int wakeup, bool head)
+static void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
 {
-        if (wakeup)
-                p->se.start_runtime = p->se.sum_exec_runtime;
-
+        update_rq_clock(rq);
         sched_info_queued(p);
-        p->sched_class->enqueue_task(rq, p, wakeup, head);
+        p->sched_class->enqueue_task(rq, p, flags);
         p->se.on_rq = 1;
 }
 
-static void dequeue_task(struct rq *rq, struct task_struct *p, int sleep)
+static void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
 {
-        if (sleep) {
-                if (p->se.last_wakeup) {
-                        update_avg(&p->se.avg_overlap,
-                                p->se.sum_exec_runtime - p->se.last_wakeup);
-                        p->se.last_wakeup = 0;
-                } else {
-                        update_avg(&p->se.avg_wakeup,
-                                sysctl_sched_wakeup_granularity);
-                }
-        }
-
+        update_rq_clock(rq);
         sched_info_dequeued(p);
-        p->sched_class->dequeue_task(rq, p, sleep);
+        p->sched_class->dequeue_task(rq, p, flags);
         p->se.on_rq = 0;
 }
 
 /*
  * activate_task - move a task to the runqueue.
  */
-static void activate_task(struct rq *rq, struct task_struct *p, int wakeup)
+static void activate_task(struct rq *rq, struct task_struct *p, int flags)
 {
         if (task_contributes_to_load(p))
                 rq->nr_uninterruptible--;
 
-        enqueue_task(rq, p, wakeup, false);
+        enqueue_task(rq, p, flags);
         inc_nr_running(rq);
 }
 
 /*
  * deactivate_task - remove a task from the runqueue.
  */
-static void deactivate_task(struct rq *rq, struct task_struct *p, int sleep)
+static void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
 {
         if (task_contributes_to_load(p))
                 rq->nr_uninterruptible++;
 
-        dequeue_task(rq, p, sleep);
+        dequeue_task(rq, p, flags);
         dec_nr_running(rq);
 }
 
@@ -2054,21 +2038,18 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
         __set_task_cpu(p, new_cpu);
 }
 
-struct migration_req {
-        struct list_head list;
-
+struct migration_arg {
         struct task_struct *task;
         int dest_cpu;
-
-        struct completion done;
 };
 
+static int migration_cpu_stop(void *data);
+
 /*
  * The task's runqueue lock must be held.
  * Returns true if you have to wait for migration thread.
  */
-static int
-migrate_task(struct task_struct *p, int dest_cpu, struct migration_req *req)
+static bool migrate_task(struct task_struct *p, int dest_cpu)
 {
         struct rq *rq = task_rq(p);
 
@@ -2076,58 +2057,7 @@ migrate_task(struct task_struct *p, int dest_cpu, struct migration_req *req)
          * If the task is not on a runqueue (and not running), then
          * the next wake-up will properly place the task.
          */
-        if (!p->se.on_rq && !task_running(rq, p))
-                return 0;
-
-        init_completion(&req->done);
-        req->task = p;
-        req->dest_cpu = dest_cpu;
-        list_add(&req->list, &rq->migration_queue);
-
-        return 1;
-}
-
-/*
- * wait_task_context_switch -   wait for a thread to complete at least one
- *                              context switch.
- *
- * @p must not be current.
- */
-void wait_task_context_switch(struct task_struct *p)
-{
-        unsigned long nvcsw, nivcsw, flags;
-        int running;
-        struct rq *rq;
-
-        nvcsw   = p->nvcsw;
-        nivcsw  = p->nivcsw;
-        for (;;) {
-                /*
-                 * The runqueue is assigned before the actual context
-                 * switch. We need to take the runqueue lock.
-                 *
-                 * We could check initially without the lock but it is
-                 * very likely that we need to take the lock in every
-                 * iteration.
-                 */
-                rq = task_rq_lock(p, &flags);
-                running = task_running(rq, p);
-                task_rq_unlock(rq, &flags);
-
-                if (likely(!running))
-                        break;
-                /*
-                 * The switch count is incremented before the actual
-                 * context switch. We thus wait for two switches to be
-                 * sure at least one completed.
-                 */
-                if ((p->nvcsw - nvcsw) > 1)
-                        break;
-                if ((p->nivcsw - nivcsw) > 1)
-                        break;
-
-                cpu_relax();
-        }
+        return p->se.on_rq || task_running(rq, p);
 }
 
 /*
@@ -2185,7 +2115,7 @@ unsigned long wait_task_inactive(struct task_struct *p, long match_state)
                  * just go back and repeat.
                  */
                 rq = task_rq_lock(p, &flags);
-                trace_sched_wait_task(rq, p);
+                trace_sched_wait_task(p);
                 running = task_running(rq, p);
                 on_rq = p->se.on_rq;
                 ncsw = 0;
@@ -2283,6 +2213,9 @@ void task_oncpu_function_call(struct task_struct *p,
 }
 
 #ifdef CONFIG_SMP
+/*
+ * ->cpus_allowed is protected by either TASK_WAKING or rq->lock held.
+ */
 static int select_fallback_rq(int cpu, struct task_struct *p)
 {
         int dest_cpu;
@@ -2299,12 +2232,8 @@ static int select_fallback_rq(int cpu, struct task_struct *p)
                 return dest_cpu;
 
         /* No more Mr. Nice Guy. */
-        if (dest_cpu >= nr_cpu_ids) {
-                rcu_read_lock();
-                cpuset_cpus_allowed_locked(p, &p->cpus_allowed);
-                rcu_read_unlock();
-                dest_cpu = cpumask_any_and(cpu_active_mask, &p->cpus_allowed);
-
+        if (unlikely(dest_cpu >= nr_cpu_ids)) {
+                dest_cpu = cpuset_cpus_allowed_fallback(p);
                 /*
                  * Don't tell them about moving exiting tasks or
                  * kernel threads (both mm NULL), since they never
@@ -2321,17 +2250,12 @@ static int select_fallback_rq(int cpu, struct task_struct *p)
 }
 
 /*
- * Gets called from 3 sites (exec, fork, wakeup), since it is called without
- * holding rq->lock we need to ensure ->cpus_allowed is stable, this is done
- * by:
- *
- *  exec:           is unstable, retry loop
- *  fork & wake-up: serialize ->cpus_allowed against TASK_WAKING
+ * The caller (fork, wakeup) owns TASK_WAKING, ->cpus_allowed is stable.
  */
 static inline
-int select_task_rq(struct task_struct *p, int sd_flags, int wake_flags)
+int select_task_rq(struct rq *rq, struct task_struct *p, int sd_flags, int wake_flags)
 {
-        int cpu = p->sched_class->select_task_rq(p, sd_flags, wake_flags);
+        int cpu = p->sched_class->select_task_rq(rq, p, sd_flags, wake_flags);
 
         /*
          * In order not to call set_task_cpu() on a blocking task we need
@@ -2349,13 +2273,63 @@ int select_task_rq(struct task_struct *p, int sd_flags, int wake_flags)
 
         return cpu;
 }
+
+static void update_avg(u64 *avg, u64 sample)
+{
+        s64 diff = sample - *avg;
+        *avg += diff >> 3;
+}
 #endif
 
-/***
+static inline void ttwu_activate(struct task_struct *p, struct rq *rq,
+                                 bool is_sync, bool is_migrate, bool is_local,
+                                 unsigned long en_flags)
+{
+        schedstat_inc(p, se.statistics.nr_wakeups);
+        if (is_sync)
+                schedstat_inc(p, se.statistics.nr_wakeups_sync);
+        if (is_migrate)
+                schedstat_inc(p, se.statistics.nr_wakeups_migrate);
+        if (is_local)
+                schedstat_inc(p, se.statistics.nr_wakeups_local);
+        else
+                schedstat_inc(p, se.statistics.nr_wakeups_remote);
+
+        activate_task(rq, p, en_flags);
+}
+
+static inline void ttwu_post_activation(struct task_struct *p, struct rq *rq,
+                                        int wake_flags, bool success)
+{
+        trace_sched_wakeup(p, success);
+        check_preempt_curr(rq, p, wake_flags);
+
+        p->state = TASK_RUNNING;
+#ifdef CONFIG_SMP
+        if (p->sched_class->task_woken)
+                p->sched_class->task_woken(rq, p);
+
+        if (unlikely(rq->idle_stamp)) {
+                u64 delta = rq->clock - rq->idle_stamp;
+                u64 max = 2*sysctl_sched_migration_cost;
+
+                if (delta > max)
+                        rq->avg_idle = max;
+                else
+                        update_avg(&rq->avg_idle, delta);
+                rq->idle_stamp = 0;
+        }
+#endif
+        /* if a worker is waking up, notify workqueue */
+        if ((p->flags & PF_WQ_WORKER) && success)
+                wq_worker_waking_up(p, cpu_of(rq));
+}
+
+/**
  * try_to_wake_up - wake up a thread
- * @p: the to-be-woken-up thread
+ * @p: the thread to be awakened
  * @state: the mask of task states that can be woken
- * @sync: do a synchronous wakeup?
+ * @wake_flags: wake modifier flags (WF_*)
  *
  * Put it on the run-queue if it's not already there. The "current"
  * thread is always on the run-queue (except when the actual
@@ -2363,23 +2337,21 @@ int select_task_rq(struct task_struct *p, int sd_flags, int wake_flags)
  * the simpler "current->state = TASK_RUNNING" to mark yourself
  * runnable without the overhead of this.
  *
- * returns failure only if the task is already active.
+ * Returns %true if @p was woken up, %false if it was already running
+ * or @state didn't match @p's state.
  */
 static int try_to_wake_up(struct task_struct *p, unsigned int state,
                           int wake_flags)
 {
         int cpu, orig_cpu, this_cpu, success = 0;
         unsigned long flags;
+        unsigned long en_flags = ENQUEUE_WAKEUP;
         struct rq *rq;
 
-        if (!sched_feat(SYNC_WAKEUPS))
-                wake_flags &= ~WF_SYNC;
-
         this_cpu = get_cpu();
 
         smp_wmb();
         rq = task_rq_lock(p, &flags);
-        update_rq_clock(rq);
         if (!(p->state & state))
                 goto out;
 
@@ -2399,28 +2371,26 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state,
          *
          * First fix up the nr_uninterruptible count:
          */
-        if (task_contributes_to_load(p))
-                rq->nr_uninterruptible--;
+        if (task_contributes_to_load(p)) {
+                if (likely(cpu_online(orig_cpu)))
+                        rq->nr_uninterruptible--;
+                else
+                        this_rq()->nr_uninterruptible--;
+        }
         p->state = TASK_WAKING;
 
-        if (p->sched_class->task_waking)
+        if (p->sched_class->task_waking) {
                 p->sched_class->task_waking(rq, p);
+                en_flags |= ENQUEUE_WAKING;
+        }
 
-        __task_rq_unlock(rq);
-
-        cpu = select_task_rq(p, SD_BALANCE_WAKE, wake_flags);
-        if (cpu != orig_cpu) {
-                /*
-                 * Since we migrate the task without holding any rq->lock,
-                 * we need to be careful with task_rq_lock(), since that
-                 * might end up locking an invalid rq.
-                 */
+        cpu = select_task_rq(rq, p, SD_BALANCE_WAKE, wake_flags);
+        if (cpu != orig_cpu)
                 set_task_cpu(p, cpu);
-        }
+        __task_rq_unlock(rq);
 
         rq = cpu_rq(cpu);
         raw_spin_lock(&rq->lock);
-        update_rq_clock(rq);
 
         /*
          * We migrated the task without holding either rq->lock, however
@@ -2448,54 +2418,11 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state,
 
 out_activate:
 #endif /* CONFIG_SMP */
-        schedstat_inc(p, se.nr_wakeups);
-        if (wake_flags & WF_SYNC)
-                schedstat_inc(p, se.nr_wakeups_sync);
-        if (orig_cpu != cpu)
-                schedstat_inc(p, se.nr_wakeups_migrate);
-        if (cpu == this_cpu)
-                schedstat_inc(p, se.nr_wakeups_local);
-        else
-                schedstat_inc(p, se.nr_wakeups_remote);
-        activate_task(rq, p, 1);
+        ttwu_activate(p, rq, wake_flags & WF_SYNC, orig_cpu != cpu,
+                      cpu == this_cpu, en_flags);
         success = 1;
-
-        /*
-         * Only attribute actual wakeups done by this task.
-         */
-        if (!in_interrupt()) {
-                struct sched_entity *se = &current->se;
-                u64 sample = se->sum_exec_runtime;
-
-                if (se->last_wakeup)
-                        sample -= se->last_wakeup;
-                else
-                        sample -= se->start_runtime;
-                update_avg(&se->avg_wakeup, sample);
-
-                se->last_wakeup = se->sum_exec_runtime;
-        }
-
 out_running:
-        trace_sched_wakeup(rq, p, success);
-        check_preempt_curr(rq, p, wake_flags);
-
-        p->state = TASK_RUNNING;
-#ifdef CONFIG_SMP
-        if (p->sched_class->task_woken)
-                p->sched_class->task_woken(rq, p);
-
-        if (unlikely(rq->idle_stamp)) {
-                u64 delta = rq->clock - rq->idle_stamp;
-                u64 max = 2*sysctl_sched_migration_cost;
-
-                if (delta > max)
-                        rq->avg_idle = max;
-                else
-                        update_avg(&rq->avg_idle, delta);
-                rq->idle_stamp = 0;
-        }
-#endif
+        ttwu_post_activation(p, rq, wake_flags, success);
 out:
         task_rq_unlock(rq, &flags);
         put_cpu();
@@ -2504,6 +2431,37 @@ out:
 }
 
 /**
+ * try_to_wake_up_local - try to wake up a local task with rq lock held
+ * @p: the thread to be awakened
+ *
+ * Put @p on the run-queue if it's not alredy there.  The caller must
+ * ensure that this_rq() is locked, @p is bound to this_rq() and not
+ * the current task.  this_rq() stays locked over invocation.
+ */
+static void try_to_wake_up_local(struct task_struct *p)
+{
+        struct rq *rq = task_rq(p);
+        bool success = false;
+
+        BUG_ON(rq != this_rq());
+        BUG_ON(p == current);
+        lockdep_assert_held(&rq->lock);
+
+        if (!(p->state & TASK_NORMAL))
+                return;
+
+        if (!p->se.on_rq) {
+                if (likely(!task_running(rq, p))) {
+                        schedstat_inc(rq, ttwu_count);
+                        schedstat_inc(rq, ttwu_local);
+                }
+                ttwu_activate(p, rq, false, false, true, ENQUEUE_WAKEUP);
+                success = true;
+        }
+        ttwu_post_activation(p, rq, 0, success);
+}
+
+/**
  * wake_up_process - Wake up a specific process
  * @p: The process to be woken up.
  *
@@ -2537,42 +2495,9 @@ static void __sched_fork(struct task_struct *p)
         p->se.sum_exec_runtime          = 0;
         p->se.prev_sum_exec_runtime     = 0;
         p->se.nr_migrations             = 0;
-        p->se.last_wakeup               = 0;
-        p->se.avg_overlap               = 0;
-        p->se.start_runtime             = 0;
-        p->se.avg_wakeup                = sysctl_sched_wakeup_granularity;
 
 #ifdef CONFIG_SCHEDSTATS
-        p->se.wait_start                        = 0;
-        p->se.wait_max                          = 0;
-        p->se.wait_count                        = 0;
-        p->se.wait_sum                          = 0;
-
-        p->se.sleep_start                       = 0;
-        p->se.sleep_max                         = 0;
-        p->se.sum_sleep_runtime                 = 0;
-
-        p->se.block_start                       = 0;
-        p->se.block_max                         = 0;
-        p->se.exec_max                          = 0;
-        p->se.slice_max                         = 0;
-
-        p->se.nr_migrations_cold                = 0;
-        p->se.nr_failed_migrations_affine       = 0;
-        p->se.nr_failed_migrations_running      = 0;
-        p->se.nr_failed_migrations_hot          = 0;
-        p->se.nr_forced_migrations              = 0;
-
-        p->se.nr_wakeups                        = 0;
-        p->se.nr_wakeups_sync                   = 0;
-        p->se.nr_wakeups_migrate                = 0;
-        p->se.nr_wakeups_local                  = 0;
-        p->se.nr_wakeups_remote                 = 0;
-        p->se.nr_wakeups_affine                 = 0;
-        p->se.nr_wakeups_affine_attempts        = 0;
-        p->se.nr_wakeups_passive                = 0;
-        p->se.nr_wakeups_idle                   = 0;
-
+        memset(&p->se.statistics, 0, sizeof(p->se.statistics));
 #endif
 
         INIT_LIST_HEAD(&p->rt.run_list);
@@ -2593,11 +2518,11 @@ void sched_fork(struct task_struct *p, int clone_flags)
 
         __sched_fork(p);
         /*
-         * We mark the process as waking here. This guarantees that
+         * We mark the process as running here. This guarantees that
          * nobody will actually run it, and a signal or other external
          * event cannot wake it up and insert it on the runqueue either.
          */
-        p->state = TASK_WAKING;
+        p->state = TASK_RUNNING;
 
         /*
          * Revert to default priority/policy on fork if requested.
@@ -2632,7 +2557,16 @@ void sched_fork(struct task_struct *p, int clone_flags)
         if (p->sched_class->task_fork)
                 p->sched_class->task_fork(p);
 
+        /*
+         * The child is not yet in the pid-hash so no cgroup attach races,
+         * and the cgroup is pinned to this child due to cgroup_fork()
+         * is ran before sched_fork().
+         *
+         * Silence PROVE_RCU.
+         */
+        rcu_read_lock();
         set_task_cpu(p, cpu);
+        rcu_read_unlock();
 
 #if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
         if (likely(sched_info_on()))
@@ -2664,31 +2598,27 @@ void wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
         int cpu __maybe_unused = get_cpu();
 
 #ifdef CONFIG_SMP
+        rq = task_rq_lock(p, &flags);
+        p->state = TASK_WAKING;
+
         /*
          * Fork balancing, do it here and not earlier because:
          *  - cpus_allowed can change in the fork path
          *  - any previously selected cpu might disappear through hotplug
          *
-         * We still have TASK_WAKING but PF_STARTING is gone now, meaning
-         * ->cpus_allowed is stable, we have preemption disabled, meaning
-         * cpu_online_mask is stable.
+         * We set TASK_WAKING so that select_task_rq() can drop rq->lock
+         * without people poking at ->cpus_allowed.
          */
-        cpu = select_task_rq(p, SD_BALANCE_FORK, 0);
+        cpu = select_task_rq(rq, p, SD_BALANCE_FORK, 0);
         set_task_cpu(p, cpu);
-#endif
 
-        /*
-         * Since the task is not on the rq and we still have TASK_WAKING set
-         * nobody else will migrate this task.
-         */
-        rq = cpu_rq(cpu);
-        raw_spin_lock_irqsave(&rq->lock, flags);
-
-        BUG_ON(p->state != TASK_WAKING);
         p->state = TASK_RUNNING;
-        update_rq_clock(rq);
+        task_rq_unlock(rq, &flags);
+#endif
+
+        rq = task_rq_lock(p, &flags);
         activate_task(rq, p, 0);
-        trace_sched_wakeup_new(rq, p, 1);
+        trace_sched_wakeup_new(p, 1);
         check_preempt_curr(rq, p, WF_FORK);
 #ifdef CONFIG_SMP
         if (p->sched_class->task_woken)
@@ -2908,7 +2838,7 @@ context_switch(struct rq *rq, struct task_struct *prev,
         struct mm_struct *mm, *oldmm;
 
         prepare_task_switch(rq, prev, next);
-        trace_sched_switch(rq, prev, next);
+        trace_sched_switch(prev, next);
         mm = next->mm;
         oldmm = prev->active_mm;
         /*
@@ -3006,9 +2936,9 @@ unsigned long nr_iowait(void)
         return sum;
 }
 
-unsigned long nr_iowait_cpu(void)
+unsigned long nr_iowait_cpu(int cpu)
 {
-        struct rq *this = this_rq();
+        struct rq *this = cpu_rq(cpu);
         return atomic_read(&this->nr_iowait);
 }
 
@@ -3025,6 +2955,61 @@ static unsigned long calc_load_update;
 unsigned long avenrun[3];
 EXPORT_SYMBOL(avenrun);
 
+static long calc_load_fold_active(struct rq *this_rq)
+{
+        long nr_active, delta = 0;
+
+        nr_active = this_rq->nr_running;
+        nr_active += (long) this_rq->nr_uninterruptible;
+
+        if (nr_active != this_rq->calc_load_active) {
+                delta = nr_active - this_rq->calc_load_active;
+                this_rq->calc_load_active = nr_active;
+        }
+
+        return delta;
+}
+
+#ifdef CONFIG_NO_HZ
+/*
+ * For NO_HZ we delay the active fold to the next LOAD_FREQ update.
+ *
+ * When making the ILB scale, we should try to pull this in as well.
+ */
+static atomic_long_t calc_load_tasks_idle;
+
+static void calc_load_account_idle(struct rq *this_rq)
+{
+        long delta;
+
+        delta = calc_load_fold_active(this_rq);
+        if (delta)
+                atomic_long_add(delta, &calc_load_tasks_idle);
+}
+
+static long calc_load_fold_idle(void)
+{
+        long delta = 0;
+
+        /*
+         * Its got a race, we don't care...
+         */
+        if (atomic_long_read(&calc_load_tasks_idle))
+                delta = atomic_long_xchg(&calc_load_tasks_idle, 0);
+
+        return delta;
+}
+#else
+static void calc_load_account_idle(struct rq *this_rq)
+{
+}
+
+static inline long calc_load_fold_idle(void)
+{
+        return 0;
+}
+#endif
+
 /**
  * get_avenrun - get the load average array
  * @loads:      pointer to dest load array
@@ -3071,40 +3056,121 @@ void calc_global_load(void)
 }
 
 /*
- * Either called from update_cpu_load() or from a cpu going idle
+ * Called from update_cpu_load() to periodically update this CPU's
+ * active count.
  */
 static void calc_load_account_active(struct rq *this_rq)
 {
-        long nr_active, delta;
+        long delta;
 
-        nr_active = this_rq->nr_running;
-        nr_active += (long) this_rq->nr_uninterruptible;
+        if (time_before(jiffies, this_rq->calc_load_update))
+                return;
 
-        if (nr_active != this_rq->calc_load_active) {
-                delta = nr_active - this_rq->calc_load_active;
-                this_rq->calc_load_active = nr_active;
+        delta  = calc_load_fold_active(this_rq);
+        delta += calc_load_fold_idle();
+        if (delta)
                 atomic_long_add(delta, &calc_load_tasks);
+
+        this_rq->calc_load_update += LOAD_FREQ;
+}
+
+/*
+ * The exact cpuload at various idx values, calculated at every tick would be
+ * load = (2^idx - 1) / 2^idx * load + 1 / 2^idx * cur_load
+ *
+ * If a cpu misses updates for n-1 ticks (as it was idle) and update gets called
+ * on nth tick when cpu may be busy, then we have:
+ * load = ((2^idx - 1) / 2^idx)^(n-1) * load
+ * load = (2^idx - 1) / 2^idx) * load + 1 / 2^idx * cur_load
+ *
+ * decay_load_missed() below does efficient calculation of
+ * load = ((2^idx - 1) / 2^idx)^(n-1) * load
+ * avoiding 0..n-1 loop doing load = ((2^idx - 1) / 2^idx) * load
+ *
+ * The calculation is approximated on a 128 point scale.
+ * degrade_zero_ticks is the number of ticks after which load at any
+ * particular idx is approximated to be zero.
+ * degrade_factor is a precomputed table, a row for each load idx.
+ * Each column corresponds to degradation factor for a power of two ticks,
+ * based on 128 point scale.
+ * Example:
+ * row 2, col 3 (=12) says that the degradation at load idx 2 after
+ * 8 ticks is 12/128 (which is an approximation of exact factor 3^8/4^8).
+ *
+ * With this power of 2 load factors, we can degrade the load n times
+ * by looking at 1 bits in n and doing as many mult/shift instead of
+ * n mult/shifts needed by the exact degradation.
+ */
+#define DEGRADE_SHIFT           7
+static const unsigned char
+                degrade_zero_ticks[CPU_LOAD_IDX_MAX] = {0, 8, 32, 64, 128};
+static const unsigned char
+                degrade_factor[CPU_LOAD_IDX_MAX][DEGRADE_SHIFT + 1] = {
+                                        {0, 0, 0, 0, 0, 0, 0, 0},
+                                        {64, 32, 8, 0, 0, 0, 0, 0},
+                                        {96, 72, 40, 12, 1, 0, 0},
+                                        {112, 98, 75, 43, 15, 1, 0},
+                                        {120, 112, 98, 76, 45, 16, 2} };
+
+/*
+ * Update cpu_load for any missed ticks, due to tickless idle. The backlog
+ * would be when CPU is idle and so we just decay the old load without
+ * adding any new load.
+ */
+static unsigned long
+decay_load_missed(unsigned long load, unsigned long missed_updates, int idx)
+{
+        int j = 0;
+
+        if (!missed_updates)
+                return load;
+
+        if (missed_updates >= degrade_zero_ticks[idx])
+                return 0;
+
+        if (idx == 1)
+                return load >> missed_updates;
+
+        while (missed_updates) {
+                if (missed_updates % 2)
+                        load = (load * degrade_factor[idx][j]) >> DEGRADE_SHIFT;
+
+                missed_updates >>= 1;
+                j++;
         }
+        return load;
 }
 
 /*
  * Update rq->cpu_load[] statistics. This function is usually called every
- * scheduler tick (TICK_NSEC).
+ * scheduler tick (TICK_NSEC). With tickless idle this will not be called
+ * every tick. We fix it up based on jiffies.
  */
 static void update_cpu_load(struct rq *this_rq)
 {
         unsigned long this_load = this_rq->load.weight;
+        unsigned long curr_jiffies = jiffies;
+        unsigned long pending_updates;
         int i, scale;
 
         this_rq->nr_load_updates++;
 
+        /* Avoid repeated calls on same jiffy, when moving in and out of idle */
+        if (curr_jiffies == this_rq->last_load_update_tick)
+                return;
+
+        pending_updates = curr_jiffies - this_rq->last_load_update_tick;
+        this_rq->last_load_update_tick = curr_jiffies;
+
         /* Update our load: */
-        for (i = 0, scale = 1; i < CPU_LOAD_IDX_MAX; i++, scale += scale) {
+        this_rq->cpu_load[0] = this_load; /* Fasttrack for idx 0 */
+        for (i = 1, scale = 2; i < CPU_LOAD_IDX_MAX; i++, scale += scale) {
                 unsigned long old_load, new_load;
 
                 /* scale is effectively 1 << i now, and >> i divides by scale */
 
                 old_load = this_rq->cpu_load[i];
+                old_load = decay_load_missed(old_load, pending_updates - 1, i);
                 new_load = this_load;
                 /*
                  * Round up the averaging division if load is increasing. This
@@ -3112,16 +3178,19 @@ static void update_cpu_load(struct rq *this_rq)
                  * example.
                  */
                 if (new_load > old_load)
-                        new_load += scale-1;
-                this_rq->cpu_load[i] = (old_load*(scale-1) + new_load) >> i;
-        }
+                        new_load += scale - 1;
 
-        if (time_after_eq(jiffies, this_rq->calc_load_update)) {
-                this_rq->calc_load_update += LOAD_FREQ;
-                calc_load_account_active(this_rq);
+                this_rq->cpu_load[i] = (old_load * (scale - 1) + new_load) >> i;
         }
 }
 
+static void update_cpu_load_active(struct rq *this_rq)
+{
+        update_cpu_load(this_rq);
+
+        calc_load_account_active(this_rq);
+}
+
 #ifdef CONFIG_SMP
 
 /*
@@ -3131,44 +3200,27 @@ static void update_cpu_load(struct rq *this_rq)
 void sched_exec(void)
 {
         struct task_struct *p = current;
-        struct migration_req req;
-        int dest_cpu, this_cpu;
         unsigned long flags;
         struct rq *rq;
-
-again:
-        this_cpu = get_cpu();
-        dest_cpu = select_task_rq(p, SD_BALANCE_EXEC, 0);
-        if (dest_cpu == this_cpu) {
-                put_cpu();
-                return;
-        }
+        int dest_cpu;
 
         rq = task_rq_lock(p, &flags);
-        put_cpu();
+        dest_cpu = p->sched_class->select_task_rq(rq, p, SD_BALANCE_EXEC, 0);
+        if (dest_cpu == smp_processor_id())
+                goto unlock;
 
         /*
          * select_task_rq() can race against ->cpus_allowed
          */
-        if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed)
-            || unlikely(!cpu_active(dest_cpu))) {
-                task_rq_unlock(rq, &flags);
-                goto again;
-        }
-
-        /* force the process onto the specified CPU */
-        if (migrate_task(p, dest_cpu, &req)) {
-                /* Need to wait for migration thread (might exit: take ref). */
-                struct task_struct *mt = rq->migration_thread;
+        if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed) &&
+            likely(cpu_active(dest_cpu)) && migrate_task(p, dest_cpu)) {
+                struct migration_arg arg = { p, dest_cpu };
 
-                get_task_struct(mt);
                 task_rq_unlock(rq, &flags);
-                wake_up_process(mt);
-                put_task_struct(mt);
-                wait_for_completion(&req.done);
-
+                stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
                 return;
         }
+unlock:
         task_rq_unlock(rq, &flags);
 }
 
@@ -3522,7 +3574,7 @@ void scheduler_tick(void)
 
         raw_spin_lock(&rq->lock);
         update_rq_clock(rq);
-        update_cpu_load(rq);
+        update_cpu_load_active(rq);
         curr->sched_class->task_tick(rq, curr, 0);
         raw_spin_unlock(&rq->lock);
 
@@ -3640,23 +3692,9 @@ static inline void schedule_debug(struct task_struct *prev)
 
 static void put_prev_task(struct rq *rq, struct task_struct *prev)
 {
-        if (prev->state == TASK_RUNNING) {
-                u64 runtime = prev->se.sum_exec_runtime;
-
-                runtime -= prev->se.prev_sum_exec_runtime;
-                runtime = min_t(u64, runtime, 2*sysctl_sched_migration_cost);
-
-                /*
-                 * In order to avoid avg_overlap growing stale when we are
-                 * indeed overlapping and hence not getting put to sleep, grow
-                 * the avg_overlap on preemption.
-                 *
-                 * We use the average preemption runtime because that
-                 * correlates to the amount of cache footprint a task can
-                 * build up.
-                 */
-                update_avg(&prev->se.avg_overlap, runtime);
-        }
+        if (prev->se.on_rq)
+                update_rq_clock(rq);
+        rq->skip_clock_update = 0;
         prev->sched_class->put_prev_task(rq, prev);
 }
 
@@ -3706,9 +3744,8 @@ need_resched:
         preempt_disable();
         cpu = smp_processor_id();
         rq = cpu_rq(cpu);
-        rcu_sched_qs(cpu);
+        rcu_note_context_switch(cpu);
         prev = rq->curr;
-        switch_count = &prev->nivcsw;
 
         release_kernel_lock(prev);
 need_resched_nonpreemptible:
@@ -3719,14 +3756,28 @@ need_resched_nonpreemptible:
                 hrtick_clear(rq);
 
         raw_spin_lock_irq(&rq->lock);
-        update_rq_clock(rq);
         clear_tsk_need_resched(prev);
 
+        switch_count = &prev->nivcsw;
         if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {
-                if (unlikely(signal_pending_state(prev->state, prev)))
+                if (unlikely(signal_pending_state(prev->state, prev))) {
                         prev->state = TASK_RUNNING;
-                else
-                        deactivate_task(rq, prev, 1);
+                } else {
+                        /*
+                         * If a worker is going to sleep, notify and
+                         * ask workqueue whether it wants to wake up a
+                         * task to maintain concurrency.  If so, wake
+                         * up the task.
+                         */
+                        if (prev->flags & PF_WQ_WORKER) {
+                                struct task_struct *to_wakeup;
+
+                                to_wakeup = wq_worker_sleeping(prev, cpu);
+                                if (to_wakeup)
+                                        try_to_wake_up_local(to_wakeup);
+                        }
+                        deactivate_task(rq, prev, DEQUEUE_SLEEP);
+                }
                 switch_count = &prev->nvcsw;
         }
 
@@ -3748,8 +3799,10 @@ need_resched_nonpreemptible:
 
                 context_switch(rq, prev, next); /* unlocks the rq */
                 /*
-                 * the context switch might have flipped the stack from under
-                 * us, hence refresh the local variables.
+                 * The context switch have flipped the stack from under us
+                 * and restored the local variables which were saved when
+                 * this task called schedule() in the past. prev == current
+                 * is still correct, but it can be moved to another cpu/rq.
                  */
                 cpu = smp_processor_id();
                 rq = cpu_rq(cpu);
@@ -3758,11 +3811,8 @@ need_resched_nonpreemptible:
 
         post_schedule(rq);
 
-        if (unlikely(reacquire_kernel_lock(current) < 0)) {
-                prev = rq->curr;
-                switch_count = &prev->nivcsw;
+        if (unlikely(reacquire_kernel_lock(prev)))
                 goto need_resched_nonpreemptible;
-        }
 
         preempt_enable_no_resched();
         if (need_resched())
@@ -3837,7 +3887,7 @@ int mutex_spin_on_owner(struct mutex *lock, struct thread_info *owner)
  * off of preempt_enable. Kernel preemptions off return from interrupt
  * occur there and call schedule directly.
  */
-asmlinkage void __sched preempt_schedule(void)
+asmlinkage void __sched notrace preempt_schedule(void)
 {
         struct thread_info *ti = current_thread_info();
 
@@ -3849,9 +3899,9 @@ asmlinkage void __sched preempt_schedule(void)
                 return;
 
         do {
-                add_preempt_count(PREEMPT_ACTIVE);
+                add_preempt_count_notrace(PREEMPT_ACTIVE);
                 schedule();
-                sub_preempt_count(PREEMPT_ACTIVE);
+                sub_preempt_count_notrace(PREEMPT_ACTIVE);
 
                 /*
                  * Check again in case we missed a preemption opportunity
@@ -3950,6 +4000,7 @@ void __wake_up_locked(wait_queue_head_t *q, unsigned int mode)
 {
         __wake_up_common(q, mode, 1, 0, NULL);
 }
+EXPORT_SYMBOL_GPL(__wake_up_locked);
 
 void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key)
 {
@@ -4049,8 +4100,7 @@ do_wait_for_common(struct completion *x, long timeout, int state)
         if (!x->done) {
                 DECLARE_WAITQUEUE(wait, current);
 
-                wait.flags |= WQ_FLAG_EXCLUSIVE;
-                __add_wait_queue_tail(&x->wait, &wait);
+                __add_wait_queue_tail_exclusive(&x->wait, &wait);
                 do {
                         if (signal_pending_state(state, current)) {
                                 timeout = -ERESTARTSYS;
@@ -4161,6 +4211,23 @@ int __sched wait_for_completion_killable(struct completion *x)
 EXPORT_SYMBOL(wait_for_completion_killable);
 
 /**
+ * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable))
+ * @x:  holds the state of this particular completion
+ * @timeout:  timeout value in jiffies
+ *
+ * This waits for either a completion of a specific task to be
+ * signaled or for a specified timeout to expire. It can be
+ * interrupted by a kill signal. The timeout is in jiffies.
+ */
+unsigned long __sched
+wait_for_completion_killable_timeout(struct completion *x,
+                                     unsigned long timeout)
+{
+        return wait_for_common(x, timeout, TASK_KILLABLE);
+}
+EXPORT_SYMBOL(wait_for_completion_killable_timeout);
+
+/**
  *      try_wait_for_completion - try to decrement a completion without blocking
  *      @x:     completion structure
  *
@@ -4276,7 +4343,6 @@ void rt_mutex_setprio(struct task_struct *p, int prio)
         BUG_ON(prio < 0 || prio > MAX_PRIO);
 
         rq = task_rq_lock(p, &flags);
-        update_rq_clock(rq);
 
         oldprio = p->prio;
         prev_class = p->sched_class;
@@ -4297,7 +4363,7 @@ void rt_mutex_setprio(struct task_struct *p, int prio)
         if (running)
                 p->sched_class->set_curr_task(rq);
         if (on_rq) {
-                enqueue_task(rq, p, 0, oldprio < prio);
+                enqueue_task(rq, p, oldprio < prio ? ENQUEUE_HEAD : 0);
 
                 check_class_changed(rq, p, prev_class, oldprio, running);
         }
@@ -4319,7 +4385,6 @@ void set_user_nice(struct task_struct *p, long nice)
          * the task might be in the middle of scheduling on another CPU.
          */
         rq = task_rq_lock(p, &flags);
-        update_rq_clock(rq);
         /*
          * The RT priorities are set via sched_setscheduler(), but we still
          * allow the 'normal' nice value to be set - but as expected
@@ -4341,7 +4406,7 @@ void set_user_nice(struct task_struct *p, long nice)
         delta = p->prio - old_prio;
 
         if (on_rq) {
-                enqueue_task(rq, p, 0, false);
+                enqueue_task(rq, p, 0);
                 /*
                  * If the task increased its priority or is running and
                  * lowered its priority, then reschedule its CPU:
@@ -4537,12 +4602,8 @@ recheck:
          */
         if (user && !capable(CAP_SYS_NICE)) {
                 if (rt_policy(policy)) {
-                        unsigned long rlim_rtprio;
-
-                        if (!lock_task_sighand(p, &flags))
-                                return -ESRCH;
-                        rlim_rtprio = task_rlimit(p, RLIMIT_RTPRIO);
-                        unlock_task_sighand(p, &flags);
+                        unsigned long rlim_rtprio =
+                                        task_rlimit(p, RLIMIT_RTPRIO);
 
                         /* can't set/change the rt policy */
                         if (policy != p->policy && !rlim_rtprio)
@@ -4570,16 +4631,6 @@ recheck:
         }
 
         if (user) {
-#ifdef CONFIG_RT_GROUP_SCHED
-                /*
-                 * Do not allow realtime tasks into groups that have no runtime
-                 * assigned.
-                 */
-                if (rt_bandwidth_enabled() && rt_policy(policy) &&
-                                task_group(p)->rt_bandwidth.rt_runtime == 0)
-                        return -EPERM;
-#endif
-
                 retval = security_task_setscheduler(p, policy, param);
                 if (retval)
                         return retval;
@@ -4595,6 +4646,22 @@ recheck:
          * runqueue lock must be held.
          */
         rq = __task_rq_lock(p);
+
+#ifdef CONFIG_RT_GROUP_SCHED
+        if (user) {
+                /*
+                 * Do not allow realtime tasks into groups that have no runtime
+                 * assigned.
+                 */
+                if (rt_bandwidth_enabled() && rt_policy(policy) &&
+                                task_group(p)->rt_bandwidth.rt_runtime == 0) {
+                        __task_rq_unlock(rq);
+                        raw_spin_unlock_irqrestore(&p->pi_lock, flags);
+                        return -EPERM;
+                }
+        }
+#endif
+
         /* recheck policy now with rq lock held */
         if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
                 policy = oldpolicy = -1;
@@ -4602,7 +4669,6 @@ recheck:
                 raw_spin_unlock_irqrestore(&p->pi_lock, flags);
                 goto recheck;
         }
-        update_rq_clock(rq);
         on_rq = p->se.on_rq;
         running = task_current(rq, p);
         if (on_rq)
@@ -5339,17 +5405,15 @@ static inline void sched_init_granularity(void)
 /*
  * This is how migration works:
  *
- * 1) we queue a struct migration_req structure in the source CPU's
- *    runqueue and wake up that CPU's migration thread.
- * 2) we down() the locked semaphore => thread blocks.
- * 3) migration thread wakes up (implicitly it forces the migrated
- *    thread off the CPU)
- * 4) it gets the migration request and checks whether the migrated
- *    task is still in the wrong runqueue.
- * 5) if it's in the wrong runqueue then the migration thread removes
+ * 1) we invoke migration_cpu_stop() on the target CPU using
+ *    stop_one_cpu().
+ * 2) stopper starts to run (implicitly forcing the migrated thread
+ *    off the CPU)
+ * 3) it checks whether the migrated task is still in the wrong runqueue.
+ * 4) if it's in the wrong runqueue then the migration thread removes
  *    it and puts it into the right queue.
- * 6) migration thread up()s the semaphore.
- * 7) we wake up and the migration is done.
+ * 5) stopper completes and stop_one_cpu() returns and the migration
+ *    is done.
  */
 
 /*
@@ -5363,12 +5427,23 @@ static inline void sched_init_granularity(void)
  */
 int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
 {
-        struct migration_req req;
         unsigned long flags;
         struct rq *rq;
+        unsigned int dest_cpu;
         int ret = 0;
 
+        /*
+         * Serialize against TASK_WAKING so that ttwu() and wunt() can
+         * drop the rq->lock and still rely on ->cpus_allowed.
+         */
+again:
+        while (task_is_waking(p))
+                cpu_relax();
         rq = task_rq_lock(p, &flags);
+        if (task_is_waking(p)) {
+                task_rq_unlock(rq, &flags);
+                goto again;
+        }
 
         if (!cpumask_intersects(new_mask, cpu_active_mask)) {
                 ret = -EINVAL;
@@ -5392,15 +5467,12 @@ int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
         if (cpumask_test_cpu(task_cpu(p), new_mask))
                 goto out;
 
-        if (migrate_task(p, cpumask_any_and(cpu_active_mask, new_mask), &req)) {
+        dest_cpu = cpumask_any_and(cpu_active_mask, new_mask);
+        if (migrate_task(p, dest_cpu)) {
+                struct migration_arg arg = { p, dest_cpu };
                 /* Need help from migration thread: drop lock and wait. */
-                struct task_struct *mt = rq->migration_thread;
-
-                get_task_struct(mt);
                 task_rq_unlock(rq, &flags);
-                wake_up_process(mt);
-                put_task_struct(mt);
-                wait_for_completion(&req.done);
+                stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
                 tlb_migrate_finish(p->mm);
                 return 0;
         }
@@ -5458,98 +5530,49 @@ fail:
         return ret;
 }
 
-#define RCU_MIGRATION_IDLE      0
-#define RCU_MIGRATION_NEED_QS   1
-#define RCU_MIGRATION_GOT_QS    2
-#define RCU_MIGRATION_MUST_SYNC 3
-
 /*
- * migration_thread - this is a highprio system thread that performs
- * thread migration by bumping thread off CPU then 'pushing' onto
- * another runqueue.
+ * migration_cpu_stop - this will be executed by a highprio stopper thread
+ * and performs thread migration by bumping thread off CPU then
+ * 'pushing' onto another runqueue.
  */
-static int migration_thread(void *data)
+static int migration_cpu_stop(void *data)
 {
-        int badcpu;
-        int cpu = (long)data;
-        struct rq *rq;
-
-        rq = cpu_rq(cpu);
-        BUG_ON(rq->migration_thread != current);
-
-        set_current_state(TASK_INTERRUPTIBLE);
-        while (!kthread_should_stop()) {
-                struct migration_req *req;
-                struct list_head *head;
-
-                raw_spin_lock_irq(&rq->lock);
-
-                if (cpu_is_offline(cpu)) {
-                        raw_spin_unlock_irq(&rq->lock);
-                        break;
-                }
-
-                if (rq->active_balance) {
-                        active_load_balance(rq, cpu);
-                        rq->active_balance = 0;
-                }
-
-                head = &rq->migration_queue;
-
-                if (list_empty(head)) {
-                        raw_spin_unlock_irq(&rq->lock);
-                        schedule();
-                        set_current_state(TASK_INTERRUPTIBLE);
-                        continue;
-                }
-                req = list_entry(head->next, struct migration_req, list);
-                list_del_init(head->next);
-
-                if (req->task != NULL) {
-                        raw_spin_unlock(&rq->lock);
-                        __migrate_task(req->task, cpu, req->dest_cpu);
-                } else if (likely(cpu == (badcpu = smp_processor_id()))) {
-                        req->dest_cpu = RCU_MIGRATION_GOT_QS;
-                        raw_spin_unlock(&rq->lock);
-                } else {
-                        req->dest_cpu = RCU_MIGRATION_MUST_SYNC;
-                        raw_spin_unlock(&rq->lock);
-                        WARN_ONCE(1, "migration_thread() on CPU %d, expected %d\n", badcpu, cpu);
-                }
-                local_irq_enable();
-
-                complete(&req->done);
-        }
-        __set_current_state(TASK_RUNNING);
-
-        return 0;
-}
-
-#ifdef CONFIG_HOTPLUG_CPU
-
-static int __migrate_task_irq(struct task_struct *p, int src_cpu, int dest_cpu)
-{
-        int ret;
+        struct migration_arg *arg = data;
 
+        /*
+         * The original target cpu might have gone down and we might
+         * be on another cpu but it doesn't matter.
+         */
         local_irq_disable();
-        ret = __migrate_task(p, src_cpu, dest_cpu);
+        __migrate_task(arg->task, raw_smp_processor_id(), arg->dest_cpu);
         local_irq_enable();
-        return ret;
+        return 0;
 }
 
+#ifdef CONFIG_HOTPLUG_CPU
 /*
  * Figure out where task on dead CPU should go, use force if necessary.
  */
-static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
+void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
 {
-        int dest_cpu;
+        struct rq *rq = cpu_rq(dead_cpu);
+        int needs_cpu, uninitialized_var(dest_cpu);
+        unsigned long flags;
 
-again:
-        dest_cpu = select_fallback_rq(dead_cpu, p);
+        local_irq_save(flags);
 
-        /* It can have affinity changed while we were choosing. */
-        if (unlikely(!__migrate_task_irq(p, dead_cpu, dest_cpu)))
-                goto again;
+        raw_spin_lock(&rq->lock);
+        needs_cpu = (task_cpu(p) == dead_cpu) && (p->state != TASK_WAKING);
+        if (needs_cpu)
+                dest_cpu = select_fallback_rq(dead_cpu, p);
+        raw_spin_unlock(&rq->lock);
+        /*
+         * It can only fail if we race with set_cpus_allowed(),
+         * in the racer should migrate the task anyway.
+         */
+        if (needs_cpu)
+                __migrate_task(p, dead_cpu, dest_cpu);
+        local_irq_restore(flags);
 }
 
 /*
@@ -5613,7 +5636,6 @@ void sched_idle_next(void)
 
         __setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1);
 
-        update_rq_clock(rq);
         activate_task(rq, p, 0);
 
         raw_spin_unlock_irqrestore(&rq->lock, flags);
@@ -5668,7 +5690,6 @@ static void migrate_dead_tasks(unsigned int dead_cpu)
         for ( ; ; ) {
                 if (!rq->nr_running)
                         break;
-                update_rq_clock(rq);
                 next = pick_next_task(rq);
                 if (!next)
                         break;
@@ -5891,35 +5912,20 @@ static void set_rq_offline(struct rq *rq)
 static int __cpuinit
 migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
 {
-        struct task_struct *p;
         int cpu = (long)hcpu;
         unsigned long flags;
-        struct rq *rq;
+        struct rq *rq = cpu_rq(cpu);
 
         switch (action) {
 
         case CPU_UP_PREPARE:
         case CPU_UP_PREPARE_FROZEN:
-                p = kthread_create(migration_thread, hcpu, "migration/%d", cpu);
-                if (IS_ERR(p))
-                        return NOTIFY_BAD;
-                kthread_bind(p, cpu);
-                /* Must be high prio: stop_machine expects to yield to it. */
-                rq = task_rq_lock(p, &flags);
-                __setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1);
-                task_rq_unlock(rq, &flags);
-                get_task_struct(p);
-                cpu_rq(cpu)->migration_thread = p;
                 rq->calc_load_update = calc_load_update;
                 break;
 
         case CPU_ONLINE:
         case CPU_ONLINE_FROZEN:
-                /* Strictly unnecessary, as first user will wake it. */
-                wake_up_process(cpu_rq(cpu)->migration_thread);
-
                 /* Update our root-domain */
-                rq = cpu_rq(cpu);
                 raw_spin_lock_irqsave(&rq->lock, flags);
                 if (rq->rd) {
                         BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
@@ -5930,61 +5936,24 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
                 break;
 
 #ifdef CONFIG_HOTPLUG_CPU
-        case CPU_UP_CANCELED:
-        case CPU_UP_CANCELED_FROZEN:
-                if (!cpu_rq(cpu)->migration_thread)
-                        break;
-                /* Unbind it from offline cpu so it can run. Fall thru. */
-                kthread_bind(cpu_rq(cpu)->migration_thread,
-                             cpumask_any(cpu_online_mask));
-                kthread_stop(cpu_rq(cpu)->migration_thread);
-                put_task_struct(cpu_rq(cpu)->migration_thread);
-                cpu_rq(cpu)->migration_thread = NULL;
-                break;
-
         case CPU_DEAD:
         case CPU_DEAD_FROZEN:
-                cpuset_lock(); /* around calls to cpuset_cpus_allowed_lock() */
                 migrate_live_tasks(cpu);
-                rq = cpu_rq(cpu);
-                kthread_stop(rq->migration_thread);
-                put_task_struct(rq->migration_thread);
-                rq->migration_thread = NULL;
                 /* Idle task back to normal (off runqueue, low prio) */
                 raw_spin_lock_irq(&rq->lock);
-                update_rq_clock(rq);
                 deactivate_task(rq, rq->idle, 0);
                 __setscheduler(rq, rq->idle, SCHED_NORMAL, 0);
                 rq->idle->sched_class = &idle_sched_class;
                 migrate_dead_tasks(cpu);
                 raw_spin_unlock_irq(&rq->lock);
-                cpuset_unlock();
                 migrate_nr_uninterruptible(rq);
                 BUG_ON(rq->nr_running != 0);
                 calc_global_load_remove(rq);
-                /*
-                 * No need to migrate the tasks: it was best-effort if
-                 * they didn't take sched_hotcpu_mutex. Just wake up
-                 * the requestors.
-                 */
-                raw_spin_lock_irq(&rq->lock);
-                while (!list_empty(&rq->migration_queue)) {
-                        struct migration_req *req;
-
-                        req = list_entry(rq->migration_queue.next,
-                                         struct migration_req, list);
-                        list_del_init(&req->list);
-                        raw_spin_unlock_irq(&rq->lock);
-                        complete(&req->done);
-                        raw_spin_lock_irq(&rq->lock);
-                }
-                raw_spin_unlock_irq(&rq->lock);
                 break;
 
         case CPU_DYING:
         case CPU_DYING_FROZEN:
                 /* Update our root-domain */
-                rq = cpu_rq(cpu);
                 raw_spin_lock_irqsave(&rq->lock, flags);
                 if (rq->rd) {
                         BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
@@ -6004,20 +5973,49 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
  */
 static struct notifier_block __cpuinitdata migration_notifier = {
         .notifier_call = migration_call,
-        .priority = 10
+        .priority = CPU_PRI_MIGRATION,
 };
 
+static int __cpuinit sched_cpu_active(struct notifier_block *nfb,
+                                      unsigned long action, void *hcpu)
+{
+        switch (action & ~CPU_TASKS_FROZEN) {
+        case CPU_ONLINE:
+        case CPU_DOWN_FAILED:
+                set_cpu_active((long)hcpu, true);
+                return NOTIFY_OK;
+        default:
+                return NOTIFY_DONE;
+        }
+}
+
+static int __cpuinit sched_cpu_inactive(struct notifier_block *nfb,
+                                        unsigned long action, void *hcpu)
+{
+        switch (action & ~CPU_TASKS_FROZEN) {
+        case CPU_DOWN_PREPARE:
+                set_cpu_active((long)hcpu, false);
+                return NOTIFY_OK;
+        default:
+                return NOTIFY_DONE;
+        }
+}
+
 static int __init migration_init(void)
 {
         void *cpu = (void *)(long)smp_processor_id();
         int err;
 
-        /* Start one for the boot CPU: */
+        /* Initialize migration for the boot CPU */
         err = migration_call(&migration_notifier, CPU_UP_PREPARE, cpu);
         BUG_ON(err == NOTIFY_BAD);
         migration_call(&migration_notifier, CPU_ONLINE, cpu);
         register_cpu_notifier(&migration_notifier);
 
+        /* Register cpu active notifiers */
+        cpu_notifier(sched_cpu_active, CPU_PRI_SCHED_ACTIVE);
+        cpu_notifier(sched_cpu_inactive, CPU_PRI_SCHED_INACTIVE);
+
         return 0;
 }
 early_initcall(migration_init);
@@ -6252,23 +6250,18 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd)
                 free_rootdomain(old_rd);
 }
 
-static int init_rootdomain(struct root_domain *rd, bool bootmem)
+static int init_rootdomain(struct root_domain *rd)
 {
-        gfp_t gfp = GFP_KERNEL;
-
         memset(rd, 0, sizeof(*rd));
 
-        if (bootmem)
-                gfp = GFP_NOWAIT;
-
-        if (!alloc_cpumask_var(&rd->span, gfp))
+        if (!alloc_cpumask_var(&rd->span, GFP_KERNEL))
                 goto out;
-        if (!alloc_cpumask_var(&rd->online, gfp))
+        if (!alloc_cpumask_var(&rd->online, GFP_KERNEL))
                 goto free_span;
-        if (!alloc_cpumask_var(&rd->rto_mask, gfp))
+        if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
                 goto free_online;
 
-        if (cpupri_init(&rd->cpupri, bootmem) != 0)
+        if (cpupri_init(&rd->cpupri) != 0)
                 goto free_rto_mask;
         return 0;
 
@@ -6284,7 +6277,7 @@ out:
 
 static void init_defrootdomain(void)
 {
-        init_rootdomain(&def_root_domain, true);
+        init_rootdomain(&def_root_domain);
 
         atomic_set(&def_root_domain.refcount, 1);
 }
@@ -6297,7 +6290,7 @@ static struct root_domain *alloc_rootdomain(void)
         if (!rd)
                 return NULL;
 
-        if (init_rootdomain(rd, false) != 0) {
+        if (init_rootdomain(rd) != 0) {
                 kfree(rd);
                 return NULL;
         }
@@ -6315,6 +6308,9 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
         struct rq *rq = cpu_rq(cpu);
         struct sched_domain *tmp;
 
+        for (tmp = sd; tmp; tmp = tmp->parent)
+                tmp->span_weight = cpumask_weight(sched_domain_span(tmp));
+
         /* Remove the sched domains which do not contribute to scheduling. */
         for (tmp = sd; tmp; ) {
                 struct sched_domain *parent = tmp->parent;
@@ -7473,29 +7469,35 @@ int __init sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
 }
 #endif /* CONFIG_SCHED_MC || CONFIG_SCHED_SMT */
 
-#ifndef CONFIG_CPUSETS
 /*
- * Add online and remove offline CPUs from the scheduler domains.
- * When cpusets are enabled they take over this function.
+ * Update cpusets according to cpu_active mask.  If cpusets are
+ * disabled, cpuset_update_active_cpus() becomes a simple wrapper
+ * around partition_sched_domains().
  */
-static int update_sched_domains(struct notifier_block *nfb,
-                                unsigned long action, void *hcpu)
+static int cpuset_cpu_active(struct notifier_block *nfb, unsigned long action,
+                             void *hcpu)
 {
-        switch (action) {
+        switch (action & ~CPU_TASKS_FROZEN) {
         case CPU_ONLINE:
-        case CPU_ONLINE_FROZEN:
-        case CPU_DOWN_PREPARE:
-        case CPU_DOWN_PREPARE_FROZEN:
         case CPU_DOWN_FAILED:
-        case CPU_DOWN_FAILED_FROZEN:
-                partition_sched_domains(1, NULL, NULL);
+                cpuset_update_active_cpus();
                 return NOTIFY_OK;
+        default:
+                return NOTIFY_DONE;
+        }
+}
 
+static int cpuset_cpu_inactive(struct notifier_block *nfb, unsigned long action,
+                               void *hcpu)
+{
+        switch (action & ~CPU_TASKS_FROZEN) {
+        case CPU_DOWN_PREPARE:
+                cpuset_update_active_cpus();
+                return NOTIFY_OK;
         default:
                 return NOTIFY_DONE;
         }
 }
-#endif
 
 static int update_runtime(struct notifier_block *nfb,
                                 unsigned long action, void *hcpu)
@@ -7541,10 +7543,8 @@ void __init sched_init_smp(void)
         mutex_unlock(&sched_domains_mutex);
         put_online_cpus();
 
-#ifndef CONFIG_CPUSETS
-        /* XXX: Theoretical race here - CPU may be hotplugged now */
-        hotcpu_notifier(update_sched_domains, 0);
-#endif
+        hotcpu_notifier(cpuset_cpu_active, CPU_PRI_CPUSET_ACTIVE);
+        hotcpu_notifier(cpuset_cpu_inactive, CPU_PRI_CPUSET_INACTIVE);
 
         /* RT runtime code needs to handle some hotplug events */
         hotcpu_notifier(update_runtime, 0);
@@ -7789,20 +7789,26 @@ void __init sched_init(void)
 
                 for (j = 0; j < CPU_LOAD_IDX_MAX; j++)
                         rq->cpu_load[j] = 0;
+
+                rq->last_load_update_tick = jiffies;
+
 #ifdef CONFIG_SMP
                 rq->sd = NULL;
                 rq->rd = NULL;
+                rq->cpu_power = SCHED_LOAD_SCALE;
                 rq->post_schedule = 0;
                 rq->active_balance = 0;
                 rq->next_balance = jiffies;
                 rq->push_cpu = 0;
                 rq->cpu = i;
                 rq->online = 0;
-                rq->migration_thread = NULL;
                 rq->idle_stamp = 0;
                 rq->avg_idle = 2*sysctl_sched_migration_cost;
-                INIT_LIST_HEAD(&rq->migration_queue);
                 rq_attach_root(rq, &def_root_domain);
+#ifdef CONFIG_NO_HZ
+                rq->nohz_balance_kick = 0;
+                init_sched_softirq_csd(&per_cpu(remote_sched_softirq_cb, i));
+#endif
 #endif
                 init_rq_hrtick(rq);
                 atomic_set(&rq->nr_iowait, 0);
@@ -7847,8 +7853,11 @@ void __init sched_init(void)
         zalloc_cpumask_var(&nohz_cpu_mask, GFP_NOWAIT);
 #ifdef CONFIG_SMP
 #ifdef CONFIG_NO_HZ
-        zalloc_cpumask_var(&nohz.cpu_mask, GFP_NOWAIT);
-        alloc_cpumask_var(&nohz.ilb_grp_nohz_mask, GFP_NOWAIT);
+        zalloc_cpumask_var(&nohz.idle_cpus_mask, GFP_NOWAIT);
+        alloc_cpumask_var(&nohz.grp_idle_mask, GFP_NOWAIT);
+        atomic_set(&nohz.load_balancer, nr_cpu_ids);
+        atomic_set(&nohz.first_pick_cpu, nr_cpu_ids);
+        atomic_set(&nohz.second_pick_cpu, nr_cpu_ids);
 #endif
         /* May be allocated at isolcpus cmdline parse time */
         if (cpu_isolated_map == NULL)
@@ -7902,7 +7911,6 @@ static void normalize_task(struct rq *rq, struct task_struct *p)
 {
         int on_rq;
 
-        update_rq_clock(rq);
         on_rq = p->se.on_rq;
         if (on_rq)
                 deactivate_task(rq, p, 0);
@@ -7929,9 +7937,9 @@ void normalize_rt_tasks(void)
 
                 p->se.exec_start                = 0;
 #ifdef CONFIG_SCHEDSTATS
-                p->se.wait_start                = 0;
-                p->se.sleep_start               = 0;
-                p->se.block_start               = 0;
+                p->se.statistics.wait_start     = 0;
+                p->se.statistics.sleep_start    = 0;
+                p->se.statistics.block_start    = 0;
 #endif
 
                 if (!rt_task(p)) {
@@ -7958,9 +7966,9 @@ void normalize_rt_tasks(void)
 
 #endif /* CONFIG_MAGIC_SYSRQ */
 
-#ifdef CONFIG_IA64
+#if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB)
 /*
- * These functions are only useful for the IA64 MCA handling.
+ * These functions are only useful for the IA64 MCA handling, or kdb.
  *
  * They can only be called when the whole system has been
  * stopped - every CPU needs to be quiescent, and no scheduling
@@ -7980,6 +7988,9 @@ struct task_struct *curr_task(int cpu)
         return cpu_curr(cpu);
 }
 
+#endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */
+
+#ifdef CONFIG_IA64
 /**
  * set_curr_task - set the current task for a given cpu.
  * @cpu: the processor in question.
@@ -8264,8 +8275,6 @@ void sched_move_task(struct task_struct *tsk)
 
         rq = task_rq_lock(tsk, &flags);
 
-        update_rq_clock(rq);
-
         running = task_current(rq, tsk);
         on_rq = tsk->se.on_rq;
 
@@ -8284,7 +8293,7 @@ void sched_move_task(struct task_struct *tsk)
         if (unlikely(running))
                 tsk->sched_class->set_curr_task(rq);
         if (on_rq)
-                enqueue_task(rq, tsk, 0, false);
+                enqueue_task(rq, tsk, 0);
 
         task_rq_unlock(rq, &flags);
 }
@@ -9098,43 +9107,32 @@ struct cgroup_subsys cpuacct_subsys = {
 
 #ifndef CONFIG_SMP
 
-int rcu_expedited_torture_stats(char *page)
-{
-        return 0;
-}
-EXPORT_SYMBOL_GPL(rcu_expedited_torture_stats);
-
 void synchronize_sched_expedited(void)
 {
+        barrier();
 }
 EXPORT_SYMBOL_GPL(synchronize_sched_expedited);
 
 #else /* #ifndef CONFIG_SMP */
 
-static DEFINE_PER_CPU(struct migration_req, rcu_migration_req);
-static DEFINE_MUTEX(rcu_sched_expedited_mutex);
-
-#define RCU_EXPEDITED_STATE_POST -2
-#define RCU_EXPEDITED_STATE_IDLE -1
+static atomic_t synchronize_sched_expedited_count = ATOMIC_INIT(0);
 
-static int rcu_expedited_state = RCU_EXPEDITED_STATE_IDLE;
-
-int rcu_expedited_torture_stats(char *page)
+static int synchronize_sched_expedited_cpu_stop(void *data)
 {
-        int cnt = 0;
-        int cpu;
-
-        cnt += sprintf(&page[cnt], "state: %d /", rcu_expedited_state);
-        for_each_online_cpu(cpu) {
-                 cnt += sprintf(&page[cnt], " %d:%d",
-                                cpu, per_cpu(rcu_migration_req, cpu).dest_cpu);
-        }
-        cnt += sprintf(&page[cnt], "\n");
-        return cnt;
+        /*
+         * There must be a full memory barrier on each affected CPU
+         * between the time that try_stop_cpus() is called and the
+         * time that it returns.
+         *
+         * In the current initial implementation of cpu_stop, the
+         * above condition is already met when the control reaches
+         * this point and the following smp_mb() is not strictly
+         * necessary.  Do smp_mb() anyway for documentation and
+         * robustness against future implementation changes.
+         */
+        smp_mb(); /* See above comment block. */
+        return 0;
 }
-EXPORT_SYMBOL_GPL(rcu_expedited_torture_stats);
-
-static long synchronize_sched_expedited_count;
 
 /*
  * Wait for an rcu-sched grace period to elapse, but use "big hammer"
@@ -9148,18 +9146,14 @@ static long synchronize_sched_expedited_count;
  */
 void synchronize_sched_expedited(void)
 {
-        int cpu;
-        unsigned long flags;
-        bool need_full_sync = 0;
-        struct rq *rq;
-        struct migration_req *req;
-        long snap;
-        int trycount = 0;
+        int snap, trycount = 0;
 
         smp_mb();  /* ensure prior mod happens before capturing snap. */
-        snap = ACCESS_ONCE(synchronize_sched_expedited_count) + 1;
+        snap = atomic_read(&synchronize_sched_expedited_count) + 1;
         get_online_cpus();
-        while (!mutex_trylock(&rcu_sched_expedited_mutex)) {
+        while (try_stop_cpus(cpu_online_mask,
+                             synchronize_sched_expedited_cpu_stop,
+                             NULL) == -EAGAIN) {
                 put_online_cpus();
                 if (trycount++ < 10)
                         udelay(trycount * num_online_cpus());
@@ -9167,41 +9161,15 @@ void synchronize_sched_expedited(void)
                         synchronize_sched();
                         return;
                 }
-                if (ACCESS_ONCE(synchronize_sched_expedited_count) - snap > 0) {
+                if (atomic_read(&synchronize_sched_expedited_count) - snap > 0) {
                         smp_mb(); /* ensure test happens before caller kfree */
                         return;
                 }
                 get_online_cpus();
         }
-        rcu_expedited_state = RCU_EXPEDITED_STATE_POST;
-        for_each_online_cpu(cpu) {
-                rq = cpu_rq(cpu);
-                req = &per_cpu(rcu_migration_req, cpu);
-                init_completion(&req->done);
-                req->task = NULL;
-                req->dest_cpu = RCU_MIGRATION_NEED_QS;
-                raw_spin_lock_irqsave(&rq->lock, flags);
-                list_add(&req->list, &rq->migration_queue);
-                raw_spin_unlock_irqrestore(&rq->lock, flags);
-                wake_up_process(rq->migration_thread);
-        }
-        for_each_online_cpu(cpu) {
-                rcu_expedited_state = cpu;
-                req = &per_cpu(rcu_migration_req, cpu);
-                rq = cpu_rq(cpu);
-                wait_for_completion(&req->done);
-                raw_spin_lock_irqsave(&rq->lock, flags);
-                if (unlikely(req->dest_cpu == RCU_MIGRATION_MUST_SYNC))
-                        need_full_sync = 1;
-                req->dest_cpu = RCU_MIGRATION_IDLE;
-                raw_spin_unlock_irqrestore(&rq->lock, flags);
-        }
-        rcu_expedited_state = RCU_EXPEDITED_STATE_IDLE;
-        synchronize_sched_expedited_count++;
-        mutex_unlock(&rcu_sched_expedited_mutex);
+        atomic_inc(&synchronize_sched_expedited_count);
+        smp_mb__after_atomic_inc(); /* ensure post-GP actions seen after GP. */
         put_online_cpus();
-        if (need_full_sync)
-                synchronize_sched();
 }
 EXPORT_SYMBOL_GPL(synchronize_sched_expedited);