Merge branch 'for-3.5' of ../cgroup into block/for-3.5/core-merged

cgroup/for-3.5 contains the following changes which blk-cgroup needs
to proceed with the on-going cleanup.

* Dynamic addition and removal of cftypes to make config/stat file
  handling modular for policies.

* cgroup removal update to not wait for css references to drain to fix
  blkcg removal hang caused by cfq caching cfqgs.

Pull in cgroup/for-3.5 into block/for-3.5/core.  This causes the
following conflicts in block/blk-cgroup.c.

* 761b3ef50e "cgroup: remove cgroup_subsys argument from callbacks"
  conflicts with blkiocg_pre_destroy() addition and blkiocg_attach()
  removal.  Resolved by removing @subsys from all subsys methods.

* 676f7c8f84 "cgroup: relocate cftype and cgroup_subsys definitions in
  controllers" conflicts with ->pre_destroy() and ->attach() updates
  and removal of modular config.  Resolved by dropping forward
  declarations of the methods and applying updates to the relocated
  blkio_subsys.

* 4baf6e3325 "cgroup: convert all non-memcg controllers to the new
  cftype interface" builds upon the previous item.  Resolved by adding
  ->base_cftypes to the relocated blkio_subsys.

Signed-off-by: Tejun Heo <tj@kernel.org>

7 files changed, 402 insertions, 365 deletions

diff --git a/kernel/sched/auto_group.c b/kernel/sched/auto_group.c
index e8a1f83ee0e7..0984a21076a3 100644
--- a/kernel/sched/auto_group.c
+++ b/kernel/sched/auto_group.c
@@ -195,20 +195,20 @@ __setup("noautogroup", setup_autogroup);
 
 #ifdef CONFIG_PROC_FS
 
-int proc_sched_autogroup_set_nice(struct task_struct *p, int *nice)
+int proc_sched_autogroup_set_nice(struct task_struct *p, int nice)
 {
         static unsigned long next = INITIAL_JIFFIES;
         struct autogroup *ag;
         int err;
 
-        if (*nice < -20 || *nice > 19)
+        if (nice < -20 || nice > 19)
                 return -EINVAL;
 
-        err = security_task_setnice(current, *nice);
+        err = security_task_setnice(current, nice);
         if (err)
                 return err;
 
-        if (*nice < 0 && !can_nice(current, *nice))
+        if (nice < 0 && !can_nice(current, nice))
                 return -EPERM;
 
         /* this is a heavy operation taking global locks.. */
@@ -219,9 +219,9 @@ int proc_sched_autogroup_set_nice(struct task_struct *p, int *nice)
         ag = autogroup_task_get(p);
 
         down_write(&ag->lock);
-        err = sched_group_set_shares(ag->tg, prio_to_weight[*nice + 20]);
+        err = sched_group_set_shares(ag->tg, prio_to_weight[nice + 20]);
         if (!err)
-                ag->nice = *nice;
+                ag->nice = nice;
         up_write(&ag->lock);
 
         autogroup_kref_put(ag);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 5255c9d2e053..afc6d7e71557 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -71,7 +71,9 @@
 #include <linux/ftrace.h>
 #include <linux/slab.h>
 #include <linux/init_task.h>
+#include <linux/binfmts.h>
 
+#include <asm/switch_to.h>
 #include <asm/tlb.h>
 #include <asm/irq_regs.h>
 #include <asm/mutex.h>
@@ -162,13 +164,13 @@ static int sched_feat_show(struct seq_file *m, void *v)
 
 #ifdef HAVE_JUMP_LABEL
 
-#define jump_label_key__true  jump_label_key_enabled
-#define jump_label_key__false jump_label_key_disabled
+#define jump_label_key__true  STATIC_KEY_INIT_TRUE
+#define jump_label_key__false STATIC_KEY_INIT_FALSE
 
 #define SCHED_FEAT(name, enabled)       \
         jump_label_key__##enabled ,
 
-struct jump_label_key sched_feat_keys[__SCHED_FEAT_NR] = {
+struct static_key sched_feat_keys[__SCHED_FEAT_NR] = {
 #include "features.h"
 };
 
@@ -176,14 +178,14 @@ struct jump_label_key sched_feat_keys[__SCHED_FEAT_NR] = {
 
 static void sched_feat_disable(int i)
 {
-        if (jump_label_enabled(&sched_feat_keys[i]))
-                jump_label_dec(&sched_feat_keys[i]);
+        if (static_key_enabled(&sched_feat_keys[i]))
+                static_key_slow_dec(&sched_feat_keys[i]);
 }
 
 static void sched_feat_enable(int i)
 {
-        if (!jump_label_enabled(&sched_feat_keys[i]))
-                jump_label_inc(&sched_feat_keys[i]);
+        if (!static_key_enabled(&sched_feat_keys[i]))
+                static_key_slow_inc(&sched_feat_keys[i]);
 }
 #else
 static void sched_feat_disable(int i) { };
@@ -894,7 +896,7 @@ static void update_rq_clock_task(struct rq *rq, s64 delta)
         delta -= irq_delta;
 #endif
 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
-        if (static_branch((&paravirt_steal_rq_enabled))) {
+        if (static_key_false((&paravirt_steal_rq_enabled))) {
                 u64 st;
 
                 steal = paravirt_steal_clock(cpu_of(rq));
@@ -1263,29 +1265,59 @@ EXPORT_SYMBOL_GPL(kick_process);
  */
 static int select_fallback_rq(int cpu, struct task_struct *p)
 {
-        int dest_cpu;
         const struct cpumask *nodemask = cpumask_of_node(cpu_to_node(cpu));
+        enum { cpuset, possible, fail } state = cpuset;
+        int dest_cpu;
 
         /* Look for allowed, online CPU in same node. */
-        for_each_cpu_and(dest_cpu, nodemask, cpu_active_mask)
+        for_each_cpu(dest_cpu, nodemask) {
+                if (!cpu_online(dest_cpu))
+                        continue;
+                if (!cpu_active(dest_cpu))
+                        continue;
                 if (cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p)))
                         return dest_cpu;
+        }
 
-        /* Any allowed, online CPU? */
-        dest_cpu = cpumask_any_and(tsk_cpus_allowed(p), cpu_active_mask);
-        if (dest_cpu < nr_cpu_ids)
-                return dest_cpu;
+        for (;;) {
+                /* Any allowed, online CPU? */
+                for_each_cpu(dest_cpu, tsk_cpus_allowed(p)) {
+                        if (!cpu_online(dest_cpu))
+                                continue;
+                        if (!cpu_active(dest_cpu))
+                                continue;
+                        goto out;
+                }
 
-        /* No more Mr. Nice Guy. */
-        dest_cpu = cpuset_cpus_allowed_fallback(p);
-        /*
-         * Don't tell them about moving exiting tasks or
-         * kernel threads (both mm NULL), since they never
-         * leave kernel.
-         */
-        if (p->mm && printk_ratelimit()) {
-                printk(KERN_INFO "process %d (%s) no longer affine to cpu%d\n",
-                                task_pid_nr(p), p->comm, cpu);
+                switch (state) {
+                case cpuset:
+                        /* No more Mr. Nice Guy. */
+                        cpuset_cpus_allowed_fallback(p);
+                        state = possible;
+                        break;
+
+                case possible:
+                        do_set_cpus_allowed(p, cpu_possible_mask);
+                        state = fail;
+                        break;
+
+                case fail:
+                        BUG();
+                        break;
+                }
+        }
+
+out:
+        if (state != cpuset) {
+                /*
+                 * Don't tell them about moving exiting tasks or
+                 * kernel threads (both mm NULL), since they never
+                 * leave kernel.
+                 */
+                if (p->mm && printk_ratelimit()) {
+                        printk_sched("process %d (%s) no longer affine to cpu%d\n",
+                                        task_pid_nr(p), p->comm, cpu);
+                }
         }
 
         return dest_cpu;
@@ -1507,7 +1539,7 @@ static int ttwu_activate_remote(struct task_struct *p, int wake_flags)
 }
 #endif /* __ARCH_WANT_INTERRUPTS_ON_CTXSW */
 
-static inline int ttwu_share_cache(int this_cpu, int that_cpu)
+bool cpus_share_cache(int this_cpu, int that_cpu)
 {
         return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);
 }
@@ -1518,7 +1550,7 @@ static void ttwu_queue(struct task_struct *p, int cpu)
         struct rq *rq = cpu_rq(cpu);
 
 #if defined(CONFIG_SMP)
-        if (sched_feat(TTWU_QUEUE) && !ttwu_share_cache(smp_processor_id(), cpu)) {
+        if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) {
                 sched_clock_cpu(cpu); /* sync clocks x-cpu */
                 ttwu_queue_remote(p, cpu);
                 return;
@@ -1932,7 +1964,7 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev)
         local_irq_enable();
 #endif /* __ARCH_WANT_INTERRUPTS_ON_CTXSW */
         finish_lock_switch(rq, prev);
-        trace_sched_stat_sleeptime(current, rq->clock);
+        finish_arch_post_lock_switch();
 
         fire_sched_in_preempt_notifiers(current);
         if (mm)
@@ -2267,13 +2299,10 @@ calc_load_n(unsigned long load, unsigned long exp,
  * Once we've updated the global active value, we need to apply the exponential
  * weights adjusted to the number of cycles missed.
  */
-static void calc_global_nohz(unsigned long ticks)
+static void calc_global_nohz(void)
 {
         long delta, active, n;
 
-        if (time_before(jiffies, calc_load_update))
-                return;
-
         /*
          * If we crossed a calc_load_update boundary, make sure to fold
          * any pending idle changes, the respective CPUs might have
@@ -2285,31 +2314,25 @@ static void calc_global_nohz(unsigned long ticks)
                 atomic_long_add(delta, &calc_load_tasks);
 
         /*
-         * If we were idle for multiple load cycles, apply them.
+         * It could be the one fold was all it took, we done!
          */
-        if (ticks >= LOAD_FREQ) {
-                n = ticks / LOAD_FREQ;
+        if (time_before(jiffies, calc_load_update + 10))
+                return;
 
-                active = atomic_long_read(&calc_load_tasks);
-                active = active > 0 ? active * FIXED_1 : 0;
+        /*
+         * Catch-up, fold however many we are behind still
+         */
+        delta = jiffies - calc_load_update - 10;
+        n = 1 + (delta / LOAD_FREQ);
 
-                avenrun[0] = calc_load_n(avenrun[0], EXP_1, active, n);
-                avenrun[1] = calc_load_n(avenrun[1], EXP_5, active, n);
-                avenrun[2] = calc_load_n(avenrun[2], EXP_15, active, n);
+        active = atomic_long_read(&calc_load_tasks);
+        active = active > 0 ? active * FIXED_1 : 0;
 
-                calc_load_update += n * LOAD_FREQ;
-        }
+        avenrun[0] = calc_load_n(avenrun[0], EXP_1, active, n);
+        avenrun[1] = calc_load_n(avenrun[1], EXP_5, active, n);
+        avenrun[2] = calc_load_n(avenrun[2], EXP_15, active, n);
 
-        /*
-         * Its possible the remainder of the above division also crosses
-         * a LOAD_FREQ period, the regular check in calc_global_load()
-         * which comes after this will take care of that.
-         *
-         * Consider us being 11 ticks before a cycle completion, and us
-         * sleeping for 4*LOAD_FREQ + 22 ticks, then the above code will
-         * age us 4 cycles, and the test in calc_global_load() will
-         * pick up the final one.
-         */
+        calc_load_update += n * LOAD_FREQ;
 }
 #else
 void calc_load_account_idle(struct rq *this_rq)
@@ -2321,7 +2344,7 @@ static inline long calc_load_fold_idle(void)
         return 0;
 }
 
-static void calc_global_nohz(unsigned long ticks)
+static void calc_global_nohz(void)
 {
 }
 #endif
@@ -2349,8 +2372,6 @@ void calc_global_load(unsigned long ticks)
 {
         long active;
 
-        calc_global_nohz(ticks);
-
         if (time_before(jiffies, calc_load_update + 10))
                 return;
 
@@ -2362,6 +2383,16 @@ void calc_global_load(unsigned long ticks)
         avenrun[2] = calc_load(avenrun[2], EXP_15, active);
 
         calc_load_update += LOAD_FREQ;
+
+        /*
+         * Account one period with whatever state we found before
+         * folding in the nohz state and ageing the entire idle period.
+         *
+         * This avoids loosing a sample when we go idle between 
+         * calc_load_account_active() (10 ticks ago) and now and thus
+         * under-accounting.
+         */
+        calc_global_nohz();
 }
 
 /*
@@ -2756,7 +2787,7 @@ void account_idle_time(cputime_t cputime)
 static __always_inline bool steal_account_process_tick(void)
 {
 #ifdef CONFIG_PARAVIRT
-        if (static_branch(&paravirt_steal_enabled)) {
+        if (static_key_false(&paravirt_steal_enabled)) {
                 u64 steal, st = 0;
 
                 steal = paravirt_steal_clock(smp_processor_id());
@@ -3071,8 +3102,6 @@ EXPORT_SYMBOL(sub_preempt_count);
  */
 static noinline void __schedule_bug(struct task_struct *prev)
 {
-        struct pt_regs *regs = get_irq_regs();
-
         if (oops_in_progress)
                 return;
 
@@ -3083,11 +3112,7 @@ static noinline void __schedule_bug(struct task_struct *prev)
         print_modules();
         if (irqs_disabled())
                 print_irqtrace_events(prev);
-
-        if (regs)
-                show_regs(regs);
-        else
-                dump_stack();
+        dump_stack();
 }
 
 /*
@@ -3221,14 +3246,14 @@ need_resched:
 
         post_schedule(rq);
 
-        preempt_enable_no_resched();
+        sched_preempt_enable_no_resched();
         if (need_resched())
                 goto need_resched;
 }
 
 static inline void sched_submit_work(struct task_struct *tsk)
 {
-        if (!tsk->state)
+        if (!tsk->state || tsk_is_pi_blocked(tsk))
                 return;
         /*
          * If we are going to sleep and we have plugged IO queued,
@@ -3247,6 +3272,18 @@ asmlinkage void __sched schedule(void)
 }
 EXPORT_SYMBOL(schedule);
 
+/**
+ * schedule_preempt_disabled - called with preemption disabled
+ *
+ * Returns with preemption disabled. Note: preempt_count must be 1
+ */
+void __sched schedule_preempt_disabled(void)
+{
+        sched_preempt_enable_no_resched();
+        schedule();
+        preempt_disable();
+}
+
 #ifdef CONFIG_MUTEX_SPIN_ON_OWNER
 
 static inline bool owner_running(struct mutex *lock, struct task_struct *owner)
@@ -3407,9 +3444,9 @@ EXPORT_SYMBOL(__wake_up);
 /*
  * Same as __wake_up but called with the spinlock in wait_queue_head_t held.
  */
-void __wake_up_locked(wait_queue_head_t *q, unsigned int mode)
+void __wake_up_locked(wait_queue_head_t *q, unsigned int mode, int nr)
 {
-        __wake_up_common(q, mode, 1, 0, NULL);
+        __wake_up_common(q, mode, nr, 0, NULL);
 }
 EXPORT_SYMBOL_GPL(__wake_up_locked);
 
@@ -3768,6 +3805,24 @@ void rt_mutex_setprio(struct task_struct *p, int prio)
 
         rq = __task_rq_lock(p);
 
+        /*
+         * Idle task boosting is a nono in general. There is one
+         * exception, when PREEMPT_RT and NOHZ is active:
+         *
+         * The idle task calls get_next_timer_interrupt() and holds
+         * the timer wheel base->lock on the CPU and another CPU wants
+         * to access the timer (probably to cancel it). We can safely
+         * ignore the boosting request, as the idle CPU runs this code
+         * with interrupts disabled and will complete the lock
+         * protected section without being interrupted. So there is no
+         * real need to boost.
+         */
+        if (unlikely(p == rq->idle)) {
+                WARN_ON(p != rq->curr);
+                WARN_ON(p->pi_blocked_on);
+                goto out_unlock;
+        }
+
         trace_sched_pi_setprio(p, prio);
         oldprio = p->prio;
         prev_class = p->sched_class;
@@ -3791,11 +3846,10 @@ void rt_mutex_setprio(struct task_struct *p, int prio)
                 enqueue_task(rq, p, oldprio < prio ? ENQUEUE_HEAD : 0);
 
         check_class_changed(rq, p, prev_class, oldprio);
+out_unlock:
         __task_rq_unlock(rq);
 }
-
 #endif
-
 void set_user_nice(struct task_struct *p, long nice)
 {
         int old_prio, delta, on_rq;
@@ -4475,7 +4529,7 @@ SYSCALL_DEFINE0(sched_yield)
         __release(rq->lock);
         spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
         do_raw_spin_unlock(&rq->lock);
-        preempt_enable_no_resched();
+        sched_preempt_enable_no_resched();
 
         schedule();
 
@@ -4549,8 +4603,24 @@ EXPORT_SYMBOL(__cond_resched_softirq);
 /**
  * yield - yield the current processor to other threads.
  *
- * This is a shortcut for kernel-space yielding - it marks the
- * thread runnable and calls sys_sched_yield().
+ * Do not ever use this function, there's a 99% chance you're doing it wrong.
+ *
+ * The scheduler is at all times free to pick the calling task as the most
+ * eligible task to run, if removing the yield() call from your code breaks
+ * it, its already broken.
+ *
+ * Typical broken usage is:
+ *
+ * while (!event)
+ *      yield();
+ *
+ * where one assumes that yield() will let 'the other' process run that will
+ * make event true. If the current task is a SCHED_FIFO task that will never
+ * happen. Never use yield() as a progress guarantee!!
+ *
+ * If you want to use yield() to wait for something, use wait_event().
+ * If you want to use yield() to be 'nice' for others, use cond_resched().
+ * If you still want to use yield(), do not!
  */
 void __sched yield(void)
 {
@@ -5382,7 +5452,7 @@ static int __cpuinit sched_cpu_active(struct notifier_block *nfb,
                                       unsigned long action, void *hcpu)
 {
         switch (action & ~CPU_TASKS_FROZEN) {
-        case CPU_ONLINE:
+        case CPU_STARTING:
         case CPU_DOWN_FAILED:
                 set_cpu_active((long)hcpu, true);
                 return NOTIFY_OK;
@@ -5754,7 +5824,7 @@ static void destroy_sched_domains(struct sched_domain *sd, int cpu)
  *
  * Also keep a unique ID per domain (we use the first cpu number in
  * the cpumask of the domain), this allows us to quickly tell if
- * two cpus are in the same cache domain, see ttwu_share_cache().
+ * two cpus are in the same cache domain, see cpus_share_cache().
  */
 DEFINE_PER_CPU(struct sched_domain *, sd_llc);
 DEFINE_PER_CPU(int, sd_llc_id);
@@ -6931,6 +7001,9 @@ void __init sched_init(void)
                 rq->online = 0;
                 rq->idle_stamp = 0;
                 rq->avg_idle = 2*sysctl_sched_migration_cost;
+
+                INIT_LIST_HEAD(&rq->cfs_tasks);
+
                 rq_attach_root(rq, &def_root_domain);
 #ifdef CONFIG_NO_HZ
                 rq->nohz_flags = 0;
@@ -7525,8 +7598,7 @@ static inline struct task_group *cgroup_tg(struct cgroup *cgrp)
                             struct task_group, css);
 }
 
-static struct cgroup_subsys_state *
-cpu_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cgrp)
+static struct cgroup_subsys_state *cpu_cgroup_create(struct cgroup *cgrp)
 {
         struct task_group *tg, *parent;
 
@@ -7543,15 +7615,14 @@ cpu_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cgrp)
         return &tg->css;
 }
 
-static void
-cpu_cgroup_destroy(struct cgroup_subsys *ss, struct cgroup *cgrp)
+static void cpu_cgroup_destroy(struct cgroup *cgrp)
 {
         struct task_group *tg = cgroup_tg(cgrp);
 
         sched_destroy_group(tg);
 }
 
-static int cpu_cgroup_can_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
+static int cpu_cgroup_can_attach(struct cgroup *cgrp,
                                  struct cgroup_taskset *tset)
 {
         struct task_struct *task;
@@ -7569,7 +7640,7 @@ static int cpu_cgroup_can_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
         return 0;
 }
 
-static void cpu_cgroup_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
+static void cpu_cgroup_attach(struct cgroup *cgrp,
                               struct cgroup_taskset *tset)
 {
         struct task_struct *task;
@@ -7579,8 +7650,8 @@ static void cpu_cgroup_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
 }
 
 static void
-cpu_cgroup_exit(struct cgroup_subsys *ss, struct cgroup *cgrp,
-                struct cgroup *old_cgrp, struct task_struct *task)
+cpu_cgroup_exit(struct cgroup *cgrp, struct cgroup *old_cgrp,
+                struct task_struct *task)
 {
         /*
          * cgroup_exit() is called in the copy_process() failure path.
@@ -7899,13 +7970,9 @@ static struct cftype cpu_files[] = {
                 .write_u64 = cpu_rt_period_write_uint,
         },
 #endif
+        { }     /* terminate */
 };
 
-static int cpu_cgroup_populate(struct cgroup_subsys *ss, struct cgroup *cont)
-{
-        return cgroup_add_files(cont, ss, cpu_files, ARRAY_SIZE(cpu_files));
-}
-
 struct cgroup_subsys cpu_cgroup_subsys = {
         .name           = "cpu",
         .create         = cpu_cgroup_create,
@@ -7913,8 +7980,8 @@ struct cgroup_subsys cpu_cgroup_subsys = {
         .can_attach     = cpu_cgroup_can_attach,
         .attach         = cpu_cgroup_attach,
         .exit           = cpu_cgroup_exit,
-        .populate       = cpu_cgroup_populate,
         .subsys_id      = cpu_cgroup_subsys_id,
+        .base_cftypes   = cpu_files,
         .early_init     = 1,
 };
 
@@ -7930,8 +7997,7 @@ struct cgroup_subsys cpu_cgroup_subsys = {
  */
 
 /* create a new cpu accounting group */
-static struct cgroup_subsys_state *cpuacct_create(
-        struct cgroup_subsys *ss, struct cgroup *cgrp)
+static struct cgroup_subsys_state *cpuacct_create(struct cgroup *cgrp)
 {
         struct cpuacct *ca;
 
@@ -7961,8 +8027,7 @@ out:
 }
 
 /* destroy an existing cpu accounting group */
-static void
-cpuacct_destroy(struct cgroup_subsys *ss, struct cgroup *cgrp)
+static void cpuacct_destroy(struct cgroup *cgrp)
 {
         struct cpuacct *ca = cgroup_ca(cgrp);
 
@@ -8101,13 +8166,9 @@ static struct cftype files[] = {
                 .name = "stat",
                 .read_map = cpuacct_stats_show,
         },
+        { }     /* terminate */
 };
 
-static int cpuacct_populate(struct cgroup_subsys *ss, struct cgroup *cgrp)
-{
-        return cgroup_add_files(cgrp, ss, files, ARRAY_SIZE(files));
-}
-
 /*
  * charge this task's execution time to its accounting group.
  *
@@ -8139,7 +8200,7 @@ struct cgroup_subsys cpuacct_subsys = {
         .name = "cpuacct",
         .create = cpuacct_create,
         .destroy = cpuacct_destroy,
-        .populate = cpuacct_populate,
         .subsys_id = cpuacct_subsys_id,
+        .base_cftypes = files,
 };
 #endif  /* CONFIG_CGROUP_CPUACCT */
diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
index 2a075e10004b..09acaa15161d 100644
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -288,7 +288,6 @@ static void print_cpu(struct seq_file *m, int cpu)
 
         P(yld_count);
 
-        P(sched_switch);
         P(sched_count);
         P(sched_goidle);
 #ifdef CONFIG_SMP
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 7c6414fc669d..0d97ebdc58f0 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -416,8 +416,8 @@ find_matching_se(struct sched_entity **se, struct sched_entity **pse)
 
 #endif  /* CONFIG_FAIR_GROUP_SCHED */
 
-static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
-                                   unsigned long delta_exec);
+static __always_inline
+void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, unsigned long delta_exec);
 
 /**************************************************************
  * Scheduling class tree data structure manipulation methods:
@@ -776,29 +776,16 @@ update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
  * Scheduling class queueing methods:
  */
 
-#if defined CONFIG_SMP && defined CONFIG_FAIR_GROUP_SCHED
-static void
-add_cfs_task_weight(struct cfs_rq *cfs_rq, unsigned long weight)
-{
-        cfs_rq->task_weight += weight;
-}
-#else
-static inline void
-add_cfs_task_weight(struct cfs_rq *cfs_rq, unsigned long weight)
-{
-}
-#endif
-
 static void
 account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
         update_load_add(&cfs_rq->load, se->load.weight);
         if (!parent_entity(se))
                 update_load_add(&rq_of(cfs_rq)->load, se->load.weight);
-        if (entity_is_task(se)) {
-                add_cfs_task_weight(cfs_rq, se->load.weight);
-                list_add(&se->group_node, &cfs_rq->tasks);
-        }
+#ifdef CONFIG_SMP
+        if (entity_is_task(se))
+                list_add_tail(&se->group_node, &rq_of(cfs_rq)->cfs_tasks);
+#endif
         cfs_rq->nr_running++;
 }
 
@@ -808,10 +795,8 @@ account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
         update_load_sub(&cfs_rq->load, se->load.weight);
         if (!parent_entity(se))
                 update_load_sub(&rq_of(cfs_rq)->load, se->load.weight);
-        if (entity_is_task(se)) {
-                add_cfs_task_weight(cfs_rq, -se->load.weight);
+        if (entity_is_task(se))
                 list_del_init(&se->group_node);
-        }
         cfs_rq->nr_running--;
 }
 
@@ -1003,6 +988,7 @@ static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)
                 if (unlikely(delta > se->statistics.sleep_max))
                         se->statistics.sleep_max = delta;
 
+                se->statistics.sleep_start = 0;
                 se->statistics.sum_sleep_runtime += delta;
 
                 if (tsk) {
@@ -1019,6 +1005,7 @@ static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)
                 if (unlikely(delta > se->statistics.block_max))
                         se->statistics.block_max = delta;
 
+                se->statistics.block_start = 0;
                 se->statistics.sum_sleep_runtime += delta;
 
                 if (tsk) {
@@ -1175,7 +1162,7 @@ static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se)
                 __clear_buddies_skip(se);
 }
 
-static void return_cfs_rq_runtime(struct cfs_rq *cfs_rq);
+static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq);
 
 static void
 dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
@@ -1399,20 +1386,20 @@ entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued)
 #ifdef CONFIG_CFS_BANDWIDTH
 
 #ifdef HAVE_JUMP_LABEL
-static struct jump_label_key __cfs_bandwidth_used;
+static struct static_key __cfs_bandwidth_used;
 
 static inline bool cfs_bandwidth_used(void)
 {
-        return static_branch(&__cfs_bandwidth_used);
+        return static_key_false(&__cfs_bandwidth_used);
 }
 
 void account_cfs_bandwidth_used(int enabled, int was_enabled)
 {
         /* only need to count groups transitioning between enabled/!enabled */
         if (enabled && !was_enabled)
-                jump_label_inc(&__cfs_bandwidth_used);
+                static_key_slow_inc(&__cfs_bandwidth_used);
         else if (!enabled && was_enabled)
-                jump_label_dec(&__cfs_bandwidth_used);
+                static_key_slow_dec(&__cfs_bandwidth_used);
 }
 #else /* HAVE_JUMP_LABEL */
 static bool cfs_bandwidth_used(void)
@@ -1559,8 +1546,8 @@ static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
                 resched_task(rq_of(cfs_rq)->curr);
 }
 
-static __always_inline void account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
-                                                   unsigned long delta_exec)
+static __always_inline
+void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, unsigned long delta_exec)
 {
         if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled)
                 return;
@@ -2086,11 +2073,11 @@ void unthrottle_offline_cfs_rqs(struct rq *rq)
 }
 
 #else /* CONFIG_CFS_BANDWIDTH */
-static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
-                                     unsigned long delta_exec) {}
+static __always_inline
+void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, unsigned long delta_exec) {}
 static void check_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
 static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {}
-static void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
+static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
 
 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)
 {
@@ -2670,8 +2657,6 @@ static int select_idle_sibling(struct task_struct *p, int target)
         /*
          * Otherwise, iterate the domains and find an elegible idle cpu.
          */
-        rcu_read_lock();
-
         sd = rcu_dereference(per_cpu(sd_llc, target));
         for_each_lower_domain(sd) {
                 sg = sd->groups;
@@ -2693,8 +2678,6 @@ next:
                 } while (sg != sd->groups);
         }
 done:
-        rcu_read_unlock();
-
         return target;
 }
 
@@ -2920,7 +2903,7 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_
                 return;
 
         /*
-         * This is possible from callers such as pull_task(), in which we
+         * This is possible from callers such as move_task(), in which we
          * unconditionally check_prempt_curr() after an enqueue (which may have
          * lead to a throttle).  This both saves work and prevents false
          * next-buddy nomination below.
@@ -3084,17 +3067,39 @@ static bool yield_to_task_fair(struct rq *rq, struct task_struct *p, bool preemp
  * Fair scheduling class load-balancing methods:
  */
 
+static unsigned long __read_mostly max_load_balance_interval = HZ/10;
+
+#define LBF_ALL_PINNED  0x01
+#define LBF_NEED_BREAK  0x02
+
+struct lb_env {
+        struct sched_domain     *sd;
+
+        int                     src_cpu;
+        struct rq               *src_rq;
+
+        int                     dst_cpu;
+        struct rq               *dst_rq;
+
+        enum cpu_idle_type      idle;
+        long                    load_move;
+        unsigned int            flags;
+
+        unsigned int            loop;
+        unsigned int            loop_break;
+        unsigned int            loop_max;
+};
+
 /*
- * pull_task - move a task from a remote runqueue to the local runqueue.
+ * move_task - move a task from one runqueue to another runqueue.
  * Both runqueues must be locked.
  */
-static void pull_task(struct rq *src_rq, struct task_struct *p,
-                      struct rq *this_rq, int this_cpu)
+static void move_task(struct task_struct *p, struct lb_env *env)
 {
-        deactivate_task(src_rq, p, 0);
-        set_task_cpu(p, this_cpu);
-        activate_task(this_rq, p, 0);
-        check_preempt_curr(this_rq, p, 0);
+        deactivate_task(env->src_rq, p, 0);
+        set_task_cpu(p, env->dst_cpu);
+        activate_task(env->dst_rq, p, 0);
+        check_preempt_curr(env->dst_rq, p, 0);
 }
 
 /*
@@ -3129,19 +3134,11 @@ task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
         return delta < (s64)sysctl_sched_migration_cost;
 }
 
-#define LBF_ALL_PINNED  0x01
-#define LBF_NEED_BREAK  0x02    /* clears into HAD_BREAK */
-#define LBF_HAD_BREAK   0x04
-#define LBF_HAD_BREAKS  0x0C    /* count HAD_BREAKs overflows into ABORT */
-#define LBF_ABORT       0x10
-
 /*
  * can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
  */
 static
-int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu,
-                     struct sched_domain *sd, enum cpu_idle_type idle,
-                     int *lb_flags)
+int can_migrate_task(struct task_struct *p, struct lb_env *env)
 {
         int tsk_cache_hot = 0;
         /*
@@ -3150,13 +3147,13 @@ int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu,
          * 2) cannot be migrated to this CPU due to cpus_allowed, or
          * 3) are cache-hot on their current CPU.
          */
-        if (!cpumask_test_cpu(this_cpu, tsk_cpus_allowed(p))) {
+        if (!cpumask_test_cpu(env->dst_cpu, tsk_cpus_allowed(p))) {
                 schedstat_inc(p, se.statistics.nr_failed_migrations_affine);
                 return 0;
         }
-        *lb_flags &= ~LBF_ALL_PINNED;
+        env->flags &= ~LBF_ALL_PINNED;
 
-        if (task_running(rq, p)) {
+        if (task_running(env->src_rq, p)) {
                 schedstat_inc(p, se.statistics.nr_failed_migrations_running);
                 return 0;
         }
@@ -3167,12 +3164,12 @@ int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu,
          * 2) too many balance attempts have failed.
          */
 
-        tsk_cache_hot = task_hot(p, rq->clock_task, sd);
+        tsk_cache_hot = task_hot(p, env->src_rq->clock_task, env->sd);
         if (!tsk_cache_hot ||
-                sd->nr_balance_failed > sd->cache_nice_tries) {
+                env->sd->nr_balance_failed > env->sd->cache_nice_tries) {
 #ifdef CONFIG_SCHEDSTATS
                 if (tsk_cache_hot) {
-                        schedstat_inc(sd, lb_hot_gained[idle]);
+                        schedstat_inc(env->sd, lb_hot_gained[env->idle]);
                         schedstat_inc(p, se.statistics.nr_forced_migrations);
                 }
 #endif
@@ -3193,65 +3190,80 @@ int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu,
  *
  * Called with both runqueues locked.
  */
-static int
-move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
-              struct sched_domain *sd, enum cpu_idle_type idle)
+static int move_one_task(struct lb_env *env)
 {
         struct task_struct *p, *n;
-        struct cfs_rq *cfs_rq;
-        int pinned = 0;
 
-        for_each_leaf_cfs_rq(busiest, cfs_rq) {
-                list_for_each_entry_safe(p, n, &cfs_rq->tasks, se.group_node) {
-                        if (throttled_lb_pair(task_group(p),
-                                              busiest->cpu, this_cpu))
-                                break;
+        list_for_each_entry_safe(p, n, &env->src_rq->cfs_tasks, se.group_node) {
+                if (throttled_lb_pair(task_group(p), env->src_rq->cpu, env->dst_cpu))
+                        continue;
 
-                        if (!can_migrate_task(p, busiest, this_cpu,
-                                                sd, idle, &pinned))
-                                continue;
+                if (!can_migrate_task(p, env))
+                        continue;
 
-                        pull_task(busiest, p, this_rq, this_cpu);
-                        /*
-                         * Right now, this is only the second place pull_task()
-                         * is called, so we can safely collect pull_task()
-                         * stats here rather than inside pull_task().
-                         */
-                        schedstat_inc(sd, lb_gained[idle]);
-                        return 1;
-                }
+                move_task(p, env);
+                /*
+                 * Right now, this is only the second place move_task()
+                 * is called, so we can safely collect move_task()
+                 * stats here rather than inside move_task().
+                 */
+                schedstat_inc(env->sd, lb_gained[env->idle]);
+                return 1;
         }
-
         return 0;
 }
 
-static unsigned long
-balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
-              unsigned long max_load_move, struct sched_domain *sd,
-              enum cpu_idle_type idle, int *lb_flags,
-              struct cfs_rq *busiest_cfs_rq)
+static unsigned long task_h_load(struct task_struct *p);
+
+/*
+ * move_tasks tries to move up to load_move weighted load from busiest to
+ * this_rq, as part of a balancing operation within domain "sd".
+ * Returns 1 if successful and 0 otherwise.
+ *
+ * Called with both runqueues locked.
+ */
+static int move_tasks(struct lb_env *env)
 {
-        int loops = 0, pulled = 0;
-        long rem_load_move = max_load_move;
-        struct task_struct *p, *n;
+        struct list_head *tasks = &env->src_rq->cfs_tasks;
+        struct task_struct *p;
+        unsigned long load;
+        int pulled = 0;
+
+        if (env->load_move <= 0)
+                return 0;
 
-        if (max_load_move == 0)
-                goto out;
+        while (!list_empty(tasks)) {
+                p = list_first_entry(tasks, struct task_struct, se.group_node);
 
-        list_for_each_entry_safe(p, n, &busiest_cfs_rq->tasks, se.group_node) {
-                if (loops++ > sysctl_sched_nr_migrate) {
-                        *lb_flags |= LBF_NEED_BREAK;
+                env->loop++;
+                /* We've more or less seen every task there is, call it quits */
+                if (env->loop > env->loop_max)
+                        break;
+
+                /* take a breather every nr_migrate tasks */
+                if (env->loop > env->loop_break) {
+                        env->loop_break += sysctl_sched_nr_migrate;
+                        env->flags |= LBF_NEED_BREAK;
                         break;
                 }
 
-                if ((p->se.load.weight >> 1) > rem_load_move ||
-                    !can_migrate_task(p, busiest, this_cpu, sd, idle,
-                                      lb_flags))
-                        continue;
+                if (throttled_lb_pair(task_group(p), env->src_cpu, env->dst_cpu))
+                        goto next;
+
+                load = task_h_load(p);
+
+                if (load < 16 && !env->sd->nr_balance_failed)
+                        goto next;
+
+                if ((load / 2) > env->load_move)
+                        goto next;
 
-                pull_task(busiest, p, this_rq, this_cpu);
+                if (!can_migrate_task(p, env))
+                        goto next;
+
+                move_task(p, env);
                 pulled++;
-                rem_load_move -= p->se.load.weight;
+                env->load_move -= load;
 
 #ifdef CONFIG_PREEMPT
                 /*
@@ -3259,28 +3271,30 @@ balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
                  * kernels will stop after the first task is pulled to minimize
                  * the critical section.
                  */
-                if (idle == CPU_NEWLY_IDLE) {
-                        *lb_flags |= LBF_ABORT;
+                if (env->idle == CPU_NEWLY_IDLE)
                         break;
-                }
 #endif
 
                 /*
                  * We only want to steal up to the prescribed amount of
                  * weighted load.
                  */
-                if (rem_load_move <= 0)
+                if (env->load_move <= 0)
                         break;
+
+                continue;
+next:
+                list_move_tail(&p->se.group_node, tasks);
         }
-out:
+
         /*
-         * Right now, this is one of only two places pull_task() is called,
-         * so we can safely collect pull_task() stats here rather than
-         * inside pull_task().
+         * Right now, this is one of only two places move_task() is called,
+         * so we can safely collect move_task() stats here rather than
+         * inside move_task().
          */
-        schedstat_add(sd, lb_gained[idle], pulled);
+        schedstat_add(env->sd, lb_gained[env->idle], pulled);
 
-        return max_load_move - rem_load_move;
+        return pulled;
 }
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
@@ -3360,113 +3374,35 @@ static int tg_load_down(struct task_group *tg, void *data)
 
 static void update_h_load(long cpu)
 {
+        rcu_read_lock();
         walk_tg_tree(tg_load_down, tg_nop, (void *)cpu);
+        rcu_read_unlock();
 }
 
-static unsigned long
-load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
-                  unsigned long max_load_move,
-                  struct sched_domain *sd, enum cpu_idle_type idle,
-                  int *lb_flags)
+static unsigned long task_h_load(struct task_struct *p)
 {
-        long rem_load_move = max_load_move;
-        struct cfs_rq *busiest_cfs_rq;
-
-        rcu_read_lock();
-        update_h_load(cpu_of(busiest));
-
-        for_each_leaf_cfs_rq(busiest, busiest_cfs_rq) {
-                unsigned long busiest_h_load = busiest_cfs_rq->h_load;
-                unsigned long busiest_weight = busiest_cfs_rq->load.weight;
-                u64 rem_load, moved_load;
-
-                if (*lb_flags & (LBF_NEED_BREAK|LBF_ABORT))
-                        break;
-
-                /*
-                 * empty group or part of a throttled hierarchy
-                 */
-                if (!busiest_cfs_rq->task_weight ||
-                    throttled_lb_pair(busiest_cfs_rq->tg, cpu_of(busiest), this_cpu))
-                        continue;
-
-                rem_load = (u64)rem_load_move * busiest_weight;
-                rem_load = div_u64(rem_load, busiest_h_load + 1);
-
-                moved_load = balance_tasks(this_rq, this_cpu, busiest,
-                                rem_load, sd, idle, lb_flags,
-                                busiest_cfs_rq);
-
-                if (!moved_load)
-                        continue;
+        struct cfs_rq *cfs_rq = task_cfs_rq(p);
+        unsigned long load;
 
-                moved_load *= busiest_h_load;
-                moved_load = div_u64(moved_load, busiest_weight + 1);
+        load = p->se.load.weight;
+        load = div_u64(load * cfs_rq->h_load, cfs_rq->load.weight + 1);
 
-                rem_load_move -= moved_load;
-                if (rem_load_move < 0)
-                        break;
-        }
-        rcu_read_unlock();
-
-        return max_load_move - rem_load_move;
+        return load;
 }
 #else
 static inline void update_shares(int cpu)
 {
 }
 
-static unsigned long
-load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
-                  unsigned long max_load_move,
-                  struct sched_domain *sd, enum cpu_idle_type idle,
-                  int *lb_flags)
+static inline void update_h_load(long cpu)
 {
-        return balance_tasks(this_rq, this_cpu, busiest,
-                        max_load_move, sd, idle, lb_flags,
-                        &busiest->cfs);
 }
-#endif
 
-/*
- * move_tasks tries to move up to max_load_move weighted load from busiest to
- * this_rq, as part of a balancing operation within domain "sd".
- * Returns 1 if successful and 0 otherwise.
- *
- * Called with both runqueues locked.
- */
-static int move_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
-                      unsigned long max_load_move,
-                      struct sched_domain *sd, enum cpu_idle_type idle,
-                      int *lb_flags)
+static unsigned long task_h_load(struct task_struct *p)
 {
-        unsigned long total_load_moved = 0, load_moved;
-
-        do {
-                load_moved = load_balance_fair(this_rq, this_cpu, busiest,
-                                max_load_move - total_load_moved,
-                                sd, idle, lb_flags);
-
-                total_load_moved += load_moved;
-
-                if (*lb_flags & (LBF_NEED_BREAK|LBF_ABORT))
-                        break;
-
-#ifdef CONFIG_PREEMPT
-                /*
-                 * NEWIDLE balancing is a source of latency, so preemptible
-                 * kernels will stop after the first task is pulled to minimize
-                 * the critical section.
-                 */
-                if (idle == CPU_NEWLY_IDLE && this_rq->nr_running) {
-                        *lb_flags |= LBF_ABORT;
-                        break;
-                }
-#endif
-        } while (load_moved && max_load_move > total_load_moved);
-
-        return total_load_moved > 0;
+        return p->se.load.weight;
 }
+#endif
 
 /********** Helpers for find_busiest_group ************************/
 /*
@@ -3776,6 +3712,11 @@ void update_group_power(struct sched_domain *sd, int cpu)
         struct sched_domain *child = sd->child;
         struct sched_group *group, *sdg = sd->groups;
         unsigned long power;
+        unsigned long interval;
+
+        interval = msecs_to_jiffies(sd->balance_interval);
+        interval = clamp(interval, 1UL, max_load_balance_interval);
+        sdg->sgp->next_update = jiffies + interval;
 
         if (!child) {
                 update_cpu_power(sd, cpu);
@@ -3883,12 +3824,15 @@ static inline void update_sg_lb_stats(struct sched_domain *sd,
          * domains. In the newly idle case, we will allow all the cpu's
          * to do the newly idle load balance.
          */
-        if (idle != CPU_NEWLY_IDLE && local_group) {
-                if (balance_cpu != this_cpu) {
-                        *balance = 0;
-                        return;
-                }
-                update_group_power(sd, this_cpu);
+        if (local_group) {
+                if (idle != CPU_NEWLY_IDLE) {
+                        if (balance_cpu != this_cpu) {
+                                *balance = 0;
+                                return;
+                        }
+                        update_group_power(sd, this_cpu);
+                } else if (time_after_eq(jiffies, group->sgp->next_update))
+                        update_group_power(sd, this_cpu);
         }
 
         /* Adjust by relative CPU power of the group */
@@ -4451,13 +4395,21 @@ static int load_balance(int this_cpu, struct rq *this_rq,
                         struct sched_domain *sd, enum cpu_idle_type idle,
                         int *balance)
 {
-        int ld_moved, lb_flags = 0, active_balance = 0;
+        int ld_moved, active_balance = 0;
         struct sched_group *group;
         unsigned long imbalance;
         struct rq *busiest;
         unsigned long flags;
         struct cpumask *cpus = __get_cpu_var(load_balance_tmpmask);
 
+        struct lb_env env = {
+                .sd             = sd,
+                .dst_cpu        = this_cpu,
+                .dst_rq         = this_rq,
+                .idle           = idle,
+                .loop_break     = sysctl_sched_nr_migrate,
+        };
+
         cpumask_copy(cpus, cpu_active_mask);
 
         schedstat_inc(sd, lb_count[idle]);
@@ -4492,32 +4444,34 @@ redo:
                  * still unbalanced. ld_moved simply stays zero, so it is
                  * correctly treated as an imbalance.
                  */
-                lb_flags |= LBF_ALL_PINNED;
+                env.flags |= LBF_ALL_PINNED;
+                env.load_move = imbalance;
+                env.src_cpu = busiest->cpu;
+                env.src_rq = busiest;
+                env.loop_max = busiest->nr_running;
+
+more_balance:
                 local_irq_save(flags);
                 double_rq_lock(this_rq, busiest);
-                ld_moved = move_tasks(this_rq, this_cpu, busiest,
-                                      imbalance, sd, idle, &lb_flags);
+                if (!env.loop)
+                        update_h_load(env.src_cpu);
+                ld_moved += move_tasks(&env);
                 double_rq_unlock(this_rq, busiest);
                 local_irq_restore(flags);
 
+                if (env.flags & LBF_NEED_BREAK) {
+                        env.flags &= ~LBF_NEED_BREAK;
+                        goto more_balance;
+                }
+
                 /*
                  * some other cpu did the load balance for us.
                  */
                 if (ld_moved && this_cpu != smp_processor_id())
                         resched_cpu(this_cpu);
 
-                if (lb_flags & LBF_ABORT)
-                        goto out_balanced;
-
-                if (lb_flags & LBF_NEED_BREAK) {
-                        lb_flags += LBF_HAD_BREAK - LBF_NEED_BREAK;
-                        if (lb_flags & LBF_ABORT)
-                                goto out_balanced;
-                        goto redo;
-                }
-
                 /* All tasks on this runqueue were pinned by CPU affinity */
-                if (unlikely(lb_flags & LBF_ALL_PINNED)) {
+                if (unlikely(env.flags & LBF_ALL_PINNED)) {
                         cpumask_clear_cpu(cpu_of(busiest), cpus);
                         if (!cpumask_empty(cpus))
                                 goto redo;
@@ -4547,7 +4501,7 @@ redo:
                                         tsk_cpus_allowed(busiest->curr))) {
                                 raw_spin_unlock_irqrestore(&busiest->lock,
                                                             flags);
-                                lb_flags |= LBF_ALL_PINNED;
+                                env.flags |= LBF_ALL_PINNED;
                                 goto out_one_pinned;
                         }
 
@@ -4600,7 +4554,7 @@ out_balanced:
 
 out_one_pinned:
         /* tune up the balancing interval */
-        if (((lb_flags & LBF_ALL_PINNED) &&
+        if (((env.flags & LBF_ALL_PINNED) &&
                         sd->balance_interval < MAX_PINNED_INTERVAL) ||
                         (sd->balance_interval < sd->max_interval))
                 sd->balance_interval *= 2;
@@ -4710,10 +4664,18 @@ static int active_load_balance_cpu_stop(void *data)
         }
 
         if (likely(sd)) {
+                struct lb_env env = {
+                        .sd             = sd,
+                        .dst_cpu        = target_cpu,
+                        .dst_rq         = target_rq,
+                        .src_cpu        = busiest_rq->cpu,
+                        .src_rq         = busiest_rq,
+                        .idle           = CPU_IDLE,
+                };
+
                 schedstat_inc(sd, alb_count);
 
-                if (move_one_task(target_rq, target_cpu, busiest_rq,
-                                  sd, CPU_IDLE))
+                if (move_one_task(&env))
                         schedstat_inc(sd, alb_pushed);
                 else
                         schedstat_inc(sd, alb_failed);
@@ -4945,8 +4907,6 @@ static int __cpuinit sched_ilb_notifier(struct notifier_block *nfb,
 
 static DEFINE_SPINLOCK(balancing);
 
-static unsigned long __read_mostly max_load_balance_interval = HZ/10;
-
 /*
  * Scale the max load_balance interval with the number of CPUs in the system.
  * This trades load-balance latency on larger machines for less cross talk.
@@ -5340,7 +5300,6 @@ static void set_curr_task_fair(struct rq *rq)
 void init_cfs_rq(struct cfs_rq *cfs_rq)
 {
         cfs_rq->tasks_timeline = RB_ROOT;
-        INIT_LIST_HEAD(&cfs_rq->tasks);
         cfs_rq->min_vruntime = (u64)(-(1LL << 20));
 #ifndef CONFIG_64BIT
         cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime;
@@ -5612,6 +5571,7 @@ __init void init_sched_fair_class(void)
         open_softirq(SCHED_SOFTIRQ, run_rebalance_domains);
 
 #ifdef CONFIG_NO_HZ
+        nohz.next_balance = jiffies;
         zalloc_cpumask_var(&nohz.idle_cpus_mask, GFP_NOWAIT);
         cpu_notifier(sched_ilb_notifier, 0);
 #endif
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index f42ae7fb5ec5..44af55e6d5d0 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -778,12 +778,9 @@ static inline int balance_runtime(struct rt_rq *rt_rq)
 
 static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun)
 {
-        int i, idle = 1;
+        int i, idle = 1, throttled = 0;
         const struct cpumask *span;
 
-        if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF)
-                return 1;
-
         span = sched_rt_period_mask();
         for_each_cpu(i, span) {
                 int enqueue = 0;
@@ -818,12 +815,17 @@ static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun)
                         if (!rt_rq_throttled(rt_rq))
                                 enqueue = 1;
                 }
+                if (rt_rq->rt_throttled)
+                        throttled = 1;
 
                 if (enqueue)
                         sched_rt_rq_enqueue(rt_rq);
                 raw_spin_unlock(&rq->lock);
         }
 
+        if (!throttled && (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF))
+                return 1;
+
         return idle;
 }
 
@@ -855,8 +857,30 @@ static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq)
                 return 0;
 
         if (rt_rq->rt_time > runtime) {
-                rt_rq->rt_throttled = 1;
-                printk_once(KERN_WARNING "sched: RT throttling activated\n");
+                struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
+
+                /*
+                 * Don't actually throttle groups that have no runtime assigned
+                 * but accrue some time due to boosting.
+                 */
+                if (likely(rt_b->rt_runtime)) {
+                        static bool once = false;
+
+                        rt_rq->rt_throttled = 1;
+
+                        if (!once) {
+                                once = true;
+                                printk_sched("sched: RT throttling activated\n");
+                        }
+                } else {
+                        /*
+                         * In case we did anyway, make it go away,
+                         * replenishment is a joke, since it will replenish us
+                         * with exactly 0 ns.
+                         */
+                        rt_rq->rt_time = 0;
+                }
+
                 if (rt_rq_throttled(rt_rq)) {
                         sched_rt_rq_dequeue(rt_rq);
                         return 1;
@@ -884,7 +908,8 @@ static void update_curr_rt(struct rq *rq)
         if (unlikely((s64)delta_exec < 0))
                 delta_exec = 0;
 
-        schedstat_set(curr->se.statistics.exec_max, max(curr->se.statistics.exec_max, delta_exec));
+        schedstat_set(curr->se.statistics.exec_max,
+                      max(curr->se.statistics.exec_max, delta_exec));
 
         curr->se.sum_exec_runtime += delta_exec;
         account_group_exec_runtime(curr, delta_exec);
@@ -1403,7 +1428,7 @@ static struct task_struct *pick_next_highest_task_rt(struct rq *rq, int cpu)
 next_idx:
                 if (idx >= MAX_RT_PRIO)
                         continue;
-                if (next && next->prio < idx)
+                if (next && next->prio <= idx)
                         continue;
                 list_for_each_entry(rt_se, array->queue + idx, run_list) {
                         struct task_struct *p;
@@ -1972,7 +1997,7 @@ static void task_tick_rt(struct rq *rq, struct task_struct *p, int queued)
         if (--p->rt.time_slice)
                 return;
 
-        p->rt.time_slice = DEF_TIMESLICE;
+        p->rt.time_slice = RR_TIMESLICE;
 
         /*
          * Requeue to the end of queue if we are not the only element
@@ -2000,7 +2025,7 @@ static unsigned int get_rr_interval_rt(struct rq *rq, struct task_struct *task)
          * Time slice is 0 for SCHED_FIFO tasks
          */
         if (task->policy == SCHED_RR)
-                return DEF_TIMESLICE;
+                return RR_TIMESLICE;
         else
                 return 0;
 }
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 98c0c2623db8..fb3acba4d52e 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -36,11 +36,7 @@ extern __read_mostly int scheduler_running;
 
 /*
  * These are the 'tuning knobs' of the scheduler:
- *
- * default timeslice is 100 msecs (used only for SCHED_RR tasks).
- * Timeslices get refilled after they expire.
  */
-#define DEF_TIMESLICE           (100 * HZ / 1000)
 
 /*
  * single value that denotes runtime == period, ie unlimited time.
@@ -216,9 +212,6 @@ struct cfs_rq {
         struct rb_root tasks_timeline;
         struct rb_node *rb_leftmost;
 
-        struct list_head tasks;
-        struct list_head *balance_iterator;
-
         /*
          * 'curr' points to currently running entity on this cfs_rq.
          * It is set to NULL otherwise (i.e when none are currently running).
@@ -246,11 +239,6 @@ struct cfs_rq {
 
 #ifdef CONFIG_SMP
         /*
-         * the part of load.weight contributed by tasks
-         */
-        unsigned long task_weight;
-
-        /*
          *   h_load = weight * f(tg)
          *
          * Where f(tg) is the recursive weight fraction assigned to
@@ -424,6 +412,8 @@ struct rq {
         int cpu;
         int online;
 
+        struct list_head cfs_tasks;
+
         u64 rt_avg;
         u64 age_stamp;
         u64 idle_stamp;
@@ -462,7 +452,6 @@ struct rq {
         unsigned int yld_count;
 
         /* schedule() stats */
-        unsigned int sched_switch;
         unsigned int sched_count;
         unsigned int sched_goidle;
 
@@ -611,7 +600,7 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
  * Tunables that become constants when CONFIG_SCHED_DEBUG is off:
  */
 #ifdef CONFIG_SCHED_DEBUG
-# include <linux/jump_label.h>
+# include <linux/static_key.h>
 # define const_debug __read_mostly
 #else
 # define const_debug const
@@ -630,18 +619,18 @@ enum {
 #undef SCHED_FEAT
 
 #if defined(CONFIG_SCHED_DEBUG) && defined(HAVE_JUMP_LABEL)
-static __always_inline bool static_branch__true(struct jump_label_key *key)
+static __always_inline bool static_branch__true(struct static_key *key)
 {
-        return likely(static_branch(key)); /* Not out of line branch. */
+        return static_key_true(key); /* Not out of line branch. */
 }
 
-static __always_inline bool static_branch__false(struct jump_label_key *key)
+static __always_inline bool static_branch__false(struct static_key *key)
 {
-        return unlikely(static_branch(key)); /* Out of line branch. */
+        return static_key_false(key); /* Out of line branch. */
 }
 
 #define SCHED_FEAT(name, enabled)                                       \
-static __always_inline bool static_branch_##name(struct jump_label_key *key) \
+static __always_inline bool static_branch_##name(struct static_key *key) \
 {                                                                       \
         return static_branch__##enabled(key);                           \
 }
@@ -650,7 +639,7 @@ static __always_inline bool static_branch_##name(struct jump_label_key *key) \
 
 #undef SCHED_FEAT
 
-extern struct jump_label_key sched_feat_keys[__SCHED_FEAT_NR];
+extern struct static_key sched_feat_keys[__SCHED_FEAT_NR];
 #define sched_feat(x) (static_branch_##x(&sched_feat_keys[__SCHED_FEAT_##x]))
 #else /* !(SCHED_DEBUG && HAVE_JUMP_LABEL) */
 #define sched_feat(x) (sysctl_sched_features & (1UL << __SCHED_FEAT_##x))
@@ -692,6 +681,9 @@ static inline int task_running(struct rq *rq, struct task_struct *p)
 #ifndef finish_arch_switch
 # define finish_arch_switch(prev)       do { } while (0)
 #endif
+#ifndef finish_arch_post_lock_switch
+# define finish_arch_post_lock_switch() do { } while (0)
+#endif
 
 #ifndef __ARCH_WANT_UNLOCKED_CTXSW
 static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
diff --git a/kernel/sched/stats.c b/kernel/sched/stats.c
index 2a581ba8e190..903ffa9e8872 100644
--- a/kernel/sched/stats.c
+++ b/kernel/sched/stats.c
@@ -32,9 +32,9 @@ static int show_schedstat(struct seq_file *seq, void *v)
 
                 /* runqueue-specific stats */
                 seq_printf(seq,
-                    "cpu%d %u %u %u %u %u %u %llu %llu %lu",
+                    "cpu%d %u 0 %u %u %u %u %llu %llu %lu",
                     cpu, rq->yld_count,
-                    rq->sched_switch, rq->sched_count, rq->sched_goidle,
+                    rq->sched_count, rq->sched_goidle,
                     rq->ttwu_count, rq->ttwu_local,
                     rq->rq_cpu_time,
                     rq->rq_sched_info.run_delay, rq->rq_sched_info.pcount);