11 files changed, 703 insertions, 253 deletions

diff --git a/kernel/Makefile b/kernel/Makefile
index 1c9938addb9d..6c55301112e0 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -3,7 +3,7 @@
 #
 
 obj-y     = sched.o fork.o exec_domain.o panic.o printk.o profile.o \
-            exit.o itimer.o time.o softirq.o resource.o \
+            cpu.o exit.o itimer.o time.o softirq.o resource.o \
             sysctl.o capability.o ptrace.o timer.o user.o \
             signal.o sys.o kmod.o workqueue.o pid.o \
             rcupdate.o extable.o params.o posix-timers.o \
@@ -27,7 +27,7 @@ obj-$(CONFIG_RT_MUTEXES) += rtmutex.o
 obj-$(CONFIG_DEBUG_RT_MUTEXES) += rtmutex-debug.o
 obj-$(CONFIG_RT_MUTEX_TESTER) += rtmutex-tester.o
 obj-$(CONFIG_GENERIC_ISA_DMA) += dma.o
-obj-$(CONFIG_SMP) += cpu.o spinlock.o
+obj-$(CONFIG_SMP) += spinlock.o
 obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock.o
 obj-$(CONFIG_PROVE_LOCKING) += spinlock.o
 obj-$(CONFIG_UID16) += uid16.o
@@ -69,6 +69,7 @@ obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o
 obj-$(CONFIG_TASKSTATS) += taskstats.o tsacct.o
 obj-$(CONFIG_MARKERS) += marker.o
 obj-$(CONFIG_LATENCYTOP) += latencytop.o
+obj-$(CONFIG_SMP) += sched_cpupri.o
 
 ifneq ($(CONFIG_SCHED_NO_NO_OMIT_FRAME_POINTER),y)
 # According to Alan Modra <alan@linuxcare.com.au>, the -fno-omit-frame-pointer is
diff --git a/kernel/cpu.c b/kernel/cpu.c
index c77bc3a1c722..b11f06dc149a 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -15,6 +15,28 @@
 #include <linux/stop_machine.h>
 #include <linux/mutex.h>
 
+/*
+ * Represents all cpu's present in the system
+ * In systems capable of hotplug, this map could dynamically grow
+ * as new cpu's are detected in the system via any platform specific
+ * method, such as ACPI for e.g.
+ */
+cpumask_t cpu_present_map __read_mostly;
+EXPORT_SYMBOL(cpu_present_map);
+
+#ifndef CONFIG_SMP
+
+/*
+ * Represents all cpu's that are currently online.
+ */
+cpumask_t cpu_online_map __read_mostly = CPU_MASK_ALL;
+EXPORT_SYMBOL(cpu_online_map);
+
+cpumask_t cpu_possible_map __read_mostly = CPU_MASK_ALL;
+EXPORT_SYMBOL(cpu_possible_map);
+
+#else /* CONFIG_SMP */
+
 /* Serializes the updates to cpu_online_map, cpu_present_map */
 static DEFINE_MUTEX(cpu_add_remove_lock);
 
@@ -403,3 +425,5 @@ out:
         cpu_maps_update_done();
 }
 #endif /* CONFIG_PM_SLEEP_SMP */
+
+#endif /* CONFIG_SMP */
diff --git a/kernel/cpuset.c b/kernel/cpuset.c
index 9fceb97e989c..64a05da9bc4c 100644
--- a/kernel/cpuset.c
+++ b/kernel/cpuset.c
@@ -1194,6 +1194,15 @@ static int cpuset_can_attach(struct cgroup_subsys *ss,
 
         if (cpus_empty(cs->cpus_allowed) || nodes_empty(cs->mems_allowed))
                 return -ENOSPC;
+        if (tsk->flags & PF_THREAD_BOUND) {
+                cpumask_t mask;
+
+                mutex_lock(&callback_mutex);
+                mask = cs->cpus_allowed;
+                mutex_unlock(&callback_mutex);
+                if (!cpus_equal(tsk->cpus_allowed, mask))
+                        return -EINVAL;
+        }
 
         return security_task_setscheduler(tsk, 0, NULL);
 }
@@ -1207,11 +1216,14 @@ static void cpuset_attach(struct cgroup_subsys *ss,
         struct mm_struct *mm;
         struct cpuset *cs = cgroup_cs(cont);
         struct cpuset *oldcs = cgroup_cs(oldcont);
+        int err;
 
         mutex_lock(&callback_mutex);
         guarantee_online_cpus(cs, &cpus);
-        set_cpus_allowed_ptr(tsk, &cpus);
+        err = set_cpus_allowed_ptr(tsk, &cpus);
         mutex_unlock(&callback_mutex);
+        if (err)
+                return;
 
         from = oldcs->mems_allowed;
         to = cs->mems_allowed;
diff --git a/kernel/kthread.c b/kernel/kthread.c
index bd1b9ea024e1..97747cdd37c9 100644
--- a/kernel/kthread.c
+++ b/kernel/kthread.c
@@ -180,6 +180,7 @@ void kthread_bind(struct task_struct *k, unsigned int cpu)
         set_task_cpu(k, cpu);
         k->cpus_allowed = cpumask_of_cpu(cpu);
         k->rt.nr_cpus_allowed = 1;
+        k->flags |= PF_THREAD_BOUND;
 }
 EXPORT_SYMBOL(kthread_bind);
 
diff --git a/kernel/sched.c b/kernel/sched.c
index b048ad8a11af..adb2d01fccc2 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -74,6 +74,8 @@
 #include <asm/tlb.h>
 #include <asm/irq_regs.h>
 
+#include "sched_cpupri.h"
+
 /*
  * Convert user-nice values [ -20 ... 0 ... 19 ]
  * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ],
@@ -289,15 +291,15 @@ struct task_group root_task_group;
 static DEFINE_PER_CPU(struct sched_entity, init_sched_entity);
 /* Default task group's cfs_rq on each cpu */
 static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp;
-#endif
+#endif /* CONFIG_FAIR_GROUP_SCHED */
 
 #ifdef CONFIG_RT_GROUP_SCHED
 static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity);
 static DEFINE_PER_CPU(struct rt_rq, init_rt_rq) ____cacheline_aligned_in_smp;
-#endif
-#else
+#endif /* CONFIG_RT_GROUP_SCHED */
+#else /* !CONFIG_FAIR_GROUP_SCHED */
 #define root_task_group init_task_group
-#endif
+#endif /* CONFIG_FAIR_GROUP_SCHED */
 
 /* task_group_lock serializes add/remove of task groups and also changes to
  * a task group's cpu shares.
@@ -307,9 +309,9 @@ static DEFINE_SPINLOCK(task_group_lock);
 #ifdef CONFIG_FAIR_GROUP_SCHED
 #ifdef CONFIG_USER_SCHED
 # define INIT_TASK_GROUP_LOAD   (2*NICE_0_LOAD)
-#else
+#else /* !CONFIG_USER_SCHED */
 # define INIT_TASK_GROUP_LOAD   NICE_0_LOAD
-#endif
+#endif /* CONFIG_USER_SCHED */
 
 /*
  * A weight of 0 or 1 can cause arithmetics problems.
@@ -452,6 +454,9 @@ struct root_domain {
          */
         cpumask_t rto_mask;
         atomic_t rto_count;
+#ifdef CONFIG_SMP
+        struct cpupri cpupri;
+#endif
 };
 
 /*
@@ -526,6 +531,7 @@ struct rq {
         int push_cpu;
         /* cpu of this runqueue: */
         int cpu;
+        int online;
 
         struct task_struct *migration_thread;
         struct list_head migration_queue;
@@ -1313,15 +1319,15 @@ void wake_up_idle_cpu(int cpu)
         if (!tsk_is_polling(rq->idle))
                 smp_send_reschedule(cpu);
 }
-#endif
+#endif /* CONFIG_NO_HZ */
 
-#else
+#else /* !CONFIG_SMP */
 static void __resched_task(struct task_struct *p, int tif_bit)
 {
         assert_spin_locked(&task_rq(p)->lock);
         set_tsk_thread_flag(p, tif_bit);
 }
-#endif
+#endif /* CONFIG_SMP */
 
 #if BITS_PER_LONG == 32
 # define WMULT_CONST    (~0UL)
@@ -1481,16 +1487,8 @@ static unsigned long source_load(int cpu, int type);
 static unsigned long target_load(int cpu, int type);
 static unsigned long cpu_avg_load_per_task(int cpu);
 static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd);
-#else /* CONFIG_SMP */
-
-#ifdef CONFIG_FAIR_GROUP_SCHED
-static void cfs_rq_set_shares(struct cfs_rq *cfs_rq, unsigned long shares)
-{
-}
 #endif
 
-#endif /* CONFIG_SMP */
-
 #include "sched_stats.h"
 #include "sched_idletask.c"
 #include "sched_fair.c"
@@ -1500,6 +1498,8 @@ static void cfs_rq_set_shares(struct cfs_rq *cfs_rq, unsigned long shares)
 #endif
 
 #define sched_class_highest (&rt_sched_class)
+#define for_each_class(class) \
+   for (class = sched_class_highest; class; class = class->next)
 
 static inline void inc_load(struct rq *rq, const struct task_struct *p)
 {
@@ -1636,12 +1636,6 @@ inline int task_curr(const struct task_struct *p)
         return cpu_curr(task_cpu(p)) == p;
 }
 
-/* Used instead of source_load when we know the type == 0 */
-unsigned long weighted_cpuload(const int cpu)
-{
-        return cpu_rq(cpu)->load.weight;
-}
-
 static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
 {
         set_task_rq(p, cpu);
@@ -1670,6 +1664,12 @@ static inline void check_class_changed(struct rq *rq, struct task_struct *p,
 
 #ifdef CONFIG_SMP
 
+/* Used instead of source_load when we know the type == 0 */
+static unsigned long weighted_cpuload(const int cpu)
+{
+        return cpu_rq(cpu)->load.weight;
+}
+
 /*
  * Is this task likely cache-hot:
  */
@@ -2131,7 +2131,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
                         }
                 }
         }
-#endif
+#endif /* CONFIG_SCHEDSTATS */
 
 out_activate:
 #endif /* CONFIG_SMP */
@@ -2331,7 +2331,7 @@ fire_sched_out_preempt_notifiers(struct task_struct *curr,
                 notifier->ops->sched_out(notifier, next);
 }
 
-#else
+#else /* !CONFIG_PREEMPT_NOTIFIERS */
 
 static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
 {
@@ -2343,7 +2343,7 @@ fire_sched_out_preempt_notifiers(struct task_struct *curr,
 {
 }
 
-#endif
+#endif /* CONFIG_PREEMPT_NOTIFIERS */
 
 /**
  * prepare_task_switch - prepare to switch tasks
@@ -3672,6 +3672,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
         /* Earliest time when we have to do rebalance again */
         unsigned long next_balance = jiffies + 60*HZ;
         int update_next_balance = 0;
+        int need_serialize;
         cpumask_t tmp;
 
         for_each_domain(cpu, sd) {
@@ -3689,8 +3690,9 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
                 if (interval > HZ*NR_CPUS/10)
                         interval = HZ*NR_CPUS/10;
 
+                need_serialize = sd->flags & SD_SERIALIZE;
 
-                if (sd->flags & SD_SERIALIZE) {
+                if (need_serialize) {
                         if (!spin_trylock(&balancing))
                                 goto out;
                 }
@@ -3706,7 +3708,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
                         }
                         sd->last_balance = jiffies;
                 }
-                if (sd->flags & SD_SERIALIZE)
+                if (need_serialize)
                         spin_unlock(&balancing);
 out:
                 if (time_after(next_balance, sd->last_balance + interval)) {
@@ -4070,6 +4072,7 @@ static noinline void __schedule_bug(struct task_struct *prev)
                 prev->comm, prev->pid, preempt_count());
 
         debug_show_held_locks(prev);
+        print_modules();
         if (irqs_disabled())
                 print_irqtrace_events(prev);
 
@@ -4143,7 +4146,7 @@ asmlinkage void __sched schedule(void)
         struct task_struct *prev, *next;
         unsigned long *switch_count;
         struct rq *rq;
-        int cpu;
+        int cpu, hrtick = sched_feat(HRTICK);
 
 need_resched:
         preempt_disable();
@@ -4158,7 +4161,8 @@ need_resched_nonpreemptible:
 
         schedule_debug(prev);
 
-        hrtick_clear(rq);
+        if (hrtick)
+                hrtick_clear(rq);
 
         /*
          * Do the rq-clock update outside the rq lock:
@@ -4204,7 +4208,8 @@ need_resched_nonpreemptible:
         } else
                 spin_unlock_irq(&rq->lock);
 
-        hrtick_set(rq);
+        if (hrtick)
+                hrtick_set(rq);
 
         if (unlikely(reacquire_kernel_lock(current) < 0))
                 goto need_resched_nonpreemptible;
@@ -5072,24 +5077,6 @@ asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len,
         return sched_setaffinity(pid, &new_mask);
 }
 
-/*
- * Represents all cpu's present in the system
- * In systems capable of hotplug, this map could dynamically grow
- * as new cpu's are detected in the system via any platform specific
- * method, such as ACPI for e.g.
- */
-
-cpumask_t cpu_present_map __read_mostly;
-EXPORT_SYMBOL(cpu_present_map);
-
-#ifndef CONFIG_SMP
-cpumask_t cpu_online_map __read_mostly = CPU_MASK_ALL;
-EXPORT_SYMBOL(cpu_online_map);
-
-cpumask_t cpu_possible_map __read_mostly = CPU_MASK_ALL;
-EXPORT_SYMBOL(cpu_possible_map);
-#endif
-
 long sched_getaffinity(pid_t pid, cpumask_t *mask)
 {
         struct task_struct *p;
@@ -5573,6 +5560,12 @@ int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask)
                 goto out;
         }
 
+        if (unlikely((p->flags & PF_THREAD_BOUND) && p != current &&
+                     !cpus_equal(p->cpus_allowed, *new_mask))) {
+                ret = -EINVAL;
+                goto out;
+        }
+
         if (p->sched_class->set_cpus_allowed)
                 p->sched_class->set_cpus_allowed(p, new_mask);
         else {
@@ -6060,6 +6053,36 @@ static void unregister_sched_domain_sysctl(void)
 }
 #endif
 
+static void set_rq_online(struct rq *rq)
+{
+        if (!rq->online) {
+                const struct sched_class *class;
+
+                cpu_set(rq->cpu, rq->rd->online);
+                rq->online = 1;
+
+                for_each_class(class) {
+                        if (class->rq_online)
+                                class->rq_online(rq);
+                }
+        }
+}
+
+static void set_rq_offline(struct rq *rq)
+{
+        if (rq->online) {
+                const struct sched_class *class;
+
+                for_each_class(class) {
+                        if (class->rq_offline)
+                                class->rq_offline(rq);
+                }
+
+                cpu_clear(rq->cpu, rq->rd->online);
+                rq->online = 0;
+        }
+}
+
 /*
  * migration_call - callback that gets triggered when a CPU is added.
  * Here we can start up the necessary migration thread for the new CPU.
@@ -6097,7 +6120,8 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
                 spin_lock_irqsave(&rq->lock, flags);
                 if (rq->rd) {
                         BUG_ON(!cpu_isset(cpu, rq->rd->span));
-                        cpu_set(cpu, rq->rd->online);
+
+                        set_rq_online(rq);
                 }
                 spin_unlock_irqrestore(&rq->lock, flags);
                 break;
@@ -6158,7 +6182,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
                 spin_lock_irqsave(&rq->lock, flags);
                 if (rq->rd) {
                         BUG_ON(!cpu_isset(cpu, rq->rd->span));
-                        cpu_clear(cpu, rq->rd->online);
+                        set_rq_offline(rq);
                 }
                 spin_unlock_irqrestore(&rq->lock, flags);
                 break;
@@ -6192,6 +6216,28 @@ void __init migration_init(void)
 
 #ifdef CONFIG_SCHED_DEBUG
 
+static inline const char *sd_level_to_string(enum sched_domain_level lvl)
+{
+        switch (lvl) {
+        case SD_LV_NONE:
+                        return "NONE";
+        case SD_LV_SIBLING:
+                        return "SIBLING";
+        case SD_LV_MC:
+                        return "MC";
+        case SD_LV_CPU:
+                        return "CPU";
+        case SD_LV_NODE:
+                        return "NODE";
+        case SD_LV_ALLNODES:
+                        return "ALLNODES";
+        case SD_LV_MAX:
+                        return "MAX";
+
+        }
+        return "MAX";
+}
+
 static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
                                   cpumask_t *groupmask)
 {
@@ -6211,7 +6257,8 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
                 return -1;
         }
 
-        printk(KERN_CONT "span %s\n", str);
+        printk(KERN_CONT "span %s level %s\n",
+                str, sd_level_to_string(sd->level));
 
         if (!cpu_isset(cpu, sd->span)) {
                 printk(KERN_ERR "ERROR: domain->span does not contain "
@@ -6295,9 +6342,9 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
         }
         kfree(groupmask);
 }
-#else
+#else /* !CONFIG_SCHED_DEBUG */
 # define sched_domain_debug(sd, cpu) do { } while (0)
-#endif
+#endif /* CONFIG_SCHED_DEBUG */
 
 static int sd_degenerate(struct sched_domain *sd)
 {
@@ -6357,20 +6404,16 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
 static void rq_attach_root(struct rq *rq, struct root_domain *rd)
 {
         unsigned long flags;
-        const struct sched_class *class;
 
         spin_lock_irqsave(&rq->lock, flags);
 
         if (rq->rd) {
                 struct root_domain *old_rd = rq->rd;
 
-                for (class = sched_class_highest; class; class = class->next) {
-                        if (class->leave_domain)
-                                class->leave_domain(rq);
-                }
+                if (cpu_isset(rq->cpu, old_rd->online))
+                        set_rq_offline(rq);
 
                 cpu_clear(rq->cpu, old_rd->span);
-                cpu_clear(rq->cpu, old_rd->online);
 
                 if (atomic_dec_and_test(&old_rd->refcount))
                         kfree(old_rd);
@@ -6381,12 +6424,7 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd)
 
         cpu_set(rq->cpu, rd->span);
         if (cpu_isset(rq->cpu, cpu_online_map))
-                cpu_set(rq->cpu, rd->online);
-
-        for (class = sched_class_highest; class; class = class->next) {
-                if (class->join_domain)
-                        class->join_domain(rq);
-        }
+                set_rq_online(rq);
 
         spin_unlock_irqrestore(&rq->lock, flags);
 }
@@ -6397,6 +6435,8 @@ static void init_rootdomain(struct root_domain *rd)
 
         cpus_clear(rd->span);
         cpus_clear(rd->online);
+
+        cpupri_init(&rd->cpupri);
 }
 
 static void init_defrootdomain(void)
@@ -6591,7 +6631,7 @@ static void sched_domain_node_span(int node, cpumask_t *span)
                 cpus_or(*span, *span, *nodemask);
         }
 }
-#endif
+#endif /* CONFIG_NUMA */
 
 int sched_smt_power_savings = 0, sched_mc_power_savings = 0;
 
@@ -6610,7 +6650,7 @@ cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
                 *sg = &per_cpu(sched_group_cpus, cpu);
         return cpu;
 }
-#endif
+#endif /* CONFIG_SCHED_SMT */
 
 /*
  * multi-core sched-domains:
@@ -6618,7 +6658,7 @@ cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
 #ifdef CONFIG_SCHED_MC
 static DEFINE_PER_CPU(struct sched_domain, core_domains);
 static DEFINE_PER_CPU(struct sched_group, sched_group_core);
-#endif
+#endif /* CONFIG_SCHED_MC */
 
 #if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT)
 static int
@@ -6720,7 +6760,7 @@ static void init_numa_sched_groups_power(struct sched_group *group_head)
                 sg = sg->next;
         } while (sg != group_head);
 }
-#endif
+#endif /* CONFIG_NUMA */
 
 #ifdef CONFIG_NUMA
 /* Free memory allocated for various sched_group structures */
@@ -6757,11 +6797,11 @@ next_sg:
                 sched_group_nodes_bycpu[cpu] = NULL;
         }
 }
-#else
+#else /* !CONFIG_NUMA */
 static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask)
 {
 }
-#endif
+#endif /* CONFIG_NUMA */
 
 /*
  * Initialize sched groups cpu_power.
@@ -7470,7 +7510,7 @@ int sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
 #endif
         return err;
 }
-#endif
+#endif /* CONFIG_SCHED_MC || CONFIG_SCHED_SMT */
 
 /*
  * Force a reinitialization of the sched domains hierarchy. The domains
@@ -7481,21 +7521,28 @@ int sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
 static int update_sched_domains(struct notifier_block *nfb,
                                 unsigned long action, void *hcpu)
 {
+        int cpu = (int)(long)hcpu;
+
         switch (action) {
-        case CPU_UP_PREPARE:
-        case CPU_UP_PREPARE_FROZEN:
         case CPU_DOWN_PREPARE:
         case CPU_DOWN_PREPARE_FROZEN:
+                disable_runtime(cpu_rq(cpu));
+                /* fall-through */
+        case CPU_UP_PREPARE:
+        case CPU_UP_PREPARE_FROZEN:
                 detach_destroy_domains(&cpu_online_map);
                 free_sched_domains();
                 return NOTIFY_OK;
 
-        case CPU_UP_CANCELED:
-        case CPU_UP_CANCELED_FROZEN:
+
         case CPU_DOWN_FAILED:
         case CPU_DOWN_FAILED_FROZEN:
         case CPU_ONLINE:
         case CPU_ONLINE_FROZEN:
+                enable_runtime(cpu_rq(cpu));
+                /* fall-through */
+        case CPU_UP_CANCELED:
+        case CPU_UP_CANCELED_FROZEN:
         case CPU_DEAD:
         case CPU_DEAD_FROZEN:
                 /*
@@ -7695,8 +7742,8 @@ void __init sched_init(void)
 
                 root_task_group.cfs_rq = (struct cfs_rq **)ptr;
                 ptr += nr_cpu_ids * sizeof(void **);
-#endif
-#endif
+#endif /* CONFIG_USER_SCHED */
+#endif /* CONFIG_FAIR_GROUP_SCHED */
 #ifdef CONFIG_RT_GROUP_SCHED
                 init_task_group.rt_se = (struct sched_rt_entity **)ptr;
                 ptr += nr_cpu_ids * sizeof(void **);
@@ -7710,8 +7757,8 @@ void __init sched_init(void)
 
                 root_task_group.rt_rq = (struct rt_rq **)ptr;
                 ptr += nr_cpu_ids * sizeof(void **);
-#endif
-#endif
+#endif /* CONFIG_USER_SCHED */
+#endif /* CONFIG_RT_GROUP_SCHED */
         }
 
 #ifdef CONFIG_SMP
@@ -7727,8 +7774,8 @@ void __init sched_init(void)
 #ifdef CONFIG_USER_SCHED
         init_rt_bandwidth(&root_task_group.rt_bandwidth,
                         global_rt_period(), RUNTIME_INF);
-#endif
-#endif
+#endif /* CONFIG_USER_SCHED */
+#endif /* CONFIG_RT_GROUP_SCHED */
 
 #ifdef CONFIG_GROUP_SCHED
         list_add(&init_task_group.list, &task_groups);
@@ -7738,8 +7785,8 @@ void __init sched_init(void)
         INIT_LIST_HEAD(&root_task_group.children);
         init_task_group.parent = &root_task_group;
         list_add(&init_task_group.siblings, &root_task_group.children);
-#endif
-#endif
+#endif /* CONFIG_USER_SCHED */
+#endif /* CONFIG_GROUP_SCHED */
 
         for_each_possible_cpu(i) {
                 struct rq *rq;
@@ -7819,6 +7866,7 @@ void __init sched_init(void)
                 rq->next_balance = jiffies;
                 rq->push_cpu = 0;
                 rq->cpu = i;
+                rq->online = 0;
                 rq->migration_thread = NULL;
                 INIT_LIST_HEAD(&rq->migration_queue);
                 rq_attach_root(rq, &def_root_domain);
@@ -8058,7 +8106,7 @@ static inline void unregister_fair_sched_group(struct task_group *tg, int cpu)
 {
         list_del_rcu(&tg->cfs_rq[cpu]->leaf_cfs_rq_list);
 }
-#else
+#else /* !CONFG_FAIR_GROUP_SCHED */
 static inline void free_fair_sched_group(struct task_group *tg)
 {
 }
@@ -8076,7 +8124,7 @@ static inline void register_fair_sched_group(struct task_group *tg, int cpu)
 static inline void unregister_fair_sched_group(struct task_group *tg, int cpu)
 {
 }
-#endif
+#endif /* CONFIG_FAIR_GROUP_SCHED */
 
 #ifdef CONFIG_RT_GROUP_SCHED
 static void free_rt_sched_group(struct task_group *tg)
@@ -8147,7 +8195,7 @@ static inline void unregister_rt_sched_group(struct task_group *tg, int cpu)
 {
         list_del_rcu(&tg->rt_rq[cpu]->leaf_rt_rq_list);
 }
-#else
+#else /* !CONFIG_RT_GROUP_SCHED */
 static inline void free_rt_sched_group(struct task_group *tg)
 {
 }
@@ -8165,7 +8213,7 @@ static inline void register_rt_sched_group(struct task_group *tg, int cpu)
 static inline void unregister_rt_sched_group(struct task_group *tg, int cpu)
 {
 }
-#endif
+#endif /* CONFIG_RT_GROUP_SCHED */
 
 #ifdef CONFIG_GROUP_SCHED
 static void free_sched_group(struct task_group *tg)
@@ -8276,7 +8324,7 @@ void sched_move_task(struct task_struct *tsk)
 
         task_rq_unlock(rq, &flags);
 }
-#endif
+#endif /* CONFIG_GROUP_SCHED */
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
 static void set_se_shares(struct sched_entity *se, unsigned long shares)
@@ -8376,7 +8424,7 @@ static unsigned long to_ratio(u64 period, u64 runtime)
 #ifdef CONFIG_CGROUP_SCHED
 static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
 {
-        struct task_group *tgi, *parent = tg ? tg->parent : NULL;
+        struct task_group *tgi, *parent = tg->parent;
         unsigned long total = 0;
 
         if (!parent) {
@@ -8400,7 +8448,7 @@ static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
         }
         rcu_read_unlock();
 
-        return total + to_ratio(period, runtime) <
+        return total + to_ratio(period, runtime) <=
                 to_ratio(ktime_to_ns(parent->rt_bandwidth.rt_period),
                                 parent->rt_bandwidth.rt_runtime);
 }
@@ -8517,16 +8565,21 @@ long sched_group_rt_period(struct task_group *tg)
 
 static int sched_rt_global_constraints(void)
 {
+        struct task_group *tg = &root_task_group;
+        u64 rt_runtime, rt_period;
         int ret = 0;
 
+        rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
+        rt_runtime = tg->rt_bandwidth.rt_runtime;
+
         mutex_lock(&rt_constraints_mutex);
-        if (!__rt_schedulable(NULL, 1, 0))
+        if (!__rt_schedulable(tg, rt_period, rt_runtime))
                 ret = -EINVAL;
         mutex_unlock(&rt_constraints_mutex);
 
         return ret;
 }
-#else
+#else /* !CONFIG_RT_GROUP_SCHED */
 static int sched_rt_global_constraints(void)
 {
         unsigned long flags;
@@ -8544,7 +8597,7 @@ static int sched_rt_global_constraints(void)
 
         return 0;
 }
-#endif
+#endif /* CONFIG_RT_GROUP_SCHED */
 
 int sched_rt_handler(struct ctl_table *table, int write,
                 struct file *filp, void __user *buffer, size_t *lenp,
@@ -8652,7 +8705,7 @@ static u64 cpu_shares_read_u64(struct cgroup *cgrp, struct cftype *cft)
 
         return (u64) tg->shares;
 }
-#endif
+#endif /* CONFIG_FAIR_GROUP_SCHED */
 
 #ifdef CONFIG_RT_GROUP_SCHED
 static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft,
@@ -8676,7 +8729,7 @@ static u64 cpu_rt_period_read_uint(struct cgroup *cgrp, struct cftype *cft)
 {
         return sched_group_rt_period(cgroup_tg(cgrp));
 }
-#endif
+#endif /* CONFIG_RT_GROUP_SCHED */
 
 static struct cftype cpu_files[] = {
 #ifdef CONFIG_FAIR_GROUP_SCHED
diff --git a/kernel/sched_cpupri.c b/kernel/sched_cpupri.c
new file mode 100644
index 000000000000..52154fefab7e
--- /dev/null
+++ b/kernel/sched_cpupri.c
@@ -0,0 +1,174 @@
+/*
+ *  kernel/sched_cpupri.c
+ *
+ *  CPU priority management
+ *
+ *  Copyright (C) 2007-2008 Novell
+ *
+ *  Author: Gregory Haskins <ghaskins@novell.com>
+ *
+ *  This code tracks the priority of each CPU so that global migration
+ *  decisions are easy to calculate.  Each CPU can be in a state as follows:
+ *
+ *                 (INVALID), IDLE, NORMAL, RT1, ... RT99
+ *
+ *  going from the lowest priority to the highest.  CPUs in the INVALID state
+ *  are not eligible for routing.  The system maintains this state with
+ *  a 2 dimensional bitmap (the first for priority class, the second for cpus
+ *  in that class).  Therefore a typical application without affinity
+ *  restrictions can find a suitable CPU with O(1) complexity (e.g. two bit
+ *  searches).  For tasks with affinity restrictions, the algorithm has a
+ *  worst case complexity of O(min(102, nr_domcpus)), though the scenario that
+ *  yields the worst case search is fairly contrived.
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; version 2
+ *  of the License.
+ */
+
+#include "sched_cpupri.h"
+
+/* Convert between a 140 based task->prio, and our 102 based cpupri */
+static int convert_prio(int prio)
+{
+        int cpupri;
+
+        if (prio == CPUPRI_INVALID)
+                cpupri = CPUPRI_INVALID;
+        else if (prio == MAX_PRIO)
+                cpupri = CPUPRI_IDLE;
+        else if (prio >= MAX_RT_PRIO)
+                cpupri = CPUPRI_NORMAL;
+        else
+                cpupri = MAX_RT_PRIO - prio + 1;
+
+        return cpupri;
+}
+
+#define for_each_cpupri_active(array, idx)                    \
+  for (idx = find_first_bit(array, CPUPRI_NR_PRIORITIES);     \
+       idx < CPUPRI_NR_PRIORITIES;                            \
+       idx = find_next_bit(array, CPUPRI_NR_PRIORITIES, idx+1))
+
+/**
+ * cpupri_find - find the best (lowest-pri) CPU in the system
+ * @cp: The cpupri context
+ * @p: The task
+ * @lowest_mask: A mask to fill in with selected CPUs
+ *
+ * Note: This function returns the recommended CPUs as calculated during the
+ * current invokation.  By the time the call returns, the CPUs may have in
+ * fact changed priorities any number of times.  While not ideal, it is not
+ * an issue of correctness since the normal rebalancer logic will correct
+ * any discrepancies created by racing against the uncertainty of the current
+ * priority configuration.
+ *
+ * Returns: (int)bool - CPUs were found
+ */
+int cpupri_find(struct cpupri *cp, struct task_struct *p,
+                cpumask_t *lowest_mask)
+{
+        int                  idx      = 0;
+        int                  task_pri = convert_prio(p->prio);
+
+        for_each_cpupri_active(cp->pri_active, idx) {
+                struct cpupri_vec *vec  = &cp->pri_to_cpu[idx];
+                cpumask_t mask;
+
+                if (idx >= task_pri)
+                        break;
+
+                cpus_and(mask, p->cpus_allowed, vec->mask);
+
+                if (cpus_empty(mask))
+                        continue;
+
+                *lowest_mask = mask;
+                return 1;
+        }
+
+        return 0;
+}
+
+/**
+ * cpupri_set - update the cpu priority setting
+ * @cp: The cpupri context
+ * @cpu: The target cpu
+ * @pri: The priority (INVALID-RT99) to assign to this CPU
+ *
+ * Note: Assumes cpu_rq(cpu)->lock is locked
+ *
+ * Returns: (void)
+ */
+void cpupri_set(struct cpupri *cp, int cpu, int newpri)
+{
+        int                 *currpri = &cp->cpu_to_pri[cpu];
+        int                  oldpri  = *currpri;
+        unsigned long        flags;
+
+        newpri = convert_prio(newpri);
+
+        BUG_ON(newpri >= CPUPRI_NR_PRIORITIES);
+
+        if (newpri == oldpri)
+                return;
+
+        /*
+         * If the cpu was currently mapped to a different value, we
+         * first need to unmap the old value
+         */
+        if (likely(oldpri != CPUPRI_INVALID)) {
+                struct cpupri_vec *vec  = &cp->pri_to_cpu[oldpri];
+
+                spin_lock_irqsave(&vec->lock, flags);
+
+                vec->count--;
+                if (!vec->count)
+                        clear_bit(oldpri, cp->pri_active);
+                cpu_clear(cpu, vec->mask);
+
+                spin_unlock_irqrestore(&vec->lock, flags);
+        }
+
+        if (likely(newpri != CPUPRI_INVALID)) {
+                struct cpupri_vec *vec = &cp->pri_to_cpu[newpri];
+
+                spin_lock_irqsave(&vec->lock, flags);
+
+                cpu_set(cpu, vec->mask);
+                vec->count++;
+                if (vec->count == 1)
+                        set_bit(newpri, cp->pri_active);
+
+                spin_unlock_irqrestore(&vec->lock, flags);
+        }
+
+        *currpri = newpri;
+}
+
+/**
+ * cpupri_init - initialize the cpupri structure
+ * @cp: The cpupri context
+ *
+ * Returns: (void)
+ */
+void cpupri_init(struct cpupri *cp)
+{
+        int i;
+
+        memset(cp, 0, sizeof(*cp));
+
+        for (i = 0; i < CPUPRI_NR_PRIORITIES; i++) {
+                struct cpupri_vec *vec = &cp->pri_to_cpu[i];
+
+                spin_lock_init(&vec->lock);
+                vec->count = 0;
+                cpus_clear(vec->mask);
+        }
+
+        for_each_possible_cpu(i)
+                cp->cpu_to_pri[i] = CPUPRI_INVALID;
+}
+
+
diff --git a/kernel/sched_cpupri.h b/kernel/sched_cpupri.h
new file mode 100644
index 000000000000..f25811b0f931
--- /dev/null
+++ b/kernel/sched_cpupri.h
@@ -0,0 +1,36 @@
+#ifndef _LINUX_CPUPRI_H
+#define _LINUX_CPUPRI_H
+
+#include <linux/sched.h>
+
+#define CPUPRI_NR_PRIORITIES    (MAX_RT_PRIO + 2)
+#define CPUPRI_NR_PRI_WORDS     BITS_TO_LONGS(CPUPRI_NR_PRIORITIES)
+
+#define CPUPRI_INVALID -1
+#define CPUPRI_IDLE     0
+#define CPUPRI_NORMAL   1
+/* values 2-101 are RT priorities 0-99 */
+
+struct cpupri_vec {
+        spinlock_t lock;
+        int        count;
+        cpumask_t  mask;
+};
+
+struct cpupri {
+        struct cpupri_vec pri_to_cpu[CPUPRI_NR_PRIORITIES];
+        long              pri_active[CPUPRI_NR_PRI_WORDS];
+        int               cpu_to_pri[NR_CPUS];
+};
+
+#ifdef CONFIG_SMP
+int  cpupri_find(struct cpupri *cp,
+                 struct task_struct *p, cpumask_t *lowest_mask);
+void cpupri_set(struct cpupri *cp, int cpu, int pri);
+void cpupri_init(struct cpupri *cp);
+#else
+#define cpupri_set(cp, cpu, pri) do { } while (0)
+#define cpupri_init() do { } while (0)
+#endif
+
+#endif /* _LINUX_CPUPRI_H */
diff --git a/kernel/sched_debug.c b/kernel/sched_debug.c
index 8bb713040ac9..8e077b9c91cb 100644
--- a/kernel/sched_debug.c
+++ b/kernel/sched_debug.c
@@ -119,9 +119,7 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
         struct sched_entity *last;
         unsigned long flags;
 
-#if !defined(CONFIG_CGROUP_SCHED) || !defined(CONFIG_USER_SCHED)
-        SEQ_printf(m, "\ncfs_rq[%d]:\n", cpu);
-#else
+#if defined(CONFIG_CGROUP_SCHED) && defined(CONFIG_FAIR_GROUP_SCHED)
         char path[128] = "";
         struct cgroup *cgroup = NULL;
         struct task_group *tg = cfs_rq->tg;
@@ -133,6 +131,8 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
                 cgroup_path(cgroup, path, sizeof(path));
 
         SEQ_printf(m, "\ncfs_rq[%d]:%s\n", cpu, path);
+#else
+        SEQ_printf(m, "\ncfs_rq[%d]:\n", cpu);
 #endif
 
         SEQ_printf(m, "  .%-30s: %Ld.%06ld\n", "exec_clock",
@@ -169,6 +169,39 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
                         cfs_rq->nr_spread_over);
 }
 
+void print_rt_rq(struct seq_file *m, int cpu, struct rt_rq *rt_rq)
+{
+#if defined(CONFIG_CGROUP_SCHED) && defined(CONFIG_RT_GROUP_SCHED)
+        char path[128] = "";
+        struct cgroup *cgroup = NULL;
+        struct task_group *tg = rt_rq->tg;
+
+        if (tg)
+                cgroup = tg->css.cgroup;
+
+        if (cgroup)
+                cgroup_path(cgroup, path, sizeof(path));
+
+        SEQ_printf(m, "\nrt_rq[%d]:%s\n", cpu, path);
+#else
+        SEQ_printf(m, "\nrt_rq[%d]:\n", cpu);
+#endif
+
+
+#define P(x) \
+        SEQ_printf(m, "  .%-30s: %Ld\n", #x, (long long)(rt_rq->x))
+#define PN(x) \
+        SEQ_printf(m, "  .%-30s: %Ld.%06ld\n", #x, SPLIT_NS(rt_rq->x))
+
+        P(rt_nr_running);
+        P(rt_throttled);
+        PN(rt_time);
+        PN(rt_runtime);
+
+#undef PN
+#undef P
+}
+
 static void print_cpu(struct seq_file *m, int cpu)
 {
         struct rq *rq = &per_cpu(runqueues, cpu);
@@ -208,6 +241,7 @@ static void print_cpu(struct seq_file *m, int cpu)
 #undef PN
 
         print_cfs_stats(m, cpu);
+        print_rt_stats(m, cpu);
 
         print_rq(m, rq, cpu);
 }
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index 08ae848b71d4..1fe4c65a8170 100644
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -1275,23 +1275,18 @@ __load_balance_iterator(struct cfs_rq *cfs_rq, struct list_head *next)
         struct task_struct *p = NULL;
         struct sched_entity *se;
 
-        if (next == &cfs_rq->tasks)
-                return NULL;
-
-        /* Skip over entities that are not tasks */
-        do {
+        while (next != &cfs_rq->tasks) {
                 se = list_entry(next, struct sched_entity, group_node);
                 next = next->next;
-        } while (next != &cfs_rq->tasks && !entity_is_task(se));
 
-        if (next == &cfs_rq->tasks)
-                return NULL;
+                /* Skip over entities that are not tasks */
+                if (entity_is_task(se)) {
+                        p = task_of(se);
+                        break;
+                }
+        }
 
         cfs_rq->balance_iterator = next;
-
-        if (entity_is_task(se))
-                p = task_of(se);
-
         return p;
 }
 
diff --git a/kernel/sched_features.h b/kernel/sched_features.h
index 1c7283cb9581..62b39ca92ebd 100644
--- a/kernel/sched_features.h
+++ b/kernel/sched_features.h
@@ -6,5 +6,3 @@ SCHED_FEAT(CACHE_HOT_BUDDY, 1)
 SCHED_FEAT(SYNC_WAKEUPS, 1)
 SCHED_FEAT(HRTICK, 1)
 SCHED_FEAT(DOUBLE_TICK, 0)
-SCHED_FEAT(NORMALIZED_SLEEPER, 1)
-SCHED_FEAT(DEADLINE, 1)
diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c
index 1dad5bbb59b6..bd90c8bb0739 100644
--- a/kernel/sched_rt.c
+++ b/kernel/sched_rt.c
@@ -12,6 +12,9 @@ static inline int rt_overloaded(struct rq *rq)
 
 static inline void rt_set_overload(struct rq *rq)
 {
+        if (!rq->online)
+                return;
+
         cpu_set(rq->cpu, rq->rd->rto_mask);
         /*
          * Make sure the mask is visible before we set
@@ -26,6 +29,9 @@ static inline void rt_set_overload(struct rq *rq)
 
 static inline void rt_clear_overload(struct rq *rq)
 {
+        if (!rq->online)
+                return;
+
         /* the order here really doesn't matter */
         atomic_dec(&rq->rd->rto_count);
         cpu_clear(rq->cpu, rq->rd->rto_mask);
@@ -222,46 +228,8 @@ static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq)
 
 #endif
 
-static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun)
-{
-        int i, idle = 1;
-        cpumask_t span;
-
-        if (rt_b->rt_runtime == RUNTIME_INF)
-                return 1;
-
-        span = sched_rt_period_mask();
-        for_each_cpu_mask(i, span) {
-                int enqueue = 0;
-                struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i);
-                struct rq *rq = rq_of_rt_rq(rt_rq);
-
-                spin_lock(&rq->lock);
-                if (rt_rq->rt_time) {
-                        u64 runtime;
-
-                        spin_lock(&rt_rq->rt_runtime_lock);
-                        runtime = rt_rq->rt_runtime;
-                        rt_rq->rt_time -= min(rt_rq->rt_time, overrun*runtime);
-                        if (rt_rq->rt_throttled && rt_rq->rt_time < runtime) {
-                                rt_rq->rt_throttled = 0;
-                                enqueue = 1;
-                        }
-                        if (rt_rq->rt_time || rt_rq->rt_nr_running)
-                                idle = 0;
-                        spin_unlock(&rt_rq->rt_runtime_lock);
-                }
-
-                if (enqueue)
-                        sched_rt_rq_enqueue(rt_rq);
-                spin_unlock(&rq->lock);
-        }
-
-        return idle;
-}
-
 #ifdef CONFIG_SMP
-static int balance_runtime(struct rt_rq *rt_rq)
+static int do_balance_runtime(struct rt_rq *rt_rq)
 {
         struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
         struct root_domain *rd = cpu_rq(smp_processor_id())->rd;
@@ -280,6 +248,9 @@ static int balance_runtime(struct rt_rq *rt_rq)
                         continue;
 
                 spin_lock(&iter->rt_runtime_lock);
+                if (iter->rt_runtime == RUNTIME_INF)
+                        goto next;
+
                 diff = iter->rt_runtime - iter->rt_time;
                 if (diff > 0) {
                         do_div(diff, weight);
@@ -293,14 +264,165 @@ static int balance_runtime(struct rt_rq *rt_rq)
                                 break;
                         }
                 }
+next:
                 spin_unlock(&iter->rt_runtime_lock);
         }
         spin_unlock(&rt_b->rt_runtime_lock);
 
         return more;
 }
+
+static void __disable_runtime(struct rq *rq)
+{
+        struct root_domain *rd = rq->rd;
+        struct rt_rq *rt_rq;
+
+        if (unlikely(!scheduler_running))
+                return;
+
+        for_each_leaf_rt_rq(rt_rq, rq) {
+                struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
+                s64 want;
+                int i;
+
+                spin_lock(&rt_b->rt_runtime_lock);
+                spin_lock(&rt_rq->rt_runtime_lock);
+                if (rt_rq->rt_runtime == RUNTIME_INF ||
+                                rt_rq->rt_runtime == rt_b->rt_runtime)
+                        goto balanced;
+                spin_unlock(&rt_rq->rt_runtime_lock);
+
+                want = rt_b->rt_runtime - rt_rq->rt_runtime;
+
+                for_each_cpu_mask(i, rd->span) {
+                        struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);
+                        s64 diff;
+
+                        if (iter == rt_rq)
+                                continue;
+
+                        spin_lock(&iter->rt_runtime_lock);
+                        if (want > 0) {
+                                diff = min_t(s64, iter->rt_runtime, want);
+                                iter->rt_runtime -= diff;
+                                want -= diff;
+                        } else {
+                                iter->rt_runtime -= want;
+                                want -= want;
+                        }
+                        spin_unlock(&iter->rt_runtime_lock);
+
+                        if (!want)
+                                break;
+                }
+
+                spin_lock(&rt_rq->rt_runtime_lock);
+                BUG_ON(want);
+balanced:
+                rt_rq->rt_runtime = RUNTIME_INF;
+                spin_unlock(&rt_rq->rt_runtime_lock);
+                spin_unlock(&rt_b->rt_runtime_lock);
+        }
+}
+
+static void disable_runtime(struct rq *rq)
+{
+        unsigned long flags;
+
+        spin_lock_irqsave(&rq->lock, flags);
+        __disable_runtime(rq);
+        spin_unlock_irqrestore(&rq->lock, flags);
+}
+
+static void __enable_runtime(struct rq *rq)
+{
+        struct root_domain *rd = rq->rd;
+        struct rt_rq *rt_rq;
+
+        if (unlikely(!scheduler_running))
+                return;
+
+        for_each_leaf_rt_rq(rt_rq, rq) {
+                struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
+
+                spin_lock(&rt_b->rt_runtime_lock);
+                spin_lock(&rt_rq->rt_runtime_lock);
+                rt_rq->rt_runtime = rt_b->rt_runtime;
+                rt_rq->rt_time = 0;
+                spin_unlock(&rt_rq->rt_runtime_lock);
+                spin_unlock(&rt_b->rt_runtime_lock);
+        }
+}
+
+static void enable_runtime(struct rq *rq)
+{
+        unsigned long flags;
+
+        spin_lock_irqsave(&rq->lock, flags);
+        __enable_runtime(rq);
+        spin_unlock_irqrestore(&rq->lock, flags);
+}
+
+static int balance_runtime(struct rt_rq *rt_rq)
+{
+        int more = 0;
+
+        if (rt_rq->rt_time > rt_rq->rt_runtime) {
+                spin_unlock(&rt_rq->rt_runtime_lock);
+                more = do_balance_runtime(rt_rq);
+                spin_lock(&rt_rq->rt_runtime_lock);
+        }
+
+        return more;
+}
+#else
+static inline int balance_runtime(struct rt_rq *rt_rq)
+{
+        return 0;
+}
 #endif
 
+static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun)
+{
+        int i, idle = 1;
+        cpumask_t span;
+
+        if (rt_b->rt_runtime == RUNTIME_INF)
+                return 1;
+
+        span = sched_rt_period_mask();
+        for_each_cpu_mask(i, span) {
+                int enqueue = 0;
+                struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i);
+                struct rq *rq = rq_of_rt_rq(rt_rq);
+
+                spin_lock(&rq->lock);
+                if (rt_rq->rt_time) {
+                        u64 runtime;
+
+                        spin_lock(&rt_rq->rt_runtime_lock);
+                        if (rt_rq->rt_throttled)
+                                balance_runtime(rt_rq);
+                        runtime = rt_rq->rt_runtime;
+                        rt_rq->rt_time -= min(rt_rq->rt_time, overrun*runtime);
+                        if (rt_rq->rt_throttled && rt_rq->rt_time < runtime) {
+                                rt_rq->rt_throttled = 0;
+                                enqueue = 1;
+                        }
+                        if (rt_rq->rt_time || rt_rq->rt_nr_running)
+                                idle = 0;
+                        spin_unlock(&rt_rq->rt_runtime_lock);
+                } else if (rt_rq->rt_nr_running)
+                        idle = 0;
+
+                if (enqueue)
+                        sched_rt_rq_enqueue(rt_rq);
+                spin_unlock(&rq->lock);
+        }
+
+        return idle;
+}
+
 static inline int rt_se_prio(struct sched_rt_entity *rt_se)
 {
 #ifdef CONFIG_RT_GROUP_SCHED
@@ -326,18 +448,10 @@ static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq)
         if (sched_rt_runtime(rt_rq) >= sched_rt_period(rt_rq))
                 return 0;
 
-#ifdef CONFIG_SMP
-        if (rt_rq->rt_time > runtime) {
-                int more;
-
-                spin_unlock(&rt_rq->rt_runtime_lock);
-                more = balance_runtime(rt_rq);
-                spin_lock(&rt_rq->rt_runtime_lock);
-
-                if (more)
-                        runtime = sched_rt_runtime(rt_rq);
-        }
-#endif
+        balance_runtime(rt_rq);
+        runtime = sched_rt_runtime(rt_rq);
+        if (runtime == RUNTIME_INF)
+                return 0;
 
         if (rt_rq->rt_time > runtime) {
                 rt_rq->rt_throttled = 1;
@@ -391,12 +505,21 @@ void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
         WARN_ON(!rt_prio(rt_se_prio(rt_se)));
         rt_rq->rt_nr_running++;
 #if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
-        if (rt_se_prio(rt_se) < rt_rq->highest_prio)
+        if (rt_se_prio(rt_se) < rt_rq->highest_prio) {
+                struct rq *rq = rq_of_rt_rq(rt_rq);
+
                 rt_rq->highest_prio = rt_se_prio(rt_se);
+#ifdef CONFIG_SMP
+                if (rq->online)
+                        cpupri_set(&rq->rd->cpupri, rq->cpu,
+                                   rt_se_prio(rt_se));
+#endif
+        }
 #endif
 #ifdef CONFIG_SMP
         if (rt_se->nr_cpus_allowed > 1) {
                 struct rq *rq = rq_of_rt_rq(rt_rq);
+
                 rq->rt.rt_nr_migratory++;
         }
 
@@ -416,6 +539,10 @@ void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
 static inline
 void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
 {
+#ifdef CONFIG_SMP
+        int highest_prio = rt_rq->highest_prio;
+#endif
+
         WARN_ON(!rt_prio(rt_se_prio(rt_se)));
         WARN_ON(!rt_rq->rt_nr_running);
         rt_rq->rt_nr_running--;
@@ -439,6 +566,14 @@ void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
                 rq->rt.rt_nr_migratory--;
         }
 
+        if (rt_rq->highest_prio != highest_prio) {
+                struct rq *rq = rq_of_rt_rq(rt_rq);
+
+                if (rq->online)
+                        cpupri_set(&rq->rd->cpupri, rq->cpu,
+                                   rt_rq->highest_prio);
+        }
+
         update_rt_migration(rq_of_rt_rq(rt_rq));
 #endif /* CONFIG_SMP */
 #ifdef CONFIG_RT_GROUP_SCHED
@@ -454,6 +589,7 @@ static void __enqueue_rt_entity(struct sched_rt_entity *rt_se)
         struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
         struct rt_prio_array *array = &rt_rq->active;
         struct rt_rq *group_rq = group_rt_rq(rt_se);
+        struct list_head *queue = array->queue + rt_se_prio(rt_se);
 
         /*
          * Don't enqueue the group if its throttled, or when empty.
@@ -464,7 +600,11 @@ static void __enqueue_rt_entity(struct sched_rt_entity *rt_se)
         if (group_rq && (rt_rq_throttled(group_rq) || !group_rq->rt_nr_running))
                 return;
 
-        list_add_tail(&rt_se->run_list, array->queue + rt_se_prio(rt_se));
+        if (rt_se->nr_cpus_allowed == 1)
+                list_add(&rt_se->run_list, queue);
+        else
+                list_add_tail(&rt_se->run_list, queue);
+
         __set_bit(rt_se_prio(rt_se), array->bitmap);
 
         inc_rt_tasks(rt_se, rt_rq);
@@ -551,8 +691,11 @@ void requeue_rt_entity(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se)
         struct rt_prio_array *array = &rt_rq->active;
         struct list_head *queue = array->queue + rt_se_prio(rt_se);
 
-        if (on_rt_rq(rt_se))
-                list_move_tail(&rt_se->run_list, queue);
+        if (on_rt_rq(rt_se)) {
+                list_del_init(&rt_se->run_list);
+                list_add_tail(&rt_se->run_list,
+                              array->queue + rt_se_prio(rt_se));
+        }
 }
 
 static void requeue_task_rt(struct rq *rq, struct task_struct *p)
@@ -615,8 +758,37 @@ static int select_task_rq_rt(struct task_struct *p, int sync)
  */
 static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p)
 {
-        if (p->prio < rq->curr->prio)
+        if (p->prio < rq->curr->prio) {
                 resched_task(rq->curr);
+                return;
+        }
+
+#ifdef CONFIG_SMP
+        /*
+         * If:
+         *
+         * - the newly woken task is of equal priority to the current task
+         * - the newly woken task is non-migratable while current is migratable
+         * - current will be preempted on the next reschedule
+         *
+         * we should check to see if current can readily move to a different
+         * cpu.  If so, we will reschedule to allow the push logic to try
+         * to move current somewhere else, making room for our non-migratable
+         * task.
+         */
+        if((p->prio == rq->curr->prio)
+           && p->rt.nr_cpus_allowed == 1
+           && rq->curr->rt.nr_cpus_allowed != 1) {
+                cpumask_t mask;
+
+                if (cpupri_find(&rq->rd->cpupri, rq->curr, &mask))
+                        /*
+                         * There appears to be other cpus that can accept
+                         * current, so lets reschedule to try and push it away
+                         */
+                        resched_task(rq->curr);
+        }
+#endif
 }
 
 static struct sched_rt_entity *pick_next_rt_entity(struct rq *rq,
@@ -719,73 +891,6 @@ static struct task_struct *pick_next_highest_task_rt(struct rq *rq, int cpu)
 
 static DEFINE_PER_CPU(cpumask_t, local_cpu_mask);
 
-static int find_lowest_cpus(struct task_struct *task, cpumask_t *lowest_mask)
-{
-        int       lowest_prio = -1;
-        int       lowest_cpu  = -1;
-        int       count       = 0;
-        int       cpu;
-
-        cpus_and(*lowest_mask, task_rq(task)->rd->online, task->cpus_allowed);
-
-        /*
-         * Scan each rq for the lowest prio.
-         */
-        for_each_cpu_mask(cpu, *lowest_mask) {
-                struct rq *rq = cpu_rq(cpu);
-
-                /* We look for lowest RT prio or non-rt CPU */
-                if (rq->rt.highest_prio >= MAX_RT_PRIO) {
-                        /*
-                         * if we already found a low RT queue
-                         * and now we found this non-rt queue
-                         * clear the mask and set our bit.
-                         * Otherwise just return the queue as is
-                         * and the count==1 will cause the algorithm
-                         * to use the first bit found.
-                         */
-                        if (lowest_cpu != -1) {
-                                cpus_clear(*lowest_mask);
-                                cpu_set(rq->cpu, *lowest_mask);
-                        }
-                        return 1;
-                }
-
-                /* no locking for now */
-                if ((rq->rt.highest_prio > task->prio)
-                    && (rq->rt.highest_prio >= lowest_prio)) {
-                        if (rq->rt.highest_prio > lowest_prio) {
-                                /* new low - clear old data */
-                                lowest_prio = rq->rt.highest_prio;
-                                lowest_cpu = cpu;
-                                count = 0;
-                        }
-                        count++;
-                } else
-                        cpu_clear(cpu, *lowest_mask);
-        }
-
-        /*
-         * Clear out all the set bits that represent
-         * runqueues that were of higher prio than
-         * the lowest_prio.
-         */
-        if (lowest_cpu > 0) {
-                /*
-                 * Perhaps we could add another cpumask op to
-                 * zero out bits. Like cpu_zero_bits(cpumask, nrbits);
-                 * Then that could be optimized to use memset and such.
-                 */
-                for_each_cpu_mask(cpu, *lowest_mask) {
-                        if (cpu >= lowest_cpu)
-                                break;
-                        cpu_clear(cpu, *lowest_mask);
-                }
-        }
-
-        return count;
-}
-
 static inline int pick_optimal_cpu(int this_cpu, cpumask_t *mask)
 {
         int first;
@@ -807,17 +912,12 @@ static int find_lowest_rq(struct task_struct *task)
         cpumask_t *lowest_mask = &__get_cpu_var(local_cpu_mask);
         int this_cpu = smp_processor_id();
         int cpu      = task_cpu(task);
-        int count    = find_lowest_cpus(task, lowest_mask);
 
-        if (!count)
-                return -1; /* No targets found */
+        if (task->rt.nr_cpus_allowed == 1)
+                return -1; /* No other targets possible */
 
-        /*
-         * There is no sense in performing an optimal search if only one
-         * target is found.
-         */
-        if (count == 1)
-                return first_cpu(*lowest_mask);
+        if (!cpupri_find(&task_rq(task)->rd->cpupri, task, lowest_mask))
+                return -1; /* No targets found */
 
         /*
          * At this point we have built a mask of cpus representing the
@@ -1162,17 +1262,25 @@ static void set_cpus_allowed_rt(struct task_struct *p,
 }
 
 /* Assumes rq->lock is held */
-static void join_domain_rt(struct rq *rq)
+static void rq_online_rt(struct rq *rq)
 {
         if (rq->rt.overloaded)
                 rt_set_overload(rq);
+
+        __enable_runtime(rq);
+
+        cpupri_set(&rq->rd->cpupri, rq->cpu, rq->rt.highest_prio);
 }
 
 /* Assumes rq->lock is held */
-static void leave_domain_rt(struct rq *rq)
+static void rq_offline_rt(struct rq *rq)
 {
         if (rq->rt.overloaded)
                 rt_clear_overload(rq);
+
+        __disable_runtime(rq);
+
+        cpupri_set(&rq->rd->cpupri, rq->cpu, CPUPRI_INVALID);
 }
 
 /*
@@ -1335,8 +1443,8 @@ static const struct sched_class rt_sched_class = {
         .load_balance           = load_balance_rt,
         .move_one_task          = move_one_task_rt,
         .set_cpus_allowed       = set_cpus_allowed_rt,
-        .join_domain            = join_domain_rt,
-        .leave_domain           = leave_domain_rt,
+        .rq_online              = rq_online_rt,
+        .rq_offline             = rq_offline_rt,
         .pre_schedule           = pre_schedule_rt,
         .post_schedule          = post_schedule_rt,
         .task_wake_up           = task_wake_up_rt,
@@ -1349,3 +1457,17 @@ static const struct sched_class rt_sched_class = {
         .prio_changed           = prio_changed_rt,
         .switched_to            = switched_to_rt,
 };
+
+#ifdef CONFIG_SCHED_DEBUG
+extern void print_rt_rq(struct seq_file *m, int cpu, struct rt_rq *rt_rq);
+
+static void print_rt_stats(struct seq_file *m, int cpu)
+{
+        struct rt_rq *rt_rq;
+
+        rcu_read_lock();
+        for_each_leaf_rt_rq(rt_rq, cpu_rq(cpu))
+                print_rt_rq(m, cpu, rt_rq);
+        rcu_read_unlock();
+}
+#endif