1 files changed, 374 insertions, 163 deletions

diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c
index da932f4c852..299d012b439 100644
--- a/kernel/sched_rt.c
+++ b/kernel/sched_rt.c
@@ -3,6 +3,40 @@
  * policies)
  */
 
+static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se)
+{
+        return container_of(rt_se, struct task_struct, rt);
+}
+
+#ifdef CONFIG_RT_GROUP_SCHED
+
+static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)
+{
+        return rt_rq->rq;
+}
+
+static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)
+{
+        return rt_se->rt_rq;
+}
+
+#else /* CONFIG_RT_GROUP_SCHED */
+
+static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)
+{
+        return container_of(rt_rq, struct rq, rt);
+}
+
+static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)
+{
+        struct task_struct *p = rt_task_of(rt_se);
+        struct rq *rq = task_rq(p);
+
+        return &rq->rt;
+}
+
+#endif /* CONFIG_RT_GROUP_SCHED */
+
 #ifdef CONFIG_SMP
 
 static inline int rt_overloaded(struct rq *rq)
@@ -37,25 +71,69 @@ static inline void rt_clear_overload(struct rq *rq)
         cpumask_clear_cpu(rq->cpu, rq->rd->rto_mask);
 }
 
-static void update_rt_migration(struct rq *rq)
+static void update_rt_migration(struct rt_rq *rt_rq)
 {
-        if (rq->rt.rt_nr_migratory && (rq->rt.rt_nr_running > 1)) {
-                if (!rq->rt.overloaded) {
-                        rt_set_overload(rq);
-                        rq->rt.overloaded = 1;
+        if (rt_rq->rt_nr_migratory && (rt_rq->rt_nr_running > 1)) {
+                if (!rt_rq->overloaded) {
+                        rt_set_overload(rq_of_rt_rq(rt_rq));
+                        rt_rq->overloaded = 1;
                 }
-        } else if (rq->rt.overloaded) {
-                rt_clear_overload(rq);
-                rq->rt.overloaded = 0;
+        } else if (rt_rq->overloaded) {
+                rt_clear_overload(rq_of_rt_rq(rt_rq));
+                rt_rq->overloaded = 0;
         }
 }
-#endif /* CONFIG_SMP */
 
-static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se)
+static void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
+{
+        if (rt_se->nr_cpus_allowed > 1)
+                rt_rq->rt_nr_migratory++;
+
+        update_rt_migration(rt_rq);
+}
+
+static void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
+{
+        if (rt_se->nr_cpus_allowed > 1)
+                rt_rq->rt_nr_migratory--;
+
+        update_rt_migration(rt_rq);
+}
+
+static void enqueue_pushable_task(struct rq *rq, struct task_struct *p)
+{
+        plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);
+        plist_node_init(&p->pushable_tasks, p->prio);
+        plist_add(&p->pushable_tasks, &rq->rt.pushable_tasks);
+}
+
+static void dequeue_pushable_task(struct rq *rq, struct task_struct *p)
+{
+        plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);
+}
+
+#else
+
+static inline void enqueue_pushable_task(struct rq *rq, struct task_struct *p)
 {
-        return container_of(rt_se, struct task_struct, rt);
 }
 
+static inline void dequeue_pushable_task(struct rq *rq, struct task_struct *p)
+{
+}
+
+static inline
+void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
+{
+}
+
+static inline
+void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
+{
+}
+
+#endif /* CONFIG_SMP */
+
 static inline int on_rt_rq(struct sched_rt_entity *rt_se)
 {
         return !list_empty(&rt_se->run_list);
@@ -79,16 +157,6 @@ static inline u64 sched_rt_period(struct rt_rq *rt_rq)
 #define for_each_leaf_rt_rq(rt_rq, rq) \
         list_for_each_entry_rcu(rt_rq, &rq->leaf_rt_rq_list, leaf_rt_rq_list)
 
-static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)
-{
-        return rt_rq->rq;
-}
-
-static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)
-{
-        return rt_se->rt_rq;
-}
-
 #define for_each_sched_rt_entity(rt_se) \
         for (; rt_se; rt_se = rt_se->parent)
 
@@ -108,7 +176,7 @@ static void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
         if (rt_rq->rt_nr_running) {
                 if (rt_se && !on_rt_rq(rt_se))
                         enqueue_rt_entity(rt_se);
-                if (rt_rq->highest_prio < curr->prio)
+                if (rt_rq->highest_prio.curr < curr->prio)
                         resched_task(curr);
         }
 }
@@ -176,19 +244,6 @@ static inline u64 sched_rt_period(struct rt_rq *rt_rq)
 #define for_each_leaf_rt_rq(rt_rq, rq) \
         for (rt_rq = &rq->rt; rt_rq; rt_rq = NULL)
 
-static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)
-{
-        return container_of(rt_rq, struct rq, rt);
-}
-
-static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)
-{
-        struct task_struct *p = rt_task_of(rt_se);
-        struct rq *rq = task_rq(p);
-
-        return &rq->rt;
-}
-
 #define for_each_sched_rt_entity(rt_se) \
         for (; rt_se; rt_se = NULL)
 
@@ -473,7 +528,7 @@ static inline int rt_se_prio(struct sched_rt_entity *rt_se)
         struct rt_rq *rt_rq = group_rt_rq(rt_se);
 
         if (rt_rq)
-                return rt_rq->highest_prio;
+                return rt_rq->highest_prio.curr;
 #endif
 
         return rt_task_of(rt_se)->prio;
@@ -547,91 +602,174 @@ static void update_curr_rt(struct rq *rq)
         }
 }
 
-static inline
-void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
+#if defined CONFIG_SMP
+
+static struct task_struct *pick_next_highest_task_rt(struct rq *rq, int cpu);
+
+static inline int next_prio(struct rq *rq)
 {
-        WARN_ON(!rt_prio(rt_se_prio(rt_se)));
-        rt_rq->rt_nr_running++;
-#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
-        if (rt_se_prio(rt_se) < rt_rq->highest_prio) {
-#ifdef CONFIG_SMP
-                struct rq *rq = rq_of_rt_rq(rt_rq);
-#endif
+        struct task_struct *next = pick_next_highest_task_rt(rq, rq->cpu);
+
+        if (next && rt_prio(next->prio))
+                return next->prio;
+        else
+                return MAX_RT_PRIO;
+}
+
+static void
+inc_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio)
+{
+        struct rq *rq = rq_of_rt_rq(rt_rq);
+