1 files changed, 60 insertions, 36 deletions

diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c
index 9bf0d2a7304..a4d790cddb1 100644
--- a/kernel/sched_rt.c
+++ b/kernel/sched_rt.c
@@ -3,13 +3,18 @@
  * policies)
  */
 
+#ifdef CONFIG_RT_GROUP_SCHED
+
+#define rt_entity_is_task(rt_se) (!(rt_se)->my_q)
+
 static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se)
 {
+#ifdef CONFIG_SCHED_DEBUG
+        WARN_ON_ONCE(!rt_entity_is_task(rt_se));
+#endif
         return container_of(rt_se, struct task_struct, rt);
 }
 
-#ifdef CONFIG_RT_GROUP_SCHED
-
 static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)
 {
         return rt_rq->rq;
@@ -22,6 +27,13 @@ static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)
 
 #else /* CONFIG_RT_GROUP_SCHED */
 
+#define rt_entity_is_task(rt_se) (1)
+
+static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se)
+{
+        return container_of(rt_se, struct task_struct, rt);
+}
+
 static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)
 {
         return container_of(rt_rq, struct rq, rt);
@@ -73,7 +85,7 @@ static inline void rt_clear_overload(struct rq *rq)
 
 static void update_rt_migration(struct rt_rq *rt_rq)
 {
-        if (rt_rq->rt_nr_migratory && (rt_rq->rt_nr_running > 1)) {
+        if (rt_rq->rt_nr_migratory && rt_rq->rt_nr_total > 1) {
                 if (!rt_rq->overloaded) {
                         rt_set_overload(rq_of_rt_rq(rt_rq));
                         rt_rq->overloaded = 1;
@@ -86,6 +98,12 @@ static void update_rt_migration(struct rt_rq *rt_rq)
 
 static void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
 {
+        if (!rt_entity_is_task(rt_se))
+                return;
+
+        rt_rq = &rq_of_rt_rq(rt_rq)->rt;
+
+        rt_rq->rt_nr_total++;
         if (rt_se->nr_cpus_allowed > 1)
                 rt_rq->rt_nr_migratory++;
 
@@ -94,6 +112,12 @@ static void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
 
 static void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
 {
+        if (!rt_entity_is_task(rt_se))
+                return;
+
+        rt_rq = &rq_of_rt_rq(rt_rq)->rt;
+
+        rt_rq->rt_nr_total--;
         if (rt_se->nr_cpus_allowed > 1)
                 rt_rq->rt_nr_migratory--;
 
@@ -112,6 +136,11 @@ static void dequeue_pushable_task(struct rq *rq, struct task_struct *p)
         plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);
 }
 
+static inline int has_pushable_tasks(struct rq *rq)
+{
+        return !plist_head_empty(&rq->rt.pushable_tasks);
+}
+
 #else
 
 static inline void enqueue_pushable_task(struct rq *rq, struct task_struct *p)
@@ -586,6 +615,8 @@ static void update_curr_rt(struct rq *rq)
         curr->se.exec_start = rq->clock;
         cpuacct_charge(curr, delta_exec);
 
+        sched_rt_avg_update(rq, delta_exec);
+
         if (!rt_bandwidth_enabled())
                 return;
 
@@ -858,8 +889,6 @@ static void enqueue_task_rt(struct rq *rq, struct task_struct *p, int wakeup)
 
         if (!task_current(rq, p) && p->rt.nr_cpus_allowed > 1)
                 enqueue_pushable_task(rq, p);
-
-        inc_cpu_load(rq, p->se.load.weight);
 }
 
 static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int sleep)
@@ -870,8 +899,6 @@ static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int sleep)
         dequeue_rt_entity(rt_se);
 
         dequeue_pushable_task(rq, p);
-
-        dec_cpu_load(rq, p->se.load.weight);
 }
 
 /*
@@ -911,10 +938,13 @@ static void yield_task_rt(struct rq *rq)
 #ifdef CONFIG_SMP
 static int find_lowest_rq(struct task_struct *task);
 
-static int select_task_rq_rt(struct task_struct *p, int sync)
+static int select_task_rq_rt(struct task_struct *p, int sd_flag, int flags)
 {
         struct rq *rq = task_rq(p);
 
+        if (sd_flag != SD_BALANCE_WAKE)
+                return smp_processor_id();
+
         /*
          * If the current task is an RT task, then
          * try to see if we can wake this RT task up on another
@@ -972,7 +1002,7 @@ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)
 /*
  * Preempt the current task with a newly woken task if needed:
  */
-static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p, int sync)
+static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p, int flags)
 {
         if (p->prio < rq->curr->prio) {
                 resched_task(rq->curr);
@@ -1048,6 +1078,14 @@ static struct task_struct *pick_next_task_rt(struct rq *rq)
         if (p)
                 dequeue_pushable_task(rq, p);
 
+#ifdef CONFIG_SMP
+        /*
+         * We detect this state here so that we can avoid taking the RQ
+         * lock again later if there is no need to push
+         */
+        rq->post_schedule = has_pushable_tasks(rq);
+#endif
+
         return p;
 }
 
@@ -1146,13 +1184,6 @@ static int find_lowest_rq(struct task_struct *task)
                 return -1; /* No targets found */
 
         /*
-         * Only consider CPUs that are usable for migration.
-         * I guess we might want to change cpupri_find() to ignore those
-         * in the first place.
-         */
-        cpumask_and(lowest_mask, lowest_mask, cpu_active_mask);
-
-        /*
          * At this point we have built a mask of cpus representing the
          * lowest priority tasks in the system.  Now we want to elect
          * the best one based on our affinity and topology.
@@ -1246,11 +1277,6 @@ static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq)
         return lowest_rq;
 }
 
-static inline int has_pushable_tasks(struct rq *rq)
-{
-        return !plist_head_empty(&rq->rt.pushable_tasks);
-}
-
 static struct task_struct *pick_next_pushable_task(struct rq *rq)
 {
         struct task_struct *p;
@@ -1450,23 +1476,9 @@ static void pre_schedule_rt(struct rq *rq, struct task_struct *prev)
                 pull_rt_task(rq);
 }
 
-/*
- * assumes rq->lock is held
- */
-static int needs_post_schedule_rt(struct rq *rq)
-{
-        return has_pushable_tasks(rq);
-}
-
 static void post_schedule_rt(struct rq *rq)
 {
-        /*
-         * This is only called if needs_post_schedule_rt() indicates that
-         * we need to push tasks away
-         */
-        spin_lock_irq(&rq->lock);
         push_rt_tasks(rq);
-        spin_unlock_irq(&rq->lock);
 }
 
 /*
@@ -1722,6 +1734,17 @@ static void set_curr_task_rt(struct rq *rq)
         dequeue_pushable_task(rq, p);
 }
 
+unsigned int get_rr_interval_rt(struct task_struct *task)
+{
+        /*
+         * Time slice is 0 for SCHED_FIFO tasks
+         */
+        if (task->policy == SCHED_RR)
+                return DEF_TIMESLICE;
+        else
+                return 0;
+}
+
 static const struct sched_class rt_sched_class = {
         .next                   = &fair_sched_class,
         .enqueue_task           = enqueue_task_rt,
@@ -1742,7 +1765,6 @@ static const struct sched_class rt_sched_class = {
         .rq_online              = rq_online_rt,
         .rq_offline             = rq_offline_rt,
         .pre_schedule           = pre_schedule_rt,
-        .needs_post_schedule    = needs_post_schedule_rt,
         .post_schedule          = post_schedule_rt,
         .task_wake_up           = task_wake_up_rt,
         .switched_from          = switched_from_rt,
@@ -1751,6 +1773,8 @@ static const struct sched_class rt_sched_class = {
         .set_curr_task          = set_curr_task_rt,
         .task_tick              = task_tick_rt,
 
+        .get_rr_interval        = get_rr_interval_rt,
+
         .prio_changed           = prio_changed_rt,
         .switched_to            = switched_to_rt,
 };