1 files changed, 21 insertions, 111 deletions

diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index 127a2c4cf4ab..01970c8e64df 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -399,20 +399,6 @@ static inline struct task_group *next_task_group(struct task_group *tg)
                 (iter = next_task_group(iter)) &&                       \
                 (rt_rq = iter->rt_rq[cpu_of(rq)]);)
 
-static inline void list_add_leaf_rt_rq(struct rt_rq *rt_rq)
-{
-        list_add_rcu(&rt_rq->leaf_rt_rq_list,
-                        &rq_of_rt_rq(rt_rq)->leaf_rt_rq_list);
-}
-
-static inline void list_del_leaf_rt_rq(struct rt_rq *rt_rq)
-{
-        list_del_rcu(&rt_rq->leaf_rt_rq_list);
-}
-
-#define for_each_leaf_rt_rq(rt_rq, rq) \
-        list_for_each_entry_rcu(rt_rq, &rq->leaf_rt_rq_list, leaf_rt_rq_list)
-
 #define for_each_sched_rt_entity(rt_se) \
         for (; rt_se; rt_se = rt_se->parent)
 
@@ -472,7 +458,7 @@ static int rt_se_boosted(struct sched_rt_entity *rt_se)
 #ifdef CONFIG_SMP
 static inline const struct cpumask *sched_rt_period_mask(void)
 {
-        return cpu_rq(smp_processor_id())->rd->span;
+        return this_rq()->rd->span;
 }
 #else
 static inline const struct cpumask *sched_rt_period_mask(void)
@@ -509,17 +495,6 @@ typedef struct rt_rq *rt_rq_iter_t;
 #define for_each_rt_rq(rt_rq, iter, rq) \
         for ((void) iter, rt_rq = &rq->rt; rt_rq; rt_rq = NULL)
 
-static inline void list_add_leaf_rt_rq(struct rt_rq *rt_rq)
-{
-}
-
-static inline void list_del_leaf_rt_rq(struct rt_rq *rt_rq)
-{
-}
-
-#define for_each_leaf_rt_rq(rt_rq, rq) \
-        for (rt_rq = &rq->rt; rt_rq; rt_rq = NULL)
-
 #define for_each_sched_rt_entity(rt_se) \
         for (; rt_se; rt_se = NULL)
 
@@ -699,15 +674,6 @@ balanced:
         }
 }
 
-static void disable_runtime(struct rq *rq)
-{
-        unsigned long flags;
-
-        raw_spin_lock_irqsave(&rq->lock, flags);
-        __disable_runtime(rq);
-        raw_spin_unlock_irqrestore(&rq->lock, flags);
-}
-
 static void __enable_runtime(struct rq *rq)
 {
         rt_rq_iter_t iter;
@@ -732,37 +698,6 @@ static void __enable_runtime(struct rq *rq)
         }
 }
 
-static void enable_runtime(struct rq *rq)
-{
-        unsigned long flags;
-
-        raw_spin_lock_irqsave(&rq->lock, flags);
-        __enable_runtime(rq);
-        raw_spin_unlock_irqrestore(&rq->lock, flags);
-}
-
-int update_runtime(struct notifier_block *nfb, unsigned long action, void *hcpu)
-{
-        int cpu = (int)(long)hcpu;
-
-        switch (action) {
-        case CPU_DOWN_PREPARE:
-        case CPU_DOWN_PREPARE_FROZEN:
-                disable_runtime(cpu_rq(cpu));
-                return NOTIFY_OK;
-
-        case CPU_DOWN_FAILED:
-        case CPU_DOWN_FAILED_FROZEN:
-        case CPU_ONLINE:
-        case CPU_ONLINE_FROZEN:
-                enable_runtime(cpu_rq(cpu));
-                return NOTIFY_OK;
-
-        default:
-                return NOTIFY_DONE;
-        }
-}
-
 static int balance_runtime(struct rt_rq *rt_rq)
 {
         int more = 0;
@@ -926,7 +861,7 @@ static void update_curr_rt(struct rq *rq)
         if (curr->sched_class != &rt_sched_class)
                 return;
 
-        delta_exec = rq->clock_task - curr->se.exec_start;
+        delta_exec = rq_clock_task(rq) - curr->se.exec_start;
         if (unlikely((s64)delta_exec <= 0))
                 return;
 
@@ -936,7 +871,7 @@ static void update_curr_rt(struct rq *rq)
         curr->se.sum_exec_runtime += delta_exec;
         account_group_exec_runtime(curr, delta_exec);
 
-        curr->se.exec_start = rq->clock_task;
+        curr->se.exec_start = rq_clock_task(rq);
         cpuacct_charge(curr, delta_exec);
 
         sched_rt_avg_update(rq, delta_exec);
@@ -1106,9 +1041,6 @@ static void __enqueue_rt_entity(struct sched_rt_entity *rt_se, bool head)
         if (group_rq && (rt_rq_throttled(group_rq) || !group_rq->rt_nr_running))
                 return;
 
-        if (!rt_rq->rt_nr_running)
-                list_add_leaf_rt_rq(rt_rq);
-
         if (head)
                 list_add(&rt_se->run_list, queue);
         else
@@ -1128,8 +1060,6 @@ static void __dequeue_rt_entity(struct sched_rt_entity *rt_se)
                 __clear_bit(rt_se_prio(rt_se), array->bitmap);
 
         dec_rt_tasks(rt_se, rt_rq);
-        if (!rt_rq->rt_nr_running)
-                list_del_leaf_rt_rq(rt_rq);
 }
 
 /*
@@ -1385,7 +1315,7 @@ static struct task_struct *_pick_next_task_rt(struct rq *rq)
         } while (rt_rq);
 
         p = rt_task_of(rt_se);
-        p->se.exec_start = rq->clock_task;
+        p->se.exec_start = rq_clock_task(rq);
 
         return p;
 }
@@ -1434,42 +1364,24 @@ static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu)
         return 0;
 }
 
-/* Return the second highest RT task, NULL otherwise */
-static struct task_struct *pick_next_highest_task_rt(struct rq *rq, int cpu)
+/*
+ * Return the highest pushable rq's task, which is suitable to be executed
+ * on the cpu, NULL otherwise
+ */
+static struct task_struct *pick_highest_pushable_task(struct rq *rq, int cpu)
 {
-        struct task_struct *next = NULL;
-        struct sched_rt_entity *rt_se;
-        struct rt_prio_array *array;
-        struct rt_rq *rt_rq;
-        int idx;
-
-        for_each_leaf_rt_rq(rt_rq, rq) {
-                array = &rt_rq->active;
-                idx = sched_find_first_bit(array->bitmap);
-next_idx:
-                if (idx >= MAX_RT_PRIO)
-                        continue;
-                if (next && next->prio <= idx)
-                        continue;
-                list_for_each_entry(rt_se, array->queue + idx, run_list) {
-                        struct task_struct *p;
+        struct plist_head *head = &rq->rt.pushable_tasks;
+        struct task_struct *p;
 
-                        if (!rt_entity_is_task(rt_se))
-                                continue;
+        if (!has_pushable_tasks(rq))
+                return NULL;
 
-                        p = rt_task_of(rt_se);
-                        if (pick_rt_task(rq, p, cpu)) {
-                                next = p;
-                                break;
-                        }
-                }
-                if (!next) {
-                        idx = find_next_bit(array->bitmap, MAX_RT_PRIO, idx+1);
-                        goto next_idx;
-                }
+        plist_for_each_entry(p, head, pushable_tasks) {
+                if (pick_rt_task(rq, p, cpu))
+                        return p;
         }
 
-        return next;
+        return NULL;
 }
 
 static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask);
@@ -1743,12 +1655,10 @@ static int pull_rt_task(struct rq *this_rq)
                 double_lock_balance(this_rq, src_rq);
 
                 /*
-                 * Are there still pullable RT tasks?
+                 * We can pull only a task, which is pushable
+                 * on its rq, and no others.
                  */
-                if (src_rq->rt.rt_nr_running <= 1)
-                        goto skip;
-
-                p = pick_next_highest_task_rt(src_rq, this_cpu);
+                p = pick_highest_pushable_task(src_rq, this_cpu);
 
                 /*
                  * Do we have an RT task that preempts
@@ -2037,7 +1947,7 @@ static void set_curr_task_rt(struct rq *rq)
 {
         struct task_struct *p = rq->curr;
 
-        p->se.exec_start = rq->clock_task;
+        p->se.exec_start = rq_clock_task(rq);
 
         /* The running task is never eligible for pushing */
         dequeue_pushable_task(rq, p);