Merge branch 'sched-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull scheduler updates from Ingo Molnar:

 - Move the nohz kick code out of the scheduler tick to a dedicated IPI,
   from Frederic Weisbecker.

  This necessiated quite some background infrastructure rework,
  including:

   * Clean up some irq-work internals
   * Implement remote irq-work
   * Implement nohz kick on top of remote irq-work
   * Move full dynticks timer enqueue notification to new kick
   * Move multi-task notification to new kick
   * Remove unecessary barriers on multi-task notification

 - Remove proliferation of wait_on_bit() action functions and allow
   wait_on_bit_action() functions to support a timeout.  (Neil Brown)

 - Another round of sched/numa improvements, cleanups and fixes.  (Rik
   van Riel)

 - Implement fast idling of CPUs when the system is partially loaded,
   for better scalability.  (Tim Chen)

 - Restructure and fix the CPU hotplug handling code that may leave
   cfs_rq and rt_rq's throttled when tasks are migrated away from a dead
   cpu.  (Kirill Tkhai)

 - Robustify the sched topology setup code.  (Peterz Zijlstra)

 - Improve sched_feat() handling wrt.  static_keys (Jason Baron)

 - Misc fixes.

* 'sched-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (37 commits)
  sched/fair: Fix 'make xmldocs' warning caused by missing description
  sched: Use macro for magic number of -1 for setparam
  sched: Robustify topology setup
  sched: Fix sched_setparam() policy == -1 logic
  sched: Allow wait_on_bit_action() functions to support a timeout
  sched: Remove proliferation of wait_on_bit() action functions
  sched/numa: Revert "Use effective_load() to balance NUMA loads"
  sched: Fix static_key race with sched_feat()
  sched: Remove extra static_key*() function indirection
  sched/rt: Fix replenish_dl_entity() comments to match the current upstream code
  sched: Transform resched_task() into resched_curr()
  sched/deadline: Kill task_struct->pi_top_task
  sched: Rework check_for_tasks()
  sched/rt: Enqueue just unthrottled rt_rq back on the stack in __disable_runtime()
  sched/fair: Disable runtime_enabled on dying rq
  sched/numa: Change scan period code to match intent
  sched/numa: Rework best node setting in task_numa_migrate()
  sched/numa: Examine a task move when examining a task swap
  sched/numa: Simplify task_numa_compare()
  sched/numa: Use effective_load() to balance NUMA loads
  ...

1 files changed, 17 insertions, 13 deletions

diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index a49083192c64..5f6edca4fafd 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -463,9 +463,10 @@ static void dequeue_rt_entity(struct sched_rt_entity *rt_se);
 static void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
 {
         struct task_struct *curr = rq_of_rt_rq(rt_rq)->curr;
+        struct rq *rq = rq_of_rt_rq(rt_rq);
         struct sched_rt_entity *rt_se;
 
-        int cpu = cpu_of(rq_of_rt_rq(rt_rq));
+        int cpu = cpu_of(rq);
 
         rt_se = rt_rq->tg->rt_se[cpu];
 
@@ -476,7 +477,7 @@ static void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
                         enqueue_rt_entity(rt_se, false);
 
                 if (rt_rq->highest_prio.curr < curr->prio)
-                        resched_task(curr);
+                        resched_curr(rq);
         }
 }
 
@@ -566,7 +567,7 @@ static inline void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
                 return;
 
         enqueue_top_rt_rq(rt_rq);
-        resched_task(rq->curr);
+        resched_curr(rq);
 }
 
 static inline void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
@@ -740,6 +741,9 @@ balanced:
                 rt_rq->rt_throttled = 0;
                 raw_spin_unlock(&rt_rq->rt_runtime_lock);
                 raw_spin_unlock(&rt_b->rt_runtime_lock);
+
+                /* Make rt_rq available for pick_next_task() */
+                sched_rt_rq_enqueue(rt_rq);
         }
 }
 
@@ -948,7 +952,7 @@ static void update_curr_rt(struct rq *rq)
                         raw_spin_lock(&rt_rq->rt_runtime_lock);
                         rt_rq->rt_time += delta_exec;
                         if (sched_rt_runtime_exceeded(rt_rq))
-                                resched_task(curr);
+                                resched_curr(rq);
                         raw_spin_unlock(&rt_rq->rt_runtime_lock);
                 }
         }
@@ -1363,7 +1367,7 @@ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)
          * to try and push current away:
          */
         requeue_task_rt(rq, p, 1);
-        resched_task(rq->curr);
+        resched_curr(rq);
 }
 
 #endif /* CONFIG_SMP */
@@ -1374,7 +1378,7 @@ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)
 static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p, int flags)
 {
         if (p->prio < rq->curr->prio) {
-                resched_task(rq->curr);
+                resched_curr(rq);
                 return;
         }
 
@@ -1690,7 +1694,7 @@ retry:
          * just reschedule current.
          */
         if (unlikely(next_task->prio < rq->curr->prio)) {
-                resched_task(rq->curr);
+                resched_curr(rq);
                 return 0;
         }
 
@@ -1737,7 +1741,7 @@ retry:
         activate_task(lowest_rq, next_task, 0);
         ret = 1;
 
-        resched_task(lowest_rq->curr);
+        resched_curr(lowest_rq);
 
         double_unlock_balance(rq, lowest_rq);
 
@@ -1936,7 +1940,7 @@ static void switched_from_rt(struct rq *rq, struct task_struct *p)
                 return;
 
         if (pull_rt_task(rq))
-                resched_task(rq->curr);
+                resched_curr(rq);
 }
 
 void __init init_sched_rt_class(void)
@@ -1974,7 +1978,7 @@ static void switched_to_rt(struct rq *rq, struct task_struct *p)
                         check_resched = 0;
 #endif /* CONFIG_SMP */
                 if (check_resched && p->prio < rq->curr->prio)
-                        resched_task(rq->curr);
+                        resched_curr(rq);
         }
 }
 
@@ -2003,11 +2007,11 @@ prio_changed_rt(struct rq *rq, struct task_struct *p, int oldprio)
                  * Only reschedule if p is still on the same runqueue.
                  */
                 if (p->prio > rq->rt.highest_prio.curr && rq->curr == p)
-                        resched_task(p);
+                        resched_curr(rq);
 #else
                 /* For UP simply resched on drop of prio */
                 if (oldprio < p->prio)
-                        resched_task(p);
+                        resched_curr(rq);
 #endif /* CONFIG_SMP */
         } else {
                 /*
@@ -2016,7 +2020,7 @@ prio_changed_rt(struct rq *rq, struct task_struct *p, int oldprio)
                  * then reschedule.
                  */
                 if (p->prio < rq->curr->prio)
-                        resched_task(rq->curr);
+                        resched_curr(rq);
         }
 }