1 files changed, 64 insertions, 28 deletions

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 123673291ffb..20b858f2db22 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -511,7 +511,7 @@ static bool set_nr_and_not_polling(struct task_struct *p)
 static bool set_nr_if_polling(struct task_struct *p)
 {
         struct thread_info *ti = task_thread_info(p);
-        typeof(ti->flags) old, val = ACCESS_ONCE(ti->flags);
+        typeof(ti->flags) old, val = READ_ONCE(ti->flags);
 
         for (;;) {
                 if (!(val & _TIF_POLLING_NRFLAG))
@@ -541,6 +541,52 @@ static bool set_nr_if_polling(struct task_struct *p)
 #endif
 #endif
 
+void wake_q_add(struct wake_q_head *head, struct task_struct *task)
+{
+        struct wake_q_node *node = &task->wake_q;
+
+        /*
+         * Atomically grab the task, if ->wake_q is !nil already it means
+         * its already queued (either by us or someone else) and will get the
+         * wakeup due to that.
+         *
+         * This cmpxchg() implies a full barrier, which pairs with the write
+         * barrier implied by the wakeup in wake_up_list().
+         */
+        if (cmpxchg(&node->next, NULL, WAKE_Q_TAIL))
+                return;
+
+        get_task_struct(task);
+
+        /*
+         * The head is context local, there can be no concurrency.
+         */
+        *head->lastp = node;
+        head->lastp = &node->next;
+}
+
+void wake_up_q(struct wake_q_head *head)
+{
+        struct wake_q_node *node = head->first;
+
+        while (node != WAKE_Q_TAIL) {
+                struct task_struct *task;
+
+                task = container_of(node, struct task_struct, wake_q);
+                BUG_ON(!task);
+                /* task can safely be re-inserted now */
+                node = node->next;
+                task->wake_q.next = NULL;
+
+                /*
+                 * wake_up_process() implies a wmb() to pair with the queueing
+                 * in wake_q_add() so as not to miss wakeups.
+                 */
+                wake_up_process(task);
+                put_task_struct(task);
+        }
+}
+
 /*
  * resched_curr - mark rq's current task 'to be rescheduled now'.
  *
@@ -2397,9 +2443,9 @@ unsigned long nr_iowait_cpu(int cpu)
 
 void get_iowait_load(unsigned long *nr_waiters, unsigned long *load)
 {
-        struct rq *this = this_rq();
-        *nr_waiters = atomic_read(&this->nr_iowait);
-        *load = this->cpu_load[0];
+        struct rq *rq = this_rq();
+        *nr_waiters = atomic_read(&rq->nr_iowait);
+        *load = rq->load.weight;
 }
 
 #ifdef CONFIG_SMP
@@ -2497,6 +2543,7 @@ void scheduler_tick(void)
         update_rq_clock(rq);
         curr->sched_class->task_tick(rq, curr, 0);
         update_cpu_load_active(rq);
+        calc_global_load_tick(rq);
         raw_spin_unlock(&rq->lock);
 
         perf_event_task_tick();
@@ -2525,7 +2572,7 @@ void scheduler_tick(void)
 u64 scheduler_tick_max_deferment(void)
 {
         struct rq *rq = this_rq();
-        unsigned long next, now = ACCESS_ONCE(jiffies);
+        unsigned long next, now = READ_ONCE(jiffies);
 
         next = rq->last_sched_tick + HZ;
 
@@ -2726,9 +2773,7 @@ again:
  *          - return from syscall or exception to user-space
  *          - return from interrupt-handler to user-space
  *
- * WARNING: all callers must re-check need_resched() afterward and reschedule
- * accordingly in case an event triggered the need for rescheduling (such as
- * an interrupt waking up a task) while preemption was disabled in __schedule().
+ * WARNING: must be called with preemption disabled!
  */
 static void __sched __schedule(void)
 {
@@ -2737,7 +2782,6 @@ static void __sched __schedule(void)
         struct rq *rq;
         int cpu;
 
-        preempt_disable();
         cpu = smp_processor_id();
         rq = cpu_rq(cpu);
         rcu_note_context_switch();
@@ -2801,8 +2845,6 @@ static void __sched __schedule(void)
                 raw_spin_unlock_irq(&rq->lock);
 
         post_schedule(rq);
-
-        sched_preempt_enable_no_resched();
 }
 
 static inline void sched_submit_work(struct task_struct *tsk)
@@ -2823,7 +2865,9 @@ asmlinkage __visible void __sched schedule(void)
 
         sched_submit_work(tsk);
         do {
+                preempt_disable();
                 __schedule();
+                sched_preempt_enable_no_resched();
         } while (need_resched());
 }
 EXPORT_SYMBOL(schedule);
@@ -2862,15 +2906,14 @@ void __sched schedule_preempt_disabled(void)
 static void __sched notrace preempt_schedule_common(void)
 {
         do {
-                __preempt_count_add(PREEMPT_ACTIVE);
+                preempt_active_enter();
                 __schedule();
-                __preempt_count_sub(PREEMPT_ACTIVE);
+                preempt_active_exit();
 
                 /*
                  * Check again in case we missed a preemption opportunity
                  * between schedule and now.
                  */
-                barrier();
         } while (need_resched());
 }
 
@@ -2917,7 +2960,7 @@ asmlinkage __visible void __sched notrace preempt_schedule_context(void)
                 return;
 
         do {
-                __preempt_count_add(PREEMPT_ACTIVE);
+                preempt_active_enter();
                 /*
                  * Needs preempt disabled in case user_exit() is traced
                  * and the tracer calls preempt_enable_notrace() causing
@@ -2927,8 +2970,7 @@ asmlinkage __visible void __sched notrace preempt_schedule_context(void)
                 __schedule();
                 exception_exit(prev_ctx);
 
-                __preempt_count_sub(PREEMPT_ACTIVE);
-                barrier();
+                preempt_active_exit();
         } while (need_resched());
 }
 EXPORT_SYMBOL_GPL(preempt_schedule_context);
@@ -2952,17 +2994,11 @@ asmlinkage __visible void __sched preempt_schedule_irq(void)
         prev_state = exception_enter();
 
         do {
-                __preempt_count_add(PREEMPT_ACTIVE);
+                preempt_active_enter();
                 local_irq_enable();
                 __schedule();
                 local_irq_disable();
-                __preempt_count_sub(PREEMPT_ACTIVE);
-
-                /*
-                 * Check again in case we missed a preemption opportunity
-                 * between schedule and now.
-                 */
-                barrier();
+                preempt_active_exit();
         } while (need_resched());
 
         exception_exit(prev_state);
@@ -5314,7 +5350,7 @@ static struct notifier_block migration_notifier = {
         .priority = CPU_PRI_MIGRATION,
 };
 
-static void __cpuinit set_cpu_rq_start_time(void)
+static void set_cpu_rq_start_time(void)
 {
         int cpu = smp_processor_id();
         struct rq *rq = cpu_rq(cpu);
@@ -7734,11 +7770,11 @@ static long sched_group_rt_runtime(struct task_group *tg)
         return rt_runtime_us;
 }
 
-static int sched_group_set_rt_period(struct task_group *tg, long rt_period_us)
+static int sched_group_set_rt_period(struct task_group *tg, u64 rt_period_us)
 {
         u64 rt_runtime, rt_period;
 
-        rt_period = (u64)rt_period_us * NSEC_PER_USEC;
+        rt_period = rt_period_us * NSEC_PER_USEC;
         rt_runtime = tg->rt_bandwidth.rt_runtime;
 
         return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);