1 files changed, 42 insertions, 88 deletions

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index ead464a0f2e5..102dfcf0a29a 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -792,10 +792,14 @@ void activate_task(struct rq *rq, struct task_struct *p, int flags)
                 rq->nr_uninterruptible--;
 
         enqueue_task(rq, p, flags);
+
+        p->on_rq = TASK_ON_RQ_QUEUED;
 }
 
 void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
 {
+        p->on_rq = (flags & DEQUEUE_SLEEP) ? 0 : TASK_ON_RQ_MIGRATING;
+
         if (task_contributes_to_load(p))
                 rq->nr_uninterruptible++;
 
@@ -920,7 +924,7 @@ static inline bool is_per_cpu_kthread(struct task_struct *p)
 }
 
 /*
- * Per-CPU kthreads are allowed to run on !actie && online CPUs, see
+ * Per-CPU kthreads are allowed to run on !active && online CPUs, see
  * __set_cpus_allowed_ptr() and select_fallback_rq().
  */
 static inline bool is_cpu_allowed(struct task_struct *p, int cpu)
@@ -1151,7 +1155,6 @@ static int __set_cpus_allowed_ptr(struct task_struct *p,
                 /* Need help from migration thread: drop lock and wait. */
                 task_rq_unlock(rq, p, &rf);
                 stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
-                tlb_migrate_finish(p->mm);
                 return 0;
         } else if (task_on_rq_queued(p)) {
                 /*
@@ -1237,11 +1240,9 @@ static void __migrate_swap_task(struct task_struct *p, int cpu)
                 rq_pin_lock(src_rq, &srf);
                 rq_pin_lock(dst_rq, &drf);
 
-                p->on_rq = TASK_ON_RQ_MIGRATING;
                 deactivate_task(src_rq, p, 0);
                 set_task_cpu(p, cpu);
                 activate_task(dst_rq, p, 0);
-                p->on_rq = TASK_ON_RQ_QUEUED;
                 check_preempt_curr(dst_rq, p, 0);
 
                 rq_unpin_lock(dst_rq, &drf);
@@ -1681,16 +1682,6 @@ ttwu_stat(struct task_struct *p, int cpu, int wake_flags)
                 __schedstat_inc(p->se.statistics.nr_wakeups_sync);
 }
 
-static inline void ttwu_activate(struct rq *rq, struct task_struct *p, int en_flags)
-{
-        activate_task(rq, p, en_flags);
-        p->on_rq = TASK_ON_RQ_QUEUED;
-
-        /* If a worker is waking up, notify the workqueue: */
-        if (p->flags & PF_WQ_WORKER)
-                wq_worker_waking_up(p, cpu_of(rq));
-}
-
 /*
  * Mark the task runnable and perform wakeup-preemption.
  */
@@ -1742,7 +1733,7 @@ ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags,
                 en_flags |= ENQUEUE_MIGRATED;
 #endif
 
-        ttwu_activate(rq, p, en_flags);
+        activate_task(rq, p, en_flags);
         ttwu_do_wakeup(rq, p, wake_flags, rf);
 }
 
@@ -2107,56 +2098,6 @@ out:
 }
 
 /**
- * try_to_wake_up_local - try to wake up a local task with rq lock held
- * @p: the thread to be awakened
- * @rf: request-queue flags for pinning
- *
- * Put @p on the run-queue if it's not already there. The caller must
- * ensure that this_rq() is locked, @p is bound to this_rq() and not
- * the current task.
- */
-static void try_to_wake_up_local(struct task_struct *p, struct rq_flags *rf)
-{
-        struct rq *rq = task_rq(p);
-
-        if (WARN_ON_ONCE(rq != this_rq()) ||
-            WARN_ON_ONCE(p == current))
-                return;
-
-        lockdep_assert_held(&rq->lock);
-
-        if (!raw_spin_trylock(&p->pi_lock)) {
-                /*
-                 * This is OK, because current is on_cpu, which avoids it being
-                 * picked for load-balance and preemption/IRQs are still
-                 * disabled avoiding further scheduler activity on it and we've
-                 * not yet picked a replacement task.
-                 */
-                rq_unlock(rq, rf);
-                raw_spin_lock(&p->pi_lock);
-                rq_relock(rq, rf);
-        }
-
-        if (!(p->state & TASK_NORMAL))
-                goto out;
-
-        trace_sched_waking(p);
-
-        if (!task_on_rq_queued(p)) {
-                if (p->in_iowait) {
-                        delayacct_blkio_end(p);
-                        atomic_dec(&rq->nr_iowait);
-                }
-                ttwu_activate(rq, p, ENQUEUE_WAKEUP | ENQUEUE_NOCLOCK);
-        }
-
-        ttwu_do_wakeup(rq, p, 0, rf);
-        ttwu_stat(p, smp_processor_id(), 0);
-out:
-        raw_spin_unlock(&p->pi_lock);
-}
-
-/**
  * wake_up_process - Wake up a specific process
  * @p: The process to be woken up.
  *
@@ -2467,7 +2408,6 @@ void wake_up_new_task(struct task_struct *p)
         post_init_entity_util_avg(p);
 
         activate_task(rq, p, ENQUEUE_NOCLOCK);
-        p->on_rq = TASK_ON_RQ_QUEUED;
         trace_sched_wakeup_new(p);
         check_preempt_curr(rq, p, WF_FORK);
 #ifdef CONFIG_SMP
@@ -3466,25 +3406,11 @@ static void __sched notrace __schedule(bool preempt)
                         prev->state = TASK_RUNNING;
                 } else {
                         deactivate_task(rq, prev, DEQUEUE_SLEEP | DEQUEUE_NOCLOCK);
-                        prev->on_rq = 0;
 
                         if (prev->in_iowait) {
                                 atomic_inc(&rq->nr_iowait);
                                 delayacct_blkio_start();
                         }
-
-                        /*
-                         * If a worker went to sleep, notify and ask workqueue
-                         * whether it wants to wake up a task to maintain
-                         * concurrency.
-                         */
-                        if (prev->flags & PF_WQ_WORKER) {
-                                struct task_struct *to_wakeup;
-
-                                to_wakeup = wq_worker_sleeping(prev);
-                                if (to_wakeup)
-                                        try_to_wake_up_local(to_wakeup, &rf);
-                        }
                 }
                 switch_count = &prev->nvcsw;
         }
@@ -3544,6 +3470,20 @@ static inline void sched_submit_work(struct task_struct *tsk)
 {
         if (!tsk->state || tsk_is_pi_blocked(tsk))
                 return;
+
+        /*
+         * If a worker went to sleep, notify and ask workqueue whether
+         * it wants to wake up a task to maintain concurrency.
+         * As this function is called inside the schedule() context,
+         * we disable preemption to avoid it calling schedule() again
+         * in the possible wakeup of a kworker.
+         */
+        if (tsk->flags & PF_WQ_WORKER) {
+                preempt_disable();
+                wq_worker_sleeping(tsk);
+                preempt_enable_no_resched();
+        }
+
         /*
          * If we are going to sleep and we have plugged IO queued,
          * make sure to submit it to avoid deadlocks.
@@ -3552,6 +3492,12 @@ static inline void sched_submit_work(struct task_struct *tsk)
                 blk_schedule_flush_plug(tsk);
 }
 
+static void sched_update_worker(struct task_struct *tsk)
+{
+        if (tsk->flags & PF_WQ_WORKER)
+                wq_worker_running(tsk);
+}
+
 asmlinkage __visible void __sched schedule(void)
 {
         struct task_struct *tsk = current;
@@ -3562,6 +3508,7 @@ asmlinkage __visible void __sched schedule(void)
                 __schedule(false);
                 sched_preempt_enable_no_resched();
         } while (need_resched());
+        sched_update_worker(tsk);
 }
 EXPORT_SYMBOL(schedule);
 
@@ -5918,7 +5865,7 @@ void __init sched_init_smp(void)
 
 static int __init migration_init(void)
 {
-        sched_rq_cpu_starting(smp_processor_id());
+        sched_cpu_starting(smp_processor_id());
         return 0;
 }
 early_initcall(migration_init);
@@ -6559,6 +6506,8 @@ static void cpu_cgroup_attach(struct cgroup_taskset *tset)
 static int cpu_shares_write_u64(struct cgroup_subsys_state *css,
                                 struct cftype *cftype, u64 shareval)
 {
+        if (shareval > scale_load_down(ULONG_MAX))
+                shareval = MAX_SHARES;
         return sched_group_set_shares(css_tg(css), scale_load(shareval));
 }
 
@@ -6574,7 +6523,7 @@ static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css,
 static DEFINE_MUTEX(cfs_constraints_mutex);
 
 const u64 max_cfs_quota_period = 1 * NSEC_PER_SEC; /* 1s */
-const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */
+static const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */
 
 static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime);
 
@@ -6654,20 +6603,22 @@ out_unlock:
         return ret;
 }
 
-int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us)
+static int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us)
 {
         u64 quota, period;
 
         period = ktime_to_ns(tg->cfs_bandwidth.period);
         if (cfs_quota_us < 0)
                 quota = RUNTIME_INF;
-        else
+        else if ((u64)cfs_quota_us <= U64_MAX / NSEC_PER_USEC)
                 quota = (u64)cfs_quota_us * NSEC_PER_USEC;
+        else
+                return -EINVAL;
 
         return tg_set_cfs_bandwidth(tg, period, quota);
 }
 
-long tg_get_cfs_quota(struct task_group *tg)
+static long tg_get_cfs_quota(struct task_group *tg)
 {
         u64 quota_us;
 
@@ -6680,17 +6631,20 @@ long tg_get_cfs_quota(struct task_group *tg)
         return quota_us;
 }
 
-int tg_set_cfs_period(struct task_group *tg, long cfs_period_us)
+static int tg_set_cfs_period(struct task_group *tg, long cfs_period_us)
 {
         u64 quota, period;
 
+        if ((u64)cfs_period_us > U64_MAX / NSEC_PER_USEC)
+                return -EINVAL;
+
         period = (u64)cfs_period_us * NSEC_PER_USEC;
         quota = tg->cfs_bandwidth.quota;
 
         return tg_set_cfs_bandwidth(tg, period, quota);
 }
 
-long tg_get_cfs_period(struct task_group *tg)
+static long tg_get_cfs_period(struct task_group *tg)
 {
         u64 cfs_period_us;
 
@@ -6998,7 +6952,7 @@ static int __maybe_unused cpu_period_quota_parse(char *buf,
 {
         char tok[21];   /* U64_MAX */
 
-        if (!sscanf(buf, "%s %llu", tok, periodp))
+        if (sscanf(buf, "%20s %llu", tok, periodp) < 1)
                 return -EINVAL;
 
         *periodp *= NSEC_PER_USEC;