1 files changed, 108 insertions, 55 deletions

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index b5797b78add6..1f37fe7f77a4 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -119,7 +119,9 @@ void update_rq_clock(struct rq *rq)
 {
         s64 delta;
 
-        if (rq->skip_clock_update > 0)
+        lockdep_assert_held(&rq->lock);
+
+        if (rq->clock_skip_update & RQCF_ACT_SKIP)
                 return;
 
         delta = sched_clock_cpu(cpu_of(rq)) - rq->clock;
@@ -490,6 +492,11 @@ static __init void init_hrtick(void)
  */
 void hrtick_start(struct rq *rq, u64 delay)
 {
+        /*
+         * Don't schedule slices shorter than 10000ns, that just
+         * doesn't make sense. Rely on vruntime for fairness.
+         */
+        delay = max_t(u64, delay, 10000LL);
         __hrtimer_start_range_ns(&rq->hrtick_timer, ns_to_ktime(delay), 0,
                         HRTIMER_MODE_REL_PINNED, 0);
 }
@@ -1046,7 +1053,7 @@ void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
          * this case, we can save a useless back to back clock update.
          */
         if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr))
-                rq->skip_clock_update = 1;
+                rq_clock_skip_update(rq, true);
 }
 
 #ifdef CONFIG_SMP
@@ -1082,7 +1089,7 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
                 if (p->sched_class->migrate_task_rq)
                         p->sched_class->migrate_task_rq(p, new_cpu);
                 p->se.nr_migrations++;
-                perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, NULL, 0);
+                perf_sw_event_sched(PERF_COUNT_SW_CPU_MIGRATIONS, 1, 0);
         }
 
         __set_task_cpu(p, new_cpu);
@@ -1814,6 +1821,10 @@ void __dl_clear_params(struct task_struct *p)
         dl_se->dl_period = 0;
         dl_se->flags = 0;
         dl_se->dl_bw = 0;
+
+        dl_se->dl_throttled = 0;
+        dl_se->dl_new = 1;
+        dl_se->dl_yielded = 0;
 }
 
 /*
@@ -1832,6 +1843,9 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
         p->se.prev_sum_exec_runtime     = 0;
         p->se.nr_migrations             = 0;
         p->se.vruntime                  = 0;
+#ifdef CONFIG_SMP
+        p->se.avg.decay_count           = 0;
+#endif
         INIT_LIST_HEAD(&p->se.group_node);
 
 #ifdef CONFIG_SCHEDSTATS
@@ -1839,7 +1853,7 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
 #endif
 
         RB_CLEAR_NODE(&p->dl.rb_node);
-        hrtimer_init(&p->dl.dl_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+        init_dl_task_timer(&p->dl);
         __dl_clear_params(p);
 
         INIT_LIST_HEAD(&p->rt.run_list);
@@ -2049,6 +2063,9 @@ static inline int dl_bw_cpus(int i)
  * allocated bandwidth to reflect the new situation.
  *
  * This function is called while holding p's rq->lock.
+ *
+ * XXX we should delay bw change until the task's 0-lag point, see
+ * __setparam_dl().
  */
 static int dl_overflow(struct task_struct *p, int policy,
                        const struct sched_attr *attr)
@@ -2748,6 +2765,10 @@ again:
  *          - explicit schedule() call
  *          - return from syscall or exception to user-space
  *          - return from interrupt-handler to user-space
+ *
+ * WARNING: all callers must re-check need_resched() afterward and reschedule
+ * accordingly in case an event triggered the need for rescheduling (such as
+ * an interrupt waking up a task) while preemption was disabled in __schedule().
  */
 static void __sched __schedule(void)
 {
@@ -2756,7 +2777,6 @@ static void __sched __schedule(void)
         struct rq *rq;
         int cpu;
 
-need_resched:
         preempt_disable();
         cpu = smp_processor_id();
         rq = cpu_rq(cpu);
@@ -2776,6 +2796,8 @@ need_resched:
         smp_mb__before_spinlock();
         raw_spin_lock_irq(&rq->lock);
 
+        rq->clock_skip_update <<= 1; /* promote REQ to ACT */
+
         switch_count = &prev->nivcsw;
         if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {
                 if (unlikely(signal_pending_state(prev->state, prev))) {
@@ -2800,13 +2822,13 @@ need_resched:
                 switch_count = &prev->nvcsw;
         }
 
-        if (task_on_rq_queued(prev) || rq->skip_clock_update < 0)
+        if (task_on_rq_queued(prev))
                 update_rq_clock(rq);
 
         next = pick_next_task(rq, prev);
         clear_tsk_need_resched(prev);
         clear_preempt_need_resched();
-        rq->skip_clock_update = 0;
+        rq->clock_skip_update = 0;
 
         if (likely(prev != next)) {
                 rq->nr_switches++;
@@ -2821,8 +2843,6 @@ need_resched:
         post_schedule(rq);
 
         sched_preempt_enable_no_resched();
-        if (need_resched())
-                goto need_resched;
 }
 
 static inline void sched_submit_work(struct task_struct *tsk)
@@ -2842,7 +2862,9 @@ asmlinkage __visible void __sched schedule(void)
         struct task_struct *tsk = current;
 
         sched_submit_work(tsk);
-        __schedule();
+        do {
+                __schedule();
+        } while (need_resched());
 }
 EXPORT_SYMBOL(schedule);
 
@@ -2877,6 +2899,21 @@ void __sched schedule_preempt_disabled(void)
         preempt_disable();
 }
 
+static void preempt_schedule_common(void)
+{
+        do {
+                __preempt_count_add(PREEMPT_ACTIVE);
+                __schedule();
+                __preempt_count_sub(PREEMPT_ACTIVE);
+
+                /*
+                 * Check again in case we missed a preemption opportunity
+                 * between schedule and now.
+                 */
+                barrier();
+        } while (need_resched());
+}
+
 #ifdef CONFIG_PREEMPT
 /*
  * this is the entry point to schedule() from in-kernel preemption
@@ -2892,17 +2929,7 @@ asmlinkage __visible void __sched notrace preempt_schedule(void)
         if (likely(!preemptible()))
                 return;
 
-        do {
-                __preempt_count_add(PREEMPT_ACTIVE);
-                __schedule();
-                __preempt_count_sub(PREEMPT_ACTIVE);
-
-                /*
-                 * Check again in case we missed a preemption opportunity
-                 * between schedule and now.
-                 */
-                barrier();
-        } while (need_resched());
+        preempt_schedule_common();
 }
 NOKPROBE_SYMBOL(preempt_schedule);
 EXPORT_SYMBOL(preempt_schedule);
@@ -3251,15 +3278,31 @@ __setparam_dl(struct task_struct *p, const struct sched_attr *attr)
 {
         struct sched_dl_entity *dl_se = &p->dl;
 
-        init_dl_task_timer(dl_se);
         dl_se->dl_runtime = attr->sched_runtime;
         dl_se->dl_deadline = attr->sched_deadline;
         dl_se->dl_period = attr->sched_period ?: dl_se->dl_deadline;
         dl_se->flags = attr->sched_flags;
         dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime);
-        dl_se->dl_throttled = 0;
-        dl_se->dl_new = 1;
-        dl_se->dl_yielded = 0;
+
+        /*
+         * Changing the parameters of a task is 'tricky' and we're not doing
+         * the correct thing -- also see task_dead_dl() and switched_from_dl().
+         *
+         * What we SHOULD do is delay the bandwidth release until the 0-lag
+         * point. This would include retaining the task_struct until that time
+         * and change dl_overflow() to not immediately decrement the current
+         * amount.
+         *
+         * Instead we retain the current runtime/deadline and let the new
+         * parameters take effect after the current reservation period lapses.
+         * This is safe (albeit pessimistic) because the 0-lag point is always
+         * before the current scheduling deadline.
+         *
+         * We can still have temporary overloads because we do not delay the
+         * change in bandwidth until that time; so admission control is
+         * not on the safe side. It does however guarantee tasks will never
+         * consume more than promised.
+         */
 }
 
 /*
@@ -3382,6 +3425,20 @@ static bool check_same_owner(struct task_struct *p)
         return match;
 }
 
+static bool dl_param_changed(struct task_struct *p,
+                const struct sched_attr *attr)
+{
+        struct sched_dl_entity *dl_se = &p->dl;
+
+        if (dl_se->dl_runtime != attr->sched_runtime ||
+                dl_se->dl_deadline != attr->sched_deadline ||
+                dl_se->dl_period != attr->sched_period ||
+                dl_se->flags != attr->sched_flags)
+                return true;
+
+        return false;
+}
+
 static int __sched_setscheduler(struct task_struct *p,
                                 const struct sched_attr *attr,
                                 bool user)
@@ -3510,7 +3567,7 @@ recheck:
                         goto change;
                 if (rt_policy(policy) && attr->sched_priority != p->rt_priority)
                         goto change;
-                if (dl_policy(policy))
+                if (dl_policy(policy) && dl_param_changed(p, attr))
                         goto change;
 
                 p->sched_reset_on_fork = reset_on_fork;
@@ -4202,17 +4259,10 @@ SYSCALL_DEFINE0(sched_yield)
         return 0;
 }
 
-static void __cond_resched(void)
-{
-        __preempt_count_add(PREEMPT_ACTIVE);
-        __schedule();
-        __preempt_count_sub(PREEMPT_ACTIVE);
-}
-
 int __sched _cond_resched(void)
 {
         if (should_resched()) {
-                __cond_resched();
+                preempt_schedule_common();
                 return 1;
         }
         return 0;
@@ -4237,7 +4287,7 @@ int __cond_resched_lock(spinlock_t *lock)
         if (spin_needbreak(lock) || resched) {
                 spin_unlock(lock);
                 if (resched)
-                        __cond_resched();
+                        preempt_schedule_common();
                 else
                         cpu_relax();
                 ret = 1;
@@ -4253,7 +4303,7 @@ int __sched __cond_resched_softirq(void)
 
         if (should_resched()) {
                 local_bh_enable();
-                __cond_resched();
+                preempt_schedule_common();
                 local_bh_disable();
                 return 1;
         }
@@ -4508,9 +4558,10 @@ void sched_show_task(struct task_struct *p)
 {
         unsigned long free = 0;
         int ppid;
-        unsigned state;
+        unsigned long state = p->state;
 
-        state = p->state ? __ffs(p->state) + 1 : 0;
+        if (state)
+                state = __ffs(state) + 1;
         printk(KERN_INFO "%-15.15s %c", p->comm,
                 state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
 #if BITS_PER_LONG == 32
@@ -4642,6 +4693,9 @@ int cpuset_cpumask_can_shrink(const struct cpumask *cur,
         struct dl_bw *cur_dl_b;
         unsigned long flags;
 
+        if (!cpumask_weight(cur))
+                return ret;
+
         rcu_read_lock_sched();
         cur_dl_b = dl_bw_of(cpumask_any(cur));
         trial_cpus = cpumask_weight(trial);
@@ -4740,7 +4794,7 @@ static struct rq *move_queued_task(struct task_struct *p, int new_cpu)
 
 void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
 {
-        if (p->sched_class && p->sched_class->set_cpus_allowed)
+        if (p->sched_class->set_cpus_allowed)
                 p->sched_class->set_cpus_allowed(p, new_mask);
 
         cpumask_copy(&p->cpus_allowed, new_mask);
@@ -7113,9 +7167,6 @@ void __init sched_init(void)
 #ifdef CONFIG_RT_GROUP_SCHED
         alloc_size += 2 * nr_cpu_ids * sizeof(void **);
 #endif
-#ifdef CONFIG_CPUMASK_OFFSTACK
-        alloc_size += num_possible_cpus() * cpumask_size();
-#endif
         if (alloc_size) {
                 ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT);
 
@@ -7135,13 +7186,13 @@ void __init sched_init(void)
                 ptr += nr_cpu_ids * sizeof(void **);
 
 #endif /* CONFIG_RT_GROUP_SCHED */
+        }
 #ifdef CONFIG_CPUMASK_OFFSTACK
-                for_each_possible_cpu(i) {
-                        per_cpu(load_balance_mask, i) = (void *)ptr;
-                        ptr += cpumask_size();
-                }
-#endif /* CONFIG_CPUMASK_OFFSTACK */
+        for_each_possible_cpu(i) {
+                per_cpu(load_balance_mask, i) = (cpumask_var_t)kzalloc_node(
+                        cpumask_size(), GFP_KERNEL, cpu_to_node(i));
         }
+#endif /* CONFIG_CPUMASK_OFFSTACK */
 
         init_rt_bandwidth(&def_rt_bandwidth,
                         global_rt_period(), global_rt_runtime());
@@ -7253,6 +7304,11 @@ void __init sched_init(void)
         enter_lazy_tlb(&init_mm, current);
 
         /*
+         * During early bootup we pretend to be a normal task:
+         */
+        current->sched_class = &fair_sched_class;
+
+        /*
          * Make us the idle thread. Technically, schedule() should not be
          * called from this thread, however somewhere below it might be,
          * but because we are the idle thread, we just pick up running again
@@ -7262,11 +7318,6 @@ void __init sched_init(void)
 
         calc_load_update = jiffies + LOAD_FREQ;
 
-        /*
-         * During early bootup we pretend to be a normal task:
-         */
-        current->sched_class = &fair_sched_class;
-
 #ifdef CONFIG_SMP
         zalloc_cpumask_var(&sched_domains_tmpmask, GFP_NOWAIT);
         /* May be allocated at isolcpus cmdline parse time */
@@ -7295,13 +7346,12 @@ void __might_sleep(const char *file, int line, int preempt_offset)
          * since we will exit with TASK_RUNNING make sure we enter with it,
          * otherwise we will destroy state.
          */
-        if (WARN_ONCE(current->state != TASK_RUNNING,
+        WARN_ONCE(current->state != TASK_RUNNING && current->task_state_change,
                         "do not call blocking ops when !TASK_RUNNING; "
                         "state=%lx set at [<%p>] %pS\n",
                         current->state,
                         (void *)current->task_state_change,
-                        (void *)current->task_state_change))
-                __set_current_state(TASK_RUNNING);
+                        (void *)current->task_state_change);
 
         ___might_sleep(file, line, preempt_offset);
 }
@@ -7328,6 +7378,9 @@ void ___might_sleep(const char *file, int line, int preempt_offset)
                         in_atomic(), irqs_disabled(),
                         current->pid, current->comm);
 
+        if (task_stack_end_corrupted(current))
+                printk(KERN_EMERG "Thread overran stack, or stack corrupted\n");
+
         debug_show_held_locks(current);
         if (irqs_disabled())
                 print_irqtrace_events(current);