diff options
Diffstat (limited to 'kernel/sched/core.c')
-rw-r--r-- | kernel/sched/core.c | 107 |
1 files changed, 69 insertions, 38 deletions
diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 1612578a5b7a..1f37fe7f77a4 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c | |||
@@ -119,7 +119,9 @@ void update_rq_clock(struct rq *rq) | |||
119 | { | 119 | { |
120 | s64 delta; | 120 | s64 delta; |
121 | 121 | ||
122 | if (rq->skip_clock_update > 0) | 122 | lockdep_assert_held(&rq->lock); |
123 | |||
124 | if (rq->clock_skip_update & RQCF_ACT_SKIP) | ||
123 | return; | 125 | return; |
124 | 126 | ||
125 | delta = sched_clock_cpu(cpu_of(rq)) - rq->clock; | 127 | delta = sched_clock_cpu(cpu_of(rq)) - rq->clock; |
@@ -490,6 +492,11 @@ static __init void init_hrtick(void) | |||
490 | */ | 492 | */ |
491 | void hrtick_start(struct rq *rq, u64 delay) | 493 | void hrtick_start(struct rq *rq, u64 delay) |
492 | { | 494 | { |
495 | /* | ||
496 | * Don't schedule slices shorter than 10000ns, that just | ||
497 | * doesn't make sense. Rely on vruntime for fairness. | ||
498 | */ | ||
499 | delay = max_t(u64, delay, 10000LL); | ||
493 | __hrtimer_start_range_ns(&rq->hrtick_timer, ns_to_ktime(delay), 0, | 500 | __hrtimer_start_range_ns(&rq->hrtick_timer, ns_to_ktime(delay), 0, |
494 | HRTIMER_MODE_REL_PINNED, 0); | 501 | HRTIMER_MODE_REL_PINNED, 0); |
495 | } | 502 | } |
@@ -1046,7 +1053,7 @@ void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags) | |||
1046 | * this case, we can save a useless back to back clock update. | 1053 | * this case, we can save a useless back to back clock update. |
1047 | */ | 1054 | */ |
1048 | if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr)) | 1055 | if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr)) |
1049 | rq->skip_clock_update = 1; | 1056 | rq_clock_skip_update(rq, true); |
1050 | } | 1057 | } |
1051 | 1058 | ||
1052 | #ifdef CONFIG_SMP | 1059 | #ifdef CONFIG_SMP |
@@ -1836,6 +1843,9 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p) | |||
1836 | p->se.prev_sum_exec_runtime = 0; | 1843 | p->se.prev_sum_exec_runtime = 0; |
1837 | p->se.nr_migrations = 0; | 1844 | p->se.nr_migrations = 0; |
1838 | p->se.vruntime = 0; | 1845 | p->se.vruntime = 0; |
1846 | #ifdef CONFIG_SMP | ||
1847 | p->se.avg.decay_count = 0; | ||
1848 | #endif | ||
1839 | INIT_LIST_HEAD(&p->se.group_node); | 1849 | INIT_LIST_HEAD(&p->se.group_node); |
1840 | 1850 | ||
1841 | #ifdef CONFIG_SCHEDSTATS | 1851 | #ifdef CONFIG_SCHEDSTATS |
@@ -2755,6 +2765,10 @@ again: | |||
2755 | * - explicit schedule() call | 2765 | * - explicit schedule() call |
2756 | * - return from syscall or exception to user-space | 2766 | * - return from syscall or exception to user-space |
2757 | * - return from interrupt-handler to user-space | 2767 | * - return from interrupt-handler to user-space |
2768 | * | ||
2769 | * WARNING: all callers must re-check need_resched() afterward and reschedule | ||
2770 | * accordingly in case an event triggered the need for rescheduling (such as | ||
2771 | * an interrupt waking up a task) while preemption was disabled in __schedule(). | ||
2758 | */ | 2772 | */ |
2759 | static void __sched __schedule(void) | 2773 | static void __sched __schedule(void) |
2760 | { | 2774 | { |
@@ -2763,7 +2777,6 @@ static void __sched __schedule(void) | |||
2763 | struct rq *rq; | 2777 | struct rq *rq; |
2764 | int cpu; | 2778 | int cpu; |
2765 | 2779 | ||
2766 | need_resched: | ||
2767 | preempt_disable(); | 2780 | preempt_disable(); |
2768 | cpu = smp_processor_id(); | 2781 | cpu = smp_processor_id(); |
2769 | rq = cpu_rq(cpu); | 2782 | rq = cpu_rq(cpu); |
@@ -2783,6 +2796,8 @@ need_resched: | |||
2783 | smp_mb__before_spinlock(); | 2796 | smp_mb__before_spinlock(); |
2784 | raw_spin_lock_irq(&rq->lock); | 2797 | raw_spin_lock_irq(&rq->lock); |
2785 | 2798 | ||
2799 | rq->clock_skip_update <<= 1; /* promote REQ to ACT */ | ||
2800 | |||
2786 | switch_count = &prev->nivcsw; | 2801 | switch_count = &prev->nivcsw; |
2787 | if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) { | 2802 | if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) { |
2788 | if (unlikely(signal_pending_state(prev->state, prev))) { | 2803 | if (unlikely(signal_pending_state(prev->state, prev))) { |
@@ -2807,13 +2822,13 @@ need_resched: | |||
2807 | switch_count = &prev->nvcsw; | 2822 | switch_count = &prev->nvcsw; |
2808 | } | 2823 | } |
2809 | 2824 | ||
2810 | if (task_on_rq_queued(prev) || rq->skip_clock_update < 0) | 2825 | if (task_on_rq_queued(prev)) |
2811 | update_rq_clock(rq); | 2826 | update_rq_clock(rq); |
2812 | 2827 | ||
2813 | next = pick_next_task(rq, prev); | 2828 | next = pick_next_task(rq, prev); |
2814 | clear_tsk_need_resched(prev); | 2829 | clear_tsk_need_resched(prev); |
2815 | clear_preempt_need_resched(); | 2830 | clear_preempt_need_resched(); |
2816 | rq->skip_clock_update = 0; | 2831 | rq->clock_skip_update = 0; |
2817 | 2832 | ||
2818 | if (likely(prev != next)) { | 2833 | if (likely(prev != next)) { |
2819 | rq->nr_switches++; | 2834 | rq->nr_switches++; |
@@ -2828,8 +2843,6 @@ need_resched: | |||
2828 | post_schedule(rq); | 2843 | post_schedule(rq); |
2829 | 2844 | ||
2830 | sched_preempt_enable_no_resched(); | 2845 | sched_preempt_enable_no_resched(); |
2831 | if (need_resched()) | ||
2832 | goto need_resched; | ||
2833 | } | 2846 | } |
2834 | 2847 | ||
2835 | static inline void sched_submit_work(struct task_struct *tsk) | 2848 | static inline void sched_submit_work(struct task_struct *tsk) |
@@ -2849,7 +2862,9 @@ asmlinkage __visible void __sched schedule(void) | |||
2849 | struct task_struct *tsk = current; | 2862 | struct task_struct *tsk = current; |
2850 | 2863 | ||
2851 | sched_submit_work(tsk); | 2864 | sched_submit_work(tsk); |
2852 | __schedule(); | 2865 | do { |
2866 | __schedule(); | ||
2867 | } while (need_resched()); | ||
2853 | } | 2868 | } |
2854 | EXPORT_SYMBOL(schedule); | 2869 | EXPORT_SYMBOL(schedule); |
2855 | 2870 | ||
@@ -2884,6 +2899,21 @@ void __sched schedule_preempt_disabled(void) | |||
2884 | preempt_disable(); | 2899 | preempt_disable(); |
2885 | } | 2900 | } |
2886 | 2901 | ||
2902 | static void preempt_schedule_common(void) | ||
2903 | { | ||
2904 | do { | ||
2905 | __preempt_count_add(PREEMPT_ACTIVE); | ||
2906 | __schedule(); | ||
2907 | __preempt_count_sub(PREEMPT_ACTIVE); | ||
2908 | |||
2909 | /* | ||
2910 | * Check again in case we missed a preemption opportunity | ||
2911 | * between schedule and now. | ||
2912 | */ | ||
2913 | barrier(); | ||
2914 | } while (need_resched()); | ||
2915 | } | ||
2916 | |||
2887 | #ifdef CONFIG_PREEMPT | 2917 | #ifdef CONFIG_PREEMPT |
2888 | /* | 2918 | /* |
2889 | * this is the entry point to schedule() from in-kernel preemption | 2919 | * this is the entry point to schedule() from in-kernel preemption |
@@ -2899,17 +2929,7 @@ asmlinkage __visible void __sched notrace preempt_schedule(void) | |||
2899 | if (likely(!preemptible())) | 2929 | if (likely(!preemptible())) |
2900 | return; | 2930 | return; |
2901 | 2931 | ||
2902 | do { | 2932 | preempt_schedule_common(); |
2903 | __preempt_count_add(PREEMPT_ACTIVE); | ||
2904 | __schedule(); | ||
2905 | __preempt_count_sub(PREEMPT_ACTIVE); | ||
2906 | |||
2907 | /* | ||
2908 | * Check again in case we missed a preemption opportunity | ||
2909 | * between schedule and now. | ||
2910 | */ | ||
2911 | barrier(); | ||
2912 | } while (need_resched()); | ||
2913 | } | 2933 | } |
2914 | NOKPROBE_SYMBOL(preempt_schedule); | 2934 | NOKPROBE_SYMBOL(preempt_schedule); |
2915 | EXPORT_SYMBOL(preempt_schedule); | 2935 | EXPORT_SYMBOL(preempt_schedule); |
@@ -3405,6 +3425,20 @@ static bool check_same_owner(struct task_struct *p) | |||
3405 | return match; | 3425 | return match; |
3406 | } | 3426 | } |
3407 | 3427 | ||
3428 | static bool dl_param_changed(struct task_struct *p, | ||
3429 | const struct sched_attr *attr) | ||
3430 | { | ||
3431 | struct sched_dl_entity *dl_se = &p->dl; | ||
3432 | |||
3433 | if (dl_se->dl_runtime != attr->sched_runtime || | ||
3434 | dl_se->dl_deadline != attr->sched_deadline || | ||
3435 | dl_se->dl_period != attr->sched_period || | ||
3436 | dl_se->flags != attr->sched_flags) | ||
3437 | return true; | ||
3438 | |||
3439 | return false; | ||
3440 | } | ||
3441 | |||
3408 | static int __sched_setscheduler(struct task_struct *p, | 3442 | static int __sched_setscheduler(struct task_struct *p, |
3409 | const struct sched_attr *attr, | 3443 | const struct sched_attr *attr, |
3410 | bool user) | 3444 | bool user) |
@@ -3533,7 +3567,7 @@ recheck: | |||
3533 | goto change; | 3567 | goto change; |
3534 | if (rt_policy(policy) && attr->sched_priority != p->rt_priority) | 3568 | if (rt_policy(policy) && attr->sched_priority != p->rt_priority) |
3535 | goto change; | 3569 | goto change; |
3536 | if (dl_policy(policy)) | 3570 | if (dl_policy(policy) && dl_param_changed(p, attr)) |
3537 | goto change; | 3571 | goto change; |
3538 | 3572 | ||
3539 | p->sched_reset_on_fork = reset_on_fork; | 3573 | p->sched_reset_on_fork = reset_on_fork; |
@@ -4225,17 +4259,10 @@ SYSCALL_DEFINE0(sched_yield) | |||
4225 | return 0; | 4259 | return 0; |
4226 | } | 4260 | } |
4227 | 4261 | ||
4228 | static void __cond_resched(void) | ||
4229 | { | ||
4230 | __preempt_count_add(PREEMPT_ACTIVE); | ||
4231 | __schedule(); | ||
4232 | __preempt_count_sub(PREEMPT_ACTIVE); | ||
4233 | } | ||
4234 | |||
4235 | int __sched _cond_resched(void) | 4262 | int __sched _cond_resched(void) |
4236 | { | 4263 | { |
4237 | if (should_resched()) { | 4264 | if (should_resched()) { |
4238 | __cond_resched(); | 4265 | preempt_schedule_common(); |
4239 | return 1; | 4266 | return 1; |
4240 | } | 4267 | } |
4241 | return 0; | 4268 | return 0; |
@@ -4260,7 +4287,7 @@ int __cond_resched_lock(spinlock_t *lock) | |||
4260 | if (spin_needbreak(lock) || resched) { | 4287 | if (spin_needbreak(lock) || resched) { |
4261 | spin_unlock(lock); | 4288 | spin_unlock(lock); |
4262 | if (resched) | 4289 | if (resched) |
4263 | __cond_resched(); | 4290 | preempt_schedule_common(); |
4264 | else | 4291 | else |
4265 | cpu_relax(); | 4292 | cpu_relax(); |
4266 | ret = 1; | 4293 | ret = 1; |
@@ -4276,7 +4303,7 @@ int __sched __cond_resched_softirq(void) | |||
4276 | 4303 | ||
4277 | if (should_resched()) { | 4304 | if (should_resched()) { |
4278 | local_bh_enable(); | 4305 | local_bh_enable(); |
4279 | __cond_resched(); | 4306 | preempt_schedule_common(); |
4280 | local_bh_disable(); | 4307 | local_bh_disable(); |
4281 | return 1; | 4308 | return 1; |
4282 | } | 4309 | } |
@@ -4531,9 +4558,10 @@ void sched_show_task(struct task_struct *p) | |||
4531 | { | 4558 | { |
4532 | unsigned long free = 0; | 4559 | unsigned long free = 0; |
4533 | int ppid; | 4560 | int ppid; |
4534 | unsigned state; | 4561 | unsigned long state = p->state; |
4535 | 4562 | ||
4536 | state = p->state ? __ffs(p->state) + 1 : 0; | 4563 | if (state) |
4564 | state = __ffs(state) + 1; | ||
4537 | printk(KERN_INFO "%-15.15s %c", p->comm, | 4565 | printk(KERN_INFO "%-15.15s %c", p->comm, |
4538 | state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?'); | 4566 | state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?'); |
4539 | #if BITS_PER_LONG == 32 | 4567 | #if BITS_PER_LONG == 32 |
@@ -4766,7 +4794,7 @@ static struct rq *move_queued_task(struct task_struct *p, int new_cpu) | |||
4766 | 4794 | ||
4767 | void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) | 4795 | void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) |
4768 | { | 4796 | { |
4769 | if (p->sched_class && p->sched_class->set_cpus_allowed) | 4797 | if (p->sched_class->set_cpus_allowed) |
4770 | p->sched_class->set_cpus_allowed(p, new_mask); | 4798 | p->sched_class->set_cpus_allowed(p, new_mask); |
4771 | 4799 | ||
4772 | cpumask_copy(&p->cpus_allowed, new_mask); | 4800 | cpumask_copy(&p->cpus_allowed, new_mask); |
@@ -7276,6 +7304,11 @@ void __init sched_init(void) | |||
7276 | enter_lazy_tlb(&init_mm, current); | 7304 | enter_lazy_tlb(&init_mm, current); |
7277 | 7305 | ||
7278 | /* | 7306 | /* |
7307 | * During early bootup we pretend to be a normal task: | ||
7308 | */ | ||
7309 | current->sched_class = &fair_sched_class; | ||
7310 | |||
7311 | /* | ||
7279 | * Make us the idle thread. Technically, schedule() should not be | 7312 | * Make us the idle thread. Technically, schedule() should not be |
7280 | * called from this thread, however somewhere below it might be, | 7313 | * called from this thread, however somewhere below it might be, |
7281 | * but because we are the idle thread, we just pick up running again | 7314 | * but because we are the idle thread, we just pick up running again |
@@ -7285,11 +7318,6 @@ void __init sched_init(void) | |||
7285 | 7318 | ||
7286 | calc_load_update = jiffies + LOAD_FREQ; | 7319 | calc_load_update = jiffies + LOAD_FREQ; |
7287 | 7320 | ||
7288 | /* | ||
7289 | * During early bootup we pretend to be a normal task: | ||
7290 | */ | ||
7291 | current->sched_class = &fair_sched_class; | ||
7292 | |||
7293 | #ifdef CONFIG_SMP | 7321 | #ifdef CONFIG_SMP |
7294 | zalloc_cpumask_var(&sched_domains_tmpmask, GFP_NOWAIT); | 7322 | zalloc_cpumask_var(&sched_domains_tmpmask, GFP_NOWAIT); |
7295 | /* May be allocated at isolcpus cmdline parse time */ | 7323 | /* May be allocated at isolcpus cmdline parse time */ |
@@ -7350,6 +7378,9 @@ void ___might_sleep(const char *file, int line, int preempt_offset) | |||
7350 | in_atomic(), irqs_disabled(), | 7378 | in_atomic(), irqs_disabled(), |
7351 | current->pid, current->comm); | 7379 | current->pid, current->comm); |
7352 | 7380 | ||
7381 | if (task_stack_end_corrupted(current)) | ||
7382 | printk(KERN_EMERG "Thread overran stack, or stack corrupted\n"); | ||
7383 | |||
7353 | debug_show_held_locks(current); | 7384 | debug_show_held_locks(current); |
7354 | if (irqs_disabled()) | 7385 | if (irqs_disabled()) |
7355 | print_irqtrace_events(current); | 7386 | print_irqtrace_events(current); |