diff options
author | Jeremy Erickson <jerickso@cs.unc.edu> | 2012-08-30 21:01:47 -0400 |
---|---|---|
committer | Jeremy Erickson <jerickso@cs.unc.edu> | 2012-08-30 21:01:47 -0400 |
commit | b1e1fea67bca3796d5f9133a92c300ec4fa93a4f (patch) | |
tree | 5cc1336e1fe1d6f93b1067e73e43381dd20db690 /kernel/sched.c | |
parent | f6f94e2ab1b33f0082ac22d71f66385a60d8157f (diff) |
Bjoern's Dissertation Code with Priority Donationwip-splitting-omlp-jerickso
Diffstat (limited to 'kernel/sched.c')
-rw-r--r-- | kernel/sched.c | 127 |
1 files changed, 115 insertions, 12 deletions
diff --git a/kernel/sched.c b/kernel/sched.c index dc85ceb90832..1f5327f8c012 100644 --- a/kernel/sched.c +++ b/kernel/sched.c | |||
@@ -79,6 +79,11 @@ | |||
79 | #include "sched_cpupri.h" | 79 | #include "sched_cpupri.h" |
80 | #include "workqueue_sched.h" | 80 | #include "workqueue_sched.h" |
81 | 81 | ||
82 | #include <litmus/sched_trace.h> | ||
83 | #include <litmus/trace.h> | ||
84 | |||
85 | static void litmus_tick(struct rq*, struct task_struct*); | ||
86 | |||
82 | #define CREATE_TRACE_POINTS | 87 | #define CREATE_TRACE_POINTS |
83 | #include <trace/events/sched.h> | 88 | #include <trace/events/sched.h> |
84 | 89 | ||
@@ -405,6 +410,12 @@ struct rt_rq { | |||
405 | #endif | 410 | #endif |
406 | }; | 411 | }; |
407 | 412 | ||
413 | /* Litmus related fields in a runqueue */ | ||
414 | struct litmus_rq { | ||
415 | unsigned long nr_running; | ||
416 | struct task_struct *prev; | ||
417 | }; | ||
418 | |||
408 | #ifdef CONFIG_SMP | 419 | #ifdef CONFIG_SMP |
409 | 420 | ||
410 | /* | 421 | /* |
@@ -471,6 +482,7 @@ struct rq { | |||
471 | 482 | ||
472 | struct cfs_rq cfs; | 483 | struct cfs_rq cfs; |
473 | struct rt_rq rt; | 484 | struct rt_rq rt; |
485 | struct litmus_rq litmus; | ||
474 | 486 | ||
475 | #ifdef CONFIG_FAIR_GROUP_SCHED | 487 | #ifdef CONFIG_FAIR_GROUP_SCHED |
476 | /* list of leaf cfs_rq on this cpu: */ | 488 | /* list of leaf cfs_rq on this cpu: */ |
@@ -566,8 +578,14 @@ void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags) | |||
566 | * A queue event has occurred, and we're going to schedule. In | 578 | * A queue event has occurred, and we're going to schedule. In |
567 | * this case, we can save a useless back to back clock update. | 579 | * this case, we can save a useless back to back clock update. |
568 | */ | 580 | */ |
581 | /* LITMUS^RT: turning off the clock update is buggy in Linux 2.6.36; | ||
582 | * the scheduler can "forget" to renable the runqueue clock in some | ||
583 | * cases. LITMUS^RT amplifies the effects of this problem. Hence, we | ||
584 | * turn it off to avoid stalling clocks. */ | ||
585 | /* | ||
569 | if (test_tsk_need_resched(p)) | 586 | if (test_tsk_need_resched(p)) |
570 | rq->skip_clock_update = 1; | 587 | rq->skip_clock_update = 1; |
588 | */ | ||
571 | } | 589 | } |
572 | 590 | ||
573 | static inline int cpu_of(struct rq *rq) | 591 | static inline int cpu_of(struct rq *rq) |
@@ -1042,6 +1060,7 @@ static enum hrtimer_restart hrtick(struct hrtimer *timer) | |||
1042 | raw_spin_lock(&rq->lock); | 1060 | raw_spin_lock(&rq->lock); |
1043 | update_rq_clock(rq); | 1061 | update_rq_clock(rq); |
1044 | rq->curr->sched_class->task_tick(rq, rq->curr, 1); | 1062 | rq->curr->sched_class->task_tick(rq, rq->curr, 1); |
1063 | litmus_tick(rq, rq->curr); | ||
1045 | raw_spin_unlock(&rq->lock); | 1064 | raw_spin_unlock(&rq->lock); |
1046 | 1065 | ||
1047 | return HRTIMER_NORESTART; | 1066 | return HRTIMER_NORESTART; |
@@ -1840,7 +1859,7 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu) | |||
1840 | 1859 | ||
1841 | static const struct sched_class rt_sched_class; | 1860 | static const struct sched_class rt_sched_class; |
1842 | 1861 | ||
1843 | #define sched_class_highest (&rt_sched_class) | 1862 | #define sched_class_highest (&litmus_sched_class) |
1844 | #define for_each_class(class) \ | 1863 | #define for_each_class(class) \ |
1845 | for (class = sched_class_highest; class; class = class->next) | 1864 | for (class = sched_class_highest; class; class = class->next) |
1846 | 1865 | ||
@@ -1920,6 +1939,7 @@ static void deactivate_task(struct rq *rq, struct task_struct *p, int flags) | |||
1920 | #include "sched_idletask.c" | 1939 | #include "sched_idletask.c" |
1921 | #include "sched_fair.c" | 1940 | #include "sched_fair.c" |
1922 | #include "sched_rt.c" | 1941 | #include "sched_rt.c" |
1942 | #include "../litmus/sched_litmus.c" | ||
1923 | #ifdef CONFIG_SCHED_DEBUG | 1943 | #ifdef CONFIG_SCHED_DEBUG |
1924 | # include "sched_debug.c" | 1944 | # include "sched_debug.c" |
1925 | #endif | 1945 | #endif |
@@ -2352,6 +2372,9 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, | |||
2352 | unsigned long en_flags = ENQUEUE_WAKEUP; | 2372 | unsigned long en_flags = ENQUEUE_WAKEUP; |
2353 | struct rq *rq; | 2373 | struct rq *rq; |
2354 | 2374 | ||
2375 | if (is_realtime(p)) | ||
2376 | TRACE_TASK(p, "try_to_wake_up() state:%d\n", p->state); | ||
2377 | |||
2355 | this_cpu = get_cpu(); | 2378 | this_cpu = get_cpu(); |
2356 | 2379 | ||
2357 | smp_wmb(); | 2380 | smp_wmb(); |
@@ -2366,7 +2389,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, | |||
2366 | orig_cpu = cpu; | 2389 | orig_cpu = cpu; |
2367 | 2390 | ||
2368 | #ifdef CONFIG_SMP | 2391 | #ifdef CONFIG_SMP |
2369 | if (unlikely(task_running(rq, p))) | 2392 | if (unlikely(task_running(rq, p)) || is_realtime(p)) |
2370 | goto out_activate; | 2393 | goto out_activate; |
2371 | 2394 | ||
2372 | /* | 2395 | /* |
@@ -2428,6 +2451,8 @@ out_activate: | |||
2428 | out_running: | 2451 | out_running: |
2429 | ttwu_post_activation(p, rq, wake_flags, success); | 2452 | ttwu_post_activation(p, rq, wake_flags, success); |
2430 | out: | 2453 | out: |
2454 | if (is_realtime(p)) | ||
2455 | TRACE_TASK(p, "try_to_wake_up() done state:%d\n", p->state); | ||
2431 | task_rq_unlock(rq, &flags); | 2456 | task_rq_unlock(rq, &flags); |
2432 | put_cpu(); | 2457 | put_cpu(); |
2433 | 2458 | ||
@@ -2532,7 +2557,8 @@ void sched_fork(struct task_struct *p, int clone_flags) | |||
2532 | * Revert to default priority/policy on fork if requested. | 2557 | * Revert to default priority/policy on fork if requested. |
2533 | */ | 2558 | */ |
2534 | if (unlikely(p->sched_reset_on_fork)) { | 2559 | if (unlikely(p->sched_reset_on_fork)) { |
2535 | if (p->policy == SCHED_FIFO || p->policy == SCHED_RR) { | 2560 | if (p->policy == SCHED_FIFO || p->policy == SCHED_RR || |
2561 | p->policy == SCHED_LITMUS) { | ||
2536 | p->policy = SCHED_NORMAL; | 2562 | p->policy = SCHED_NORMAL; |
2537 | p->normal_prio = p->static_prio; | 2563 | p->normal_prio = p->static_prio; |
2538 | } | 2564 | } |
@@ -2748,6 +2774,8 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev) | |||
2748 | */ | 2774 | */ |
2749 | prev_state = prev->state; | 2775 | prev_state = prev->state; |
2750 | finish_arch_switch(prev); | 2776 | finish_arch_switch(prev); |
2777 | litmus->finish_switch(prev); | ||
2778 | prev->rt_param.stack_in_use = NO_CPU; | ||
2751 | #ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW | 2779 | #ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW |
2752 | local_irq_disable(); | 2780 | local_irq_disable(); |
2753 | #endif /* __ARCH_WANT_INTERRUPTS_ON_CTXSW */ | 2781 | #endif /* __ARCH_WANT_INTERRUPTS_ON_CTXSW */ |
@@ -2777,6 +2805,15 @@ static inline void pre_schedule(struct rq *rq, struct task_struct *prev) | |||
2777 | { | 2805 | { |
2778 | if (prev->sched_class->pre_schedule) | 2806 | if (prev->sched_class->pre_schedule) |
2779 | prev->sched_class->pre_schedule(rq, prev); | 2807 | prev->sched_class->pre_schedule(rq, prev); |
2808 | |||
2809 | /* LITMUS^RT not very clean hack: we need to save the prev task | ||
2810 | * as our scheduling decision rely on it (as we drop the rq lock | ||
2811 | * something in prev can change...); there is no way to escape | ||
2812 | * this ack apart from modifying pick_nex_task(rq, _prev_) or | ||
2813 | * falling back on the previous solution of decoupling | ||
2814 | * scheduling decisions | ||
2815 | */ | ||
2816 | rq->litmus.prev = prev; | ||
2780 | } | 2817 | } |
2781 | 2818 | ||
2782 | /* rq->lock is NOT held, but preemption is disabled */ | 2819 | /* rq->lock is NOT held, but preemption is disabled */ |
@@ -3578,18 +3615,26 @@ void scheduler_tick(void) | |||
3578 | 3615 | ||
3579 | sched_clock_tick(); | 3616 | sched_clock_tick(); |
3580 | 3617 | ||
3618 | TS_TICK_START(current); | ||
3619 | |||
3581 | raw_spin_lock(&rq->lock); | 3620 | raw_spin_lock(&rq->lock); |
3582 | update_rq_clock(rq); | 3621 | update_rq_clock(rq); |
3583 | update_cpu_load_active(rq); | 3622 | update_cpu_load_active(rq); |
3584 | curr->sched_class->task_tick(rq, curr, 0); | 3623 | curr->sched_class->task_tick(rq, curr, 0); |
3624 | |||
3625 | /* litmus_tick may force current to resched */ | ||
3626 | litmus_tick(rq, curr); | ||
3627 | |||
3585 | raw_spin_unlock(&rq->lock); | 3628 | raw_spin_unlock(&rq->lock); |
3586 | 3629 | ||
3587 | perf_event_task_tick(curr); | 3630 | perf_event_task_tick(curr); |
3588 | 3631 | ||
3589 | #ifdef CONFIG_SMP | 3632 | #ifdef CONFIG_SMP |
3590 | rq->idle_at_tick = idle_cpu(cpu); | 3633 | rq->idle_at_tick = idle_cpu(cpu); |
3591 | trigger_load_balance(rq, cpu); | 3634 | if (!is_realtime(current)) |
3635 | trigger_load_balance(rq, cpu); | ||
3592 | #endif | 3636 | #endif |
3637 | TS_TICK_END(current); | ||
3593 | } | 3638 | } |
3594 | 3639 | ||
3595 | notrace unsigned long get_parent_ip(unsigned long addr) | 3640 | notrace unsigned long get_parent_ip(unsigned long addr) |
@@ -3716,12 +3761,20 @@ pick_next_task(struct rq *rq) | |||
3716 | /* | 3761 | /* |
3717 | * Optimization: we know that if all tasks are in | 3762 | * Optimization: we know that if all tasks are in |
3718 | * the fair class we can call that function directly: | 3763 | * the fair class we can call that function directly: |
3719 | */ | 3764 | |
3720 | if (likely(rq->nr_running == rq->cfs.nr_running)) { | 3765 | * NOT IN LITMUS^RT! |
3766 | |||
3767 | * This breaks many assumptions in the plugins. | ||
3768 | * Do not uncomment without thinking long and hard | ||
3769 | * about how this affects global plugins such as GSN-EDF. | ||
3770 | |||
3771 | if (rq->nr_running == rq->cfs.nr_running) { | ||
3772 | TRACE("taking shortcut in pick_next_task()\n"); | ||
3721 | p = fair_sched_class.pick_next_task(rq); | 3773 | p = fair_sched_class.pick_next_task(rq); |
3722 | if (likely(p)) | 3774 | if (likely(p)) |
3723 | return p; | 3775 | return p; |
3724 | } | 3776 | } |
3777 | */ | ||
3725 | 3778 | ||
3726 | class = sched_class_highest; | 3779 | class = sched_class_highest; |
3727 | for ( ; ; ) { | 3780 | for ( ; ; ) { |
@@ -3748,6 +3801,7 @@ asmlinkage void __sched schedule(void) | |||
3748 | 3801 | ||
3749 | need_resched: | 3802 | need_resched: |
3750 | preempt_disable(); | 3803 | preempt_disable(); |
3804 | sched_state_entered_schedule(); | ||
3751 | cpu = smp_processor_id(); | 3805 | cpu = smp_processor_id(); |
3752 | rq = cpu_rq(cpu); | 3806 | rq = cpu_rq(cpu); |
3753 | rcu_note_context_switch(cpu); | 3807 | rcu_note_context_switch(cpu); |
@@ -3755,6 +3809,8 @@ need_resched: | |||
3755 | 3809 | ||
3756 | release_kernel_lock(prev); | 3810 | release_kernel_lock(prev); |
3757 | need_resched_nonpreemptible: | 3811 | need_resched_nonpreemptible: |
3812 | TS_SCHED_START; | ||
3813 | sched_trace_task_switch_away(prev); | ||
3758 | 3814 | ||
3759 | schedule_debug(prev); | 3815 | schedule_debug(prev); |
3760 | 3816 | ||
@@ -3803,7 +3859,10 @@ need_resched_nonpreemptible: | |||
3803 | rq->curr = next; | 3859 | rq->curr = next; |
3804 | ++*switch_count; | 3860 | ++*switch_count; |
3805 | 3861 | ||
3862 | TS_SCHED_END(next); | ||
3863 | TS_CXS_START(next); | ||
3806 | context_switch(rq, prev, next); /* unlocks the rq */ | 3864 | context_switch(rq, prev, next); /* unlocks the rq */ |
3865 | TS_CXS_END(current); | ||
3807 | /* | 3866 | /* |
3808 | * The context switch have flipped the stack from under us | 3867 | * The context switch have flipped the stack from under us |
3809 | * and restored the local variables which were saved when | 3868 | * and restored the local variables which were saved when |
@@ -3812,17 +3871,23 @@ need_resched_nonpreemptible: | |||
3812 | */ | 3871 | */ |
3813 | cpu = smp_processor_id(); | 3872 | cpu = smp_processor_id(); |
3814 | rq = cpu_rq(cpu); | 3873 | rq = cpu_rq(cpu); |
3815 | } else | 3874 | } else { |
3875 | TS_SCHED_END(prev); | ||
3816 | raw_spin_unlock_irq(&rq->lock); | 3876 | raw_spin_unlock_irq(&rq->lock); |
3877 | } | ||
3878 | |||
3879 | sched_trace_task_switch_to(current); | ||
3817 | 3880 | ||
3818 | post_schedule(rq); | 3881 | post_schedule(rq); |
3819 | 3882 | ||
3820 | if (unlikely(reacquire_kernel_lock(prev))) | 3883 | if (sched_state_validate_switch() || unlikely(reacquire_kernel_lock(prev))) |
3821 | goto need_resched_nonpreemptible; | 3884 | goto need_resched_nonpreemptible; |
3822 | 3885 | ||
3823 | preempt_enable_no_resched(); | 3886 | preempt_enable_no_resched(); |
3824 | if (need_resched()) | 3887 | if (need_resched()) |
3825 | goto need_resched; | 3888 | goto need_resched; |
3889 | |||
3890 | srp_ceiling_block(); | ||
3826 | } | 3891 | } |
3827 | EXPORT_SYMBOL(schedule); | 3892 | EXPORT_SYMBOL(schedule); |
3828 | 3893 | ||
@@ -4108,6 +4173,17 @@ void complete_all(struct completion *x) | |||
4108 | } | 4173 | } |
4109 | EXPORT_SYMBOL(complete_all); | 4174 | EXPORT_SYMBOL(complete_all); |
4110 | 4175 | ||
4176 | void complete_n(struct completion *x, int n) | ||
4177 | { | ||
4178 | unsigned long flags; | ||
4179 | |||
4180 | spin_lock_irqsave(&x->wait.lock, flags); | ||
4181 | x->done += n; | ||
4182 | __wake_up_common(&x->wait, TASK_NORMAL, n, 0, NULL); | ||
4183 | spin_unlock_irqrestore(&x->wait.lock, flags); | ||
4184 | } | ||
4185 | EXPORT_SYMBOL(complete_n); | ||
4186 | |||
4111 | static inline long __sched | 4187 | static inline long __sched |
4112 | do_wait_for_common(struct completion *x, long timeout, int state) | 4188 | do_wait_for_common(struct completion *x, long timeout, int state) |
4113 | { | 4189 | { |
@@ -4550,7 +4626,9 @@ __setscheduler(struct rq *rq, struct task_struct *p, int policy, int prio) | |||
4550 | p->normal_prio = normal_prio(p); | 4626 | p->normal_prio = normal_prio(p); |
4551 | /* we are holding p->pi_lock already */ | 4627 | /* we are holding p->pi_lock already */ |
4552 | p->prio = rt_mutex_getprio(p); | 4628 | p->prio = rt_mutex_getprio(p); |
4553 | if (rt_prio(p->prio)) | 4629 | if (p->policy == SCHED_LITMUS) |
4630 | p->sched_class = &litmus_sched_class; | ||
4631 | else if (rt_prio(p->prio)) | ||
4554 | p->sched_class = &rt_sched_class; | 4632 | p->sched_class = &rt_sched_class; |
4555 | else | 4633 | else |
4556 | p->sched_class = &fair_sched_class; | 4634 | p->sched_class = &fair_sched_class; |
@@ -4595,7 +4673,7 @@ recheck: | |||
4595 | 4673 | ||
4596 | if (policy != SCHED_FIFO && policy != SCHED_RR && | 4674 | if (policy != SCHED_FIFO && policy != SCHED_RR && |
4597 | policy != SCHED_NORMAL && policy != SCHED_BATCH && | 4675 | policy != SCHED_NORMAL && policy != SCHED_BATCH && |
4598 | policy != SCHED_IDLE) | 4676 | policy != SCHED_IDLE && policy != SCHED_LITMUS) |
4599 | return -EINVAL; | 4677 | return -EINVAL; |
4600 | } | 4678 | } |
4601 | 4679 | ||
@@ -4610,6 +4688,8 @@ recheck: | |||
4610 | return -EINVAL; | 4688 | return -EINVAL; |
4611 | if (rt_policy(policy) != (param->sched_priority != 0)) | 4689 | if (rt_policy(policy) != (param->sched_priority != 0)) |
4612 | return -EINVAL; | 4690 | return -EINVAL; |
4691 | if (policy == SCHED_LITMUS && policy == p->policy) | ||
4692 | return -EINVAL; | ||
4613 | 4693 | ||
4614 | /* | 4694 | /* |
4615 | * Allow unprivileged RT tasks to decrease priority: | 4695 | * Allow unprivileged RT tasks to decrease priority: |
@@ -4650,6 +4730,12 @@ recheck: | |||
4650 | return retval; | 4730 | return retval; |
4651 | } | 4731 | } |
4652 | 4732 | ||
4733 | if (policy == SCHED_LITMUS) { | ||
4734 | retval = litmus_admit_task(p); | ||
4735 | if (retval) | ||
4736 | return retval; | ||
4737 | } | ||
4738 | |||
4653 | /* | 4739 | /* |
4654 | * make sure no PI-waiters arrive (or leave) while we are | 4740 | * make sure no PI-waiters arrive (or leave) while we are |
4655 | * changing the priority of the task: | 4741 | * changing the priority of the task: |
@@ -4692,10 +4778,19 @@ recheck: | |||
4692 | 4778 | ||
4693 | p->sched_reset_on_fork = reset_on_fork; | 4779 | p->sched_reset_on_fork = reset_on_fork; |
4694 | 4780 | ||
4781 | if (p->policy == SCHED_LITMUS) | ||
4782 | litmus_exit_task(p); | ||
4783 | |||
4695 | oldprio = p->prio; | 4784 | oldprio = p->prio; |
4696 | prev_class = p->sched_class; | 4785 | prev_class = p->sched_class; |
4697 | __setscheduler(rq, p, policy, param->sched_priority); | 4786 | __setscheduler(rq, p, policy, param->sched_priority); |
4698 | 4787 | ||
4788 | if (policy == SCHED_LITMUS) { | ||
4789 | p->rt_param.stack_in_use = running ? rq->cpu : NO_CPU; | ||
4790 | p->rt_param.present = running; | ||
4791 | litmus->task_new(p, on_rq, running); | ||
4792 | } | ||
4793 | |||
4699 | if (running) | 4794 | if (running) |
4700 | p->sched_class->set_curr_task(rq); | 4795 | p->sched_class->set_curr_task(rq); |
4701 | if (on_rq) { | 4796 | if (on_rq) { |
@@ -4755,6 +4850,13 @@ do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param) | |||
4755 | if (copy_from_user(&lparam, param, sizeof(struct sched_param))) | 4850 | if (copy_from_user(&lparam, param, sizeof(struct sched_param))) |
4756 | return -EFAULT; | 4851 | return -EFAULT; |
4757 | 4852 | ||
4853 | #ifdef CONFIG_LITMUS_LOCKING | ||
4854 | /* Hack to allow plugin to call into schedule | ||
4855 | * prio to a setscheduler() call. */ | ||
4856 | if (is_realtime(current)) | ||
4857 | litmus->pre_setsched(current, policy); | ||
4858 | #endif | ||
4859 | |||
4758 | rcu_read_lock(); | 4860 | rcu_read_lock(); |
4759 | retval = -ESRCH; | 4861 | retval = -ESRCH; |
4760 | p = find_process_by_pid(pid); | 4862 | p = find_process_by_pid(pid); |
@@ -4865,10 +4967,11 @@ long sched_setaffinity(pid_t pid, const struct cpumask *in_mask) | |||
4865 | rcu_read_lock(); | 4967 | rcu_read_lock(); |
4866 | 4968 | ||
4867 | p = find_process_by_pid(pid); | 4969 | p = find_process_by_pid(pid); |
4868 | if (!p) { | 4970 | /* Don't set affinity if task not found and for LITMUS tasks */ |
4971 | if (!p || is_realtime(p)) { | ||
4869 | rcu_read_unlock(); | 4972 | rcu_read_unlock(); |
4870 | put_online_cpus(); | 4973 | put_online_cpus(); |
4871 | return -ESRCH; | 4974 | return p ? -EPERM : -ESRCH; |
4872 | } | 4975 | } |
4873 | 4976 | ||
4874 | /* Prevent p going away */ | 4977 | /* Prevent p going away */ |