diff options
author | Glenn Elliott <gelliott@cs.unc.edu> | 2013-01-10 16:21:07 -0500 |
---|---|---|
committer | Glenn Elliott <gelliott@cs.unc.edu> | 2013-01-10 16:21:07 -0500 |
commit | 8d00682ce5ddaedfb62287773d21c727f08fda69 (patch) | |
tree | 61a4b7bac5960c6f0ab25fe087404e9ca1725e05 /include | |
parent | fdf0a6b73001976c5d02d631ebdd0927819d7c91 (diff) | |
parent | 1235a665a5e00dc762e6646c01381b3ed5019d86 (diff) |
Merge branch 'wip-gpu-cleanup' into wip-2012.3-gpu
Conflicts:
include/litmus/fpmath.h
include/litmus/litmus.h
include/litmus/rt_param.h
include/litmus/trace.h
kernel/sched.c
kernel/softirq.c
litmus/edf_common.c
litmus/jobs.c
litmus/litmus.c
litmus/locking.c
litmus/preempt.c
litmus/sched_cedf.c
litmus/sched_gsn_edf.c
litmus/sched_litmus.c
litmus/sync.c
Diffstat (limited to 'include')
29 files changed, 1557 insertions, 44 deletions
diff --git a/include/linux/completion.h b/include/linux/completion.h index 51494e6b5548..a64fb5680400 100644 --- a/include/linux/completion.h +++ b/include/linux/completion.h | |||
@@ -76,6 +76,7 @@ static inline void init_completion(struct completion *x) | |||
76 | init_waitqueue_head(&x->wait); | 76 | init_waitqueue_head(&x->wait); |
77 | } | 77 | } |
78 | 78 | ||
79 | extern void __wait_for_completion_locked(struct completion *); | ||
79 | extern void wait_for_completion(struct completion *); | 80 | extern void wait_for_completion(struct completion *); |
80 | extern int wait_for_completion_interruptible(struct completion *x); | 81 | extern int wait_for_completion_interruptible(struct completion *x); |
81 | extern int wait_for_completion_killable(struct completion *x); | 82 | extern int wait_for_completion_killable(struct completion *x); |
diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index f6efed0039ed..9fc31289a1bb 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h | |||
@@ -445,6 +445,7 @@ static inline void __raise_softirq_irqoff(unsigned int nr) | |||
445 | 445 | ||
446 | extern void raise_softirq_irqoff(unsigned int nr); | 446 | extern void raise_softirq_irqoff(unsigned int nr); |
447 | extern void raise_softirq(unsigned int nr); | 447 | extern void raise_softirq(unsigned int nr); |
448 | extern void wakeup_softirqd(void); | ||
448 | 449 | ||
449 | /* This is the worklist that queues up per-cpu softirq work. | 450 | /* This is the worklist that queues up per-cpu softirq work. |
450 | * | 451 | * |
@@ -500,6 +501,16 @@ struct tasklet_struct | |||
500 | atomic_t count; | 501 | atomic_t count; |
501 | void (*func)(unsigned long); | 502 | void (*func)(unsigned long); |
502 | unsigned long data; | 503 | unsigned long data; |
504 | |||
505 | #if defined(CONFIG_LITMUS_SOFTIRQD) || defined(CONFIG_LITMUS_PAI_SOFTIRQD) | ||
506 | struct task_struct *owner; | ||
507 | #endif | ||
508 | }; | ||
509 | |||
510 | struct tasklet_head | ||
511 | { | ||
512 | struct tasklet_struct *head; | ||
513 | struct tasklet_struct **tail; | ||
503 | }; | 514 | }; |
504 | 515 | ||
505 | #define DECLARE_TASKLET(name, func, data) \ | 516 | #define DECLARE_TASKLET(name, func, data) \ |
@@ -537,6 +548,7 @@ static inline void tasklet_unlock_wait(struct tasklet_struct *t) | |||
537 | #define tasklet_unlock(t) do { } while (0) | 548 | #define tasklet_unlock(t) do { } while (0) |
538 | #endif | 549 | #endif |
539 | 550 | ||
551 | extern void ___tasklet_schedule(struct tasklet_struct *t); | ||
540 | extern void __tasklet_schedule(struct tasklet_struct *t); | 552 | extern void __tasklet_schedule(struct tasklet_struct *t); |
541 | 553 | ||
542 | static inline void tasklet_schedule(struct tasklet_struct *t) | 554 | static inline void tasklet_schedule(struct tasklet_struct *t) |
@@ -545,6 +557,7 @@ static inline void tasklet_schedule(struct tasklet_struct *t) | |||
545 | __tasklet_schedule(t); | 557 | __tasklet_schedule(t); |
546 | } | 558 | } |
547 | 559 | ||
560 | extern void ___tasklet_hi_schedule(struct tasklet_struct *t); | ||
548 | extern void __tasklet_hi_schedule(struct tasklet_struct *t); | 561 | extern void __tasklet_hi_schedule(struct tasklet_struct *t); |
549 | 562 | ||
550 | static inline void tasklet_hi_schedule(struct tasklet_struct *t) | 563 | static inline void tasklet_hi_schedule(struct tasklet_struct *t) |
@@ -553,6 +566,7 @@ static inline void tasklet_hi_schedule(struct tasklet_struct *t) | |||
553 | __tasklet_hi_schedule(t); | 566 | __tasklet_hi_schedule(t); |
554 | } | 567 | } |
555 | 568 | ||
569 | extern void ___tasklet_hi_schedule_first(struct tasklet_struct *t); | ||
556 | extern void __tasklet_hi_schedule_first(struct tasklet_struct *t); | 570 | extern void __tasklet_hi_schedule_first(struct tasklet_struct *t); |
557 | 571 | ||
558 | /* | 572 | /* |
@@ -582,7 +596,7 @@ static inline void tasklet_disable(struct tasklet_struct *t) | |||
582 | } | 596 | } |
583 | 597 | ||
584 | static inline void tasklet_enable(struct tasklet_struct *t) | 598 | static inline void tasklet_enable(struct tasklet_struct *t) |
585 | { | 599 | { |
586 | smp_mb__before_atomic_dec(); | 600 | smp_mb__before_atomic_dec(); |
587 | atomic_dec(&t->count); | 601 | atomic_dec(&t->count); |
588 | } | 602 | } |
diff --git a/include/linux/mutex.h b/include/linux/mutex.h index a940fe435aca..cb47debbf24d 100644 --- a/include/linux/mutex.h +++ b/include/linux/mutex.h | |||
@@ -126,6 +126,15 @@ static inline int mutex_is_locked(struct mutex *lock) | |||
126 | return atomic_read(&lock->count) != 1; | 126 | return atomic_read(&lock->count) != 1; |
127 | } | 127 | } |
128 | 128 | ||
129 | /* return non-zero to abort. only pre-side-effects may abort */ | ||
130 | typedef int (*side_effect_t)(unsigned long); | ||
131 | extern void mutex_lock_sfx(struct mutex *lock, | ||
132 | side_effect_t pre, unsigned long pre_arg, | ||
133 | side_effect_t post, unsigned long post_arg); | ||
134 | extern void mutex_unlock_sfx(struct mutex *lock, | ||
135 | side_effect_t pre, unsigned long pre_arg, | ||
136 | side_effect_t post, unsigned long post_arg); | ||
137 | |||
129 | /* | 138 | /* |
130 | * See kernel/mutex.c for detailed documentation of these APIs. | 139 | * See kernel/mutex.c for detailed documentation of these APIs. |
131 | * Also see Documentation/mutex-design.txt. | 140 | * Also see Documentation/mutex-design.txt. |
@@ -153,6 +162,7 @@ extern void mutex_lock(struct mutex *lock); | |||
153 | extern int __must_check mutex_lock_interruptible(struct mutex *lock); | 162 | extern int __must_check mutex_lock_interruptible(struct mutex *lock); |
154 | extern int __must_check mutex_lock_killable(struct mutex *lock); | 163 | extern int __must_check mutex_lock_killable(struct mutex *lock); |
155 | 164 | ||
165 | |||
156 | # define mutex_lock_nested(lock, subclass) mutex_lock(lock) | 166 | # define mutex_lock_nested(lock, subclass) mutex_lock(lock) |
157 | # define mutex_lock_interruptible_nested(lock, subclass) mutex_lock_interruptible(lock) | 167 | # define mutex_lock_interruptible_nested(lock, subclass) mutex_lock_interruptible(lock) |
158 | # define mutex_lock_killable_nested(lock, subclass) mutex_lock_killable(lock) | 168 | # define mutex_lock_killable_nested(lock, subclass) mutex_lock_killable(lock) |
diff --git a/include/linux/sched.h b/include/linux/sched.h index 9c990d13ae35..d580959f9f5c 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h | |||
@@ -1532,8 +1532,12 @@ struct task_struct { | |||
1532 | #endif | 1532 | #endif |
1533 | struct prop_local_single dirties; | 1533 | struct prop_local_single dirties; |
1534 | 1534 | ||
1535 | /* LITMUS RT parameters and state */ | 1535 | /*** LITMUS RT parameters and state ***/ |
1536 | struct rt_param rt_param; | 1536 | struct rt_param rt_param; |
1537 | #ifdef CONFIG_REALTIME_AUX_TASKS | ||
1538 | struct aux_data aux_data; | ||
1539 | #endif | ||
1540 | /*****/ | ||
1537 | 1541 | ||
1538 | /* references to PI semaphores, etc. */ | 1542 | /* references to PI semaphores, etc. */ |
1539 | struct od_table_entry *od_table; | 1543 | struct od_table_entry *od_table; |
diff --git a/include/linux/semaphore.h b/include/linux/semaphore.h index 39fa04966aa8..c83fc2b65f01 100644 --- a/include/linux/semaphore.h +++ b/include/linux/semaphore.h | |||
@@ -43,4 +43,13 @@ extern int __must_check down_trylock(struct semaphore *sem); | |||
43 | extern int __must_check down_timeout(struct semaphore *sem, long jiffies); | 43 | extern int __must_check down_timeout(struct semaphore *sem, long jiffies); |
44 | extern void up(struct semaphore *sem); | 44 | extern void up(struct semaphore *sem); |
45 | 45 | ||
46 | extern void __down(struct semaphore *sem); | ||
47 | extern void __up(struct semaphore *sem); | ||
48 | |||
49 | struct semaphore_waiter { | ||
50 | struct list_head list; | ||
51 | struct task_struct *task; | ||
52 | int up; | ||
53 | }; | ||
54 | |||
46 | #endif /* __LINUX_SEMAPHORE_H */ | 55 | #endif /* __LINUX_SEMAPHORE_H */ |
diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h index f584aba78ca9..1ec2ec7d4e3b 100644 --- a/include/linux/workqueue.h +++ b/include/linux/workqueue.h | |||
@@ -83,6 +83,9 @@ struct work_struct { | |||
83 | #ifdef CONFIG_LOCKDEP | 83 | #ifdef CONFIG_LOCKDEP |
84 | struct lockdep_map lockdep_map; | 84 | struct lockdep_map lockdep_map; |
85 | #endif | 85 | #endif |
86 | #ifdef CONFIG_LITMUS_SOFTIRQD | ||
87 | struct task_struct *owner; | ||
88 | #endif | ||
86 | }; | 89 | }; |
87 | 90 | ||
88 | #define WORK_DATA_INIT() ATOMIC_LONG_INIT(WORK_STRUCT_NO_CPU) | 91 | #define WORK_DATA_INIT() ATOMIC_LONG_INIT(WORK_STRUCT_NO_CPU) |
@@ -115,11 +118,25 @@ struct execute_work { | |||
115 | #define __WORK_INIT_LOCKDEP_MAP(n, k) | 118 | #define __WORK_INIT_LOCKDEP_MAP(n, k) |
116 | #endif | 119 | #endif |
117 | 120 | ||
121 | #ifdef CONFIG_LITMUS_SOFTIRQD | ||
122 | #define __WORK_INIT_OWNER() \ | ||
123 | .owner = NULL, | ||
124 | |||
125 | #define PREPARE_OWNER(_work, _owner) \ | ||
126 | do { \ | ||
127 | (_work)->owner = (_owner); \ | ||
128 | } while(0) | ||
129 | #else | ||
130 | #define __WORK_INIT_OWNER() | ||
131 | #define PREPARE_OWNER(_work, _owner) | ||
132 | #endif | ||
133 | |||
118 | #define __WORK_INITIALIZER(n, f) { \ | 134 | #define __WORK_INITIALIZER(n, f) { \ |
119 | .data = WORK_DATA_STATIC_INIT(), \ | 135 | .data = WORK_DATA_STATIC_INIT(), \ |
120 | .entry = { &(n).entry, &(n).entry }, \ | 136 | .entry = { &(n).entry, &(n).entry }, \ |
121 | .func = (f), \ | 137 | .func = (f), \ |
122 | __WORK_INIT_LOCKDEP_MAP(#n, &(n)) \ | 138 | __WORK_INIT_LOCKDEP_MAP(#n, &(n)) \ |
139 | __WORK_INIT_OWNER() \ | ||
123 | } | 140 | } |
124 | 141 | ||
125 | #define __DELAYED_WORK_INITIALIZER(n, f) { \ | 142 | #define __DELAYED_WORK_INITIALIZER(n, f) { \ |
@@ -357,6 +374,7 @@ extern int queue_delayed_work_on(int cpu, struct workqueue_struct *wq, | |||
357 | extern void flush_workqueue(struct workqueue_struct *wq); | 374 | extern void flush_workqueue(struct workqueue_struct *wq); |
358 | extern void flush_scheduled_work(void); | 375 | extern void flush_scheduled_work(void); |
359 | 376 | ||
377 | extern int __schedule_work(struct work_struct *work); | ||
360 | extern int schedule_work(struct work_struct *work); | 378 | extern int schedule_work(struct work_struct *work); |
361 | extern int schedule_work_on(int cpu, struct work_struct *work); | 379 | extern int schedule_work_on(int cpu, struct work_struct *work); |
362 | extern int schedule_delayed_work(struct delayed_work *work, unsigned long delay); | 380 | extern int schedule_delayed_work(struct delayed_work *work, unsigned long delay); |
diff --git a/include/litmus/aux_tasks.h b/include/litmus/aux_tasks.h new file mode 100644 index 000000000000..255bbafcc6b7 --- /dev/null +++ b/include/litmus/aux_tasks.h | |||
@@ -0,0 +1,23 @@ | |||
1 | #ifndef LITMUS_AUX_taskS | ||
2 | #define LITMUS_AUX_taskS | ||
3 | |||
4 | struct task_struct; | ||
5 | |||
6 | int make_aux_task_if_required(struct task_struct *t); | ||
7 | |||
8 | /* call on an aux task when it exits real-time */ | ||
9 | int exit_aux_task(struct task_struct *t); | ||
10 | |||
11 | /* call when an aux_owner becomes real-time */ | ||
12 | long enable_aux_task_owner(struct task_struct *t); | ||
13 | |||
14 | /* call when an aux_owner exits real-time */ | ||
15 | long disable_aux_task_owner(struct task_struct *t); | ||
16 | |||
17 | /* call when an aux_owner increases its priority */ | ||
18 | int aux_task_owner_increase_priority(struct task_struct *t); | ||
19 | |||
20 | /* call when an aux_owner decreases its priority */ | ||
21 | int aux_task_owner_decrease_priority(struct task_struct *t); | ||
22 | |||
23 | #endif | ||
diff --git a/include/litmus/budget.h b/include/litmus/budget.h index 33344ee8d5f9..763b31c0e9f6 100644 --- a/include/litmus/budget.h +++ b/include/litmus/budget.h | |||
@@ -5,6 +5,9 @@ | |||
5 | * the next task. */ | 5 | * the next task. */ |
6 | void update_enforcement_timer(struct task_struct* t); | 6 | void update_enforcement_timer(struct task_struct* t); |
7 | 7 | ||
8 | /* Send SIG_BUDGET to a real-time task. */ | ||
9 | void send_sigbudget(struct task_struct* t); | ||
10 | |||
8 | inline static int budget_exhausted(struct task_struct* t) | 11 | inline static int budget_exhausted(struct task_struct* t) |
9 | { | 12 | { |
10 | return get_exec_time(t) >= get_exec_cost(t); | 13 | return get_exec_time(t) >= get_exec_cost(t); |
@@ -19,10 +22,21 @@ inline static lt_t budget_remaining(struct task_struct* t) | |||
19 | return 0; | 22 | return 0; |
20 | } | 23 | } |
21 | 24 | ||
22 | #define budget_enforced(t) (tsk_rt(t)->task_params.budget_policy != NO_ENFORCEMENT) | 25 | #define budget_enforced(t) (\ |
26 | tsk_rt(t)->task_params.budget_policy != NO_ENFORCEMENT) | ||
27 | |||
28 | #define budget_precisely_tracked(t) (\ | ||
29 | tsk_rt(t)->task_params.budget_policy == PRECISE_ENFORCEMENT || \ | ||
30 | tsk_rt(t)->task_params.budget_signal_policy == PRECISE_SIGNALS) | ||
31 | |||
32 | #define budget_signalled(t) (\ | ||
33 | tsk_rt(t)->task_params.budget_signal_policy != NO_SIGNALS) | ||
34 | |||
35 | #define budget_precisely_signalled(t) (\ | ||
36 | tsk_rt(t)->task_params.budget_policy == PRECISE_SIGNALS) | ||
23 | 37 | ||
24 | #define budget_precisely_enforced(t) (tsk_rt(t)->task_params.budget_policy \ | 38 | #define sigbudget_sent(t) (\ |
25 | == PRECISE_ENFORCEMENT) | 39 | test_bit(RT_JOB_SIG_BUDGET_SENT, &tsk_rt(t)->job_params.flags)) |
26 | 40 | ||
27 | static inline int requeue_preempted_job(struct task_struct* t) | 41 | static inline int requeue_preempted_job(struct task_struct* t) |
28 | { | 42 | { |
diff --git a/include/litmus/edf_common.h b/include/litmus/edf_common.h index bbaf22ea7f12..63dff7efe8fb 100644 --- a/include/litmus/edf_common.h +++ b/include/litmus/edf_common.h | |||
@@ -20,6 +20,18 @@ int edf_higher_prio(struct task_struct* first, | |||
20 | 20 | ||
21 | int edf_ready_order(struct bheap_node* a, struct bheap_node* b); | 21 | int edf_ready_order(struct bheap_node* a, struct bheap_node* b); |
22 | 22 | ||
23 | #ifdef CONFIG_LITMUS_NESTED_LOCKING | ||
24 | /* binheap_nodes must be embedded within 'struct litmus_lock' */ | ||
25 | int edf_max_heap_order(struct binheap_node *a, struct binheap_node *b); | ||
26 | int edf_min_heap_order(struct binheap_node *a, struct binheap_node *b); | ||
27 | int edf_max_heap_base_priority_order(struct binheap_node *a, struct binheap_node *b); | ||
28 | int edf_min_heap_base_priority_order(struct binheap_node *a, struct binheap_node *b); | ||
29 | |||
30 | int __edf_higher_prio(struct task_struct* first, comparison_mode_t first_mode, | ||
31 | struct task_struct* second, comparison_mode_t second_mode); | ||
32 | |||
33 | #endif | ||
34 | |||
23 | int edf_preemption_needed(rt_domain_t* rt, struct task_struct *t); | 35 | int edf_preemption_needed(rt_domain_t* rt, struct task_struct *t); |
24 | 36 | ||
25 | #endif | 37 | #endif |
diff --git a/include/litmus/fdso.h b/include/litmus/fdso.h index f2115b83f1e4..1469c0fd0460 100644 --- a/include/litmus/fdso.h +++ b/include/litmus/fdso.h | |||
@@ -24,9 +24,18 @@ typedef enum { | |||
24 | MPCP_VS_SEM = 3, | 24 | MPCP_VS_SEM = 3, |
25 | DPCP_SEM = 4, | 25 | DPCP_SEM = 4, |
26 | 26 | ||
27 | PCP_SEM = 5, | 27 | PCP_SEM = 5, |
28 | 28 | ||
29 | MAX_OBJ_TYPE = 5 | 29 | RSM_MUTEX = 6, |
30 | IKGLP_SEM = 7, | ||
31 | KFMLP_SEM = 8, | ||
32 | |||
33 | IKGLP_SIMPLE_GPU_AFF_OBS = 9, | ||
34 | IKGLP_GPU_AFF_OBS = 10, | ||
35 | KFMLP_SIMPLE_GPU_AFF_OBS = 11, | ||
36 | KFMLP_GPU_AFF_OBS = 12, | ||
37 | |||
38 | MAX_OBJ_TYPE = 12 | ||
30 | } obj_type_t; | 39 | } obj_type_t; |
31 | 40 | ||
32 | struct inode_obj_id { | 41 | struct inode_obj_id { |
@@ -70,8 +79,11 @@ static inline void* od_lookup(int od, obj_type_t type) | |||
70 | } | 79 | } |
71 | 80 | ||
72 | #define lookup_fmlp_sem(od)((struct pi_semaphore*) od_lookup(od, FMLP_SEM)) | 81 | #define lookup_fmlp_sem(od)((struct pi_semaphore*) od_lookup(od, FMLP_SEM)) |
82 | #define lookup_kfmlp_sem(od)((struct pi_semaphore*) od_lookup(od, KFMLP_SEM)) | ||
73 | #define lookup_srp_sem(od) ((struct srp_semaphore*) od_lookup(od, SRP_SEM)) | 83 | #define lookup_srp_sem(od) ((struct srp_semaphore*) od_lookup(od, SRP_SEM)) |
74 | #define lookup_ics(od) ((struct ics*) od_lookup(od, ICS_ID)) | 84 | #define lookup_ics(od) ((struct ics*) od_lookup(od, ICS_ID)) |
75 | 85 | ||
86 | #define lookup_rsm_mutex(od)((struct litmus_lock*) od_lookup(od, FMLP_SEM)) | ||
87 | |||
76 | 88 | ||
77 | #endif | 89 | #endif |
diff --git a/include/litmus/fpmath.h b/include/litmus/fpmath.h index 642de98542c8..3d545fd2f5c5 100644 --- a/include/litmus/fpmath.h +++ b/include/litmus/fpmath.h | |||
@@ -1,11 +1,12 @@ | |||
1 | #ifndef __FP_MATH_H__ | 1 | #ifndef __FP_MATH_H__ |
2 | #define __FP_MATH_H__ | 2 | #define __FP_MATH_H__ |
3 | 3 | ||
4 | #ifdef __KERNEL__ | ||
4 | #include <linux/math64.h> | 5 | #include <linux/math64.h> |
5 | 6 | #else | |
6 | #ifndef __KERNEL__ | ||
7 | #include <stdint.h> | 7 | #include <stdint.h> |
8 | #define abs(x) (((x) < 0) ? -(x) : x) | 8 | #define abs(x) (((x) < 0) ? -(x) : x) |
9 | #define div64_s64(a, b) (a)/(b) | ||
9 | #endif | 10 | #endif |
10 | 11 | ||
11 | // Use 64-bit because we want to track things at the nanosecond scale. | 12 | // Use 64-bit because we want to track things at the nanosecond scale. |
diff --git a/include/litmus/gpu_affinity.h b/include/litmus/gpu_affinity.h new file mode 100644 index 000000000000..47da725717b0 --- /dev/null +++ b/include/litmus/gpu_affinity.h | |||
@@ -0,0 +1,66 @@ | |||
1 | #ifndef LITMUS_GPU_AFFINITY_H | ||
2 | #define LITMUS_GPU_AFFINITY_H | ||
3 | |||
4 | #include <litmus/rt_param.h> | ||
5 | #include <litmus/sched_plugin.h> | ||
6 | #include <litmus/litmus.h> | ||
7 | |||
8 | void update_gpu_estimate(struct task_struct* t, lt_t observed); | ||
9 | gpu_migration_dist_t gpu_migration_distance(int a, int b); | ||
10 | |||
11 | static inline void reset_gpu_tracker(struct task_struct* t) | ||
12 | { | ||
13 | t->rt_param.accum_gpu_time = 0; | ||
14 | } | ||
15 | |||
16 | static inline void start_gpu_tracker(struct task_struct* t) | ||
17 | { | ||
18 | t->rt_param.gpu_time_stamp = litmus_clock(); | ||
19 | } | ||
20 | |||
21 | static inline void stop_gpu_tracker(struct task_struct* t) | ||
22 | { | ||
23 | lt_t now = litmus_clock(); | ||
24 | t->rt_param.accum_gpu_time += (now - t->rt_param.gpu_time_stamp); | ||
25 | } | ||
26 | |||
27 | static inline lt_t get_gpu_time(struct task_struct* t) | ||
28 | { | ||
29 | return t->rt_param.accum_gpu_time; | ||
30 | } | ||
31 | |||
32 | static inline lt_t get_gpu_estimate(struct task_struct* t, gpu_migration_dist_t dist) | ||
33 | { | ||
34 | int i; | ||
35 | lt_t val; | ||
36 | |||
37 | if(dist == MIG_NONE) { | ||
38 | dist = MIG_LOCAL; | ||
39 | } | ||
40 | |||
41 | val = t->rt_param.gpu_migration_est[dist].avg; | ||
42 | for(i = dist-1; i >= 0; --i) { | ||
43 | if(t->rt_param.gpu_migration_est[i].avg > val) { | ||
44 | val = t->rt_param.gpu_migration_est[i].avg; | ||
45 | } | ||
46 | } | ||
47 | |||
48 | #if 0 | ||
49 | // int i; | ||
50 | // fpbuf_t temp = _fp_to_integer(t->rt_param.gpu_migration_est[dist].est); | ||
51 | // lt_t val = (temp >= 0) ? temp : 0; // never allow negative estimates... | ||
52 | lt_t val = t->rt_param.gpu_migration_est[dist].avg; | ||
53 | |||
54 | // WARN_ON(temp < 0); | ||
55 | |||
56 | // lower-bound a distant migration to be at least equal to the level | ||
57 | // below it. | ||
58 | // for(i = dist-1; (val == 0) && (i >= MIG_LOCAL); --i) { | ||
59 | // val = _fp_to_integer(t->rt_param.gpu_migration_est[i].est); | ||
60 | // } | ||
61 | #endif | ||
62 | |||
63 | return ((val > 0) ? val : dist+1); | ||
64 | } | ||
65 | |||
66 | #endif | ||
diff --git a/include/litmus/ikglp_lock.h b/include/litmus/ikglp_lock.h new file mode 100644 index 000000000000..af155eadbb35 --- /dev/null +++ b/include/litmus/ikglp_lock.h | |||
@@ -0,0 +1,164 @@ | |||
1 | #ifndef LITMUS_IKGLP_H | ||
2 | #define LITMUS_IKGLP_H | ||
3 | |||
4 | #include <litmus/litmus.h> | ||
5 | #include <litmus/binheap.h> | ||
6 | #include <litmus/locking.h> | ||
7 | |||
8 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING | ||
9 | #include <litmus/kexclu_affinity.h> | ||
10 | |||
11 | struct ikglp_affinity; | ||
12 | #endif | ||
13 | |||
14 | typedef struct ikglp_heap_node | ||
15 | { | ||
16 | struct task_struct *task; | ||
17 | struct binheap_node node; | ||
18 | } ikglp_heap_node_t; | ||
19 | |||
20 | struct fifo_queue; | ||
21 | struct ikglp_wait_state; | ||
22 | |||
23 | typedef struct ikglp_donee_heap_node | ||
24 | { | ||
25 | struct task_struct *task; | ||
26 | struct fifo_queue *fq; | ||
27 | struct ikglp_wait_state *donor_info; // cross-linked with ikglp_wait_state_t of donor | ||
28 | |||
29 | struct binheap_node node; | ||
30 | } ikglp_donee_heap_node_t; | ||
31 | |||
32 | // Maintains the state of a request as it goes through the IKGLP | ||
33 | typedef struct ikglp_wait_state { | ||
34 | struct task_struct *task; // pointer back to the requesting task | ||
35 | |||
36 | // Data for while waiting in FIFO Queue | ||
37 | wait_queue_t fq_node; | ||
38 | ikglp_heap_node_t global_heap_node; | ||
39 | ikglp_donee_heap_node_t donee_heap_node; | ||
40 | |||
41 | // Data for while waiting in PQ | ||
42 | ikglp_heap_node_t pq_node; | ||
43 | |||
44 | // Data for while waiting as a donor | ||
45 | ikglp_donee_heap_node_t *donee_info; // cross-linked with donee's ikglp_donee_heap_node_t | ||
46 | struct nested_info prio_donation; | ||
47 | struct binheap_node node; | ||
48 | } ikglp_wait_state_t; | ||
49 | |||
50 | /* struct for semaphore with priority inheritance */ | ||
51 | struct fifo_queue | ||
52 | { | ||
53 | wait_queue_head_t wait; | ||
54 | struct task_struct* owner; | ||
55 | |||
56 | // used for bookkeepping | ||
57 | ikglp_heap_node_t global_heap_node; | ||
58 | ikglp_donee_heap_node_t donee_heap_node; | ||
59 | |||
60 | struct task_struct* hp_waiter; | ||
61 | int count; /* number of waiters + holder */ | ||
62 | |||
63 | struct nested_info nest; | ||
64 | }; | ||
65 | |||
66 | struct ikglp_semaphore | ||
67 | { | ||
68 | struct litmus_lock litmus_lock; | ||
69 | |||
70 | raw_spinlock_t lock; | ||
71 | raw_spinlock_t real_lock; | ||
72 | |||
73 | int nr_replicas; // AKA k | ||
74 | int m; | ||
75 | |||
76 | int max_fifo_len; // max len of a fifo queue | ||
77 | int nr_in_fifos; | ||
78 | |||
79 | struct binheap top_m; // min heap, base prio | ||
80 | int top_m_size; // number of nodes in top_m | ||
81 | |||
82 | struct binheap not_top_m; // max heap, base prio | ||
83 | |||
84 | struct binheap donees; // min-heap, base prio | ||
85 | struct fifo_queue *shortest_fifo_queue; // pointer to shortest fifo queue | ||
86 | |||
87 | /* data structures for holding requests */ | ||
88 | struct fifo_queue *fifo_queues; // array nr_replicas in length | ||
89 | struct binheap priority_queue; // max-heap, base prio | ||
90 | struct binheap donors; // max-heap, base prio | ||
91 | |||
92 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING | ||
93 | struct ikglp_affinity *aff_obs; | ||
94 | #endif | ||
95 | }; | ||
96 | |||
97 | static inline struct ikglp_semaphore* ikglp_from_lock(struct litmus_lock* lock) | ||
98 | { | ||
99 | return container_of(lock, struct ikglp_semaphore, litmus_lock); | ||
100 | } | ||
101 | |||
102 | int ikglp_lock(struct litmus_lock* l); | ||
103 | int ikglp_unlock(struct litmus_lock* l); | ||
104 | int ikglp_close(struct litmus_lock* l); | ||
105 | void ikglp_free(struct litmus_lock* l); | ||
106 | struct litmus_lock* ikglp_new(int m, struct litmus_lock_ops*, void* __user arg); | ||
107 | |||
108 | |||
109 | |||
110 | #if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA) | ||
111 | |||
112 | struct ikglp_queue_info | ||
113 | { | ||
114 | struct fifo_queue* q; | ||
115 | lt_t estimated_len; | ||
116 | int *nr_cur_users; | ||
117 | int64_t *nr_aff_users; | ||
118 | }; | ||
119 | |||
120 | struct ikglp_affinity_ops | ||
121 | { | ||
122 | struct fifo_queue* (*advise_enqueue)(struct ikglp_affinity* aff, struct task_struct* t); // select FIFO | ||
123 | ikglp_wait_state_t* (*advise_steal)(struct ikglp_affinity* aff, struct fifo_queue* dst); // select steal from FIFO | ||
124 | ikglp_donee_heap_node_t* (*advise_donee_selection)(struct ikglp_affinity* aff, struct task_struct* t); // select a donee | ||
125 | ikglp_wait_state_t* (*advise_donor_to_fq)(struct ikglp_affinity* aff, struct fifo_queue* dst); // select a donor to move to PQ | ||
126 | |||
127 | void (*notify_enqueue)(struct ikglp_affinity* aff, struct fifo_queue* fq, struct task_struct* t); // fifo enqueue | ||
128 | void (*notify_dequeue)(struct ikglp_affinity* aff, struct fifo_queue* fq, struct task_struct* t); // fifo dequeue | ||
129 | void (*notify_acquired)(struct ikglp_affinity* aff, struct fifo_queue* fq, struct task_struct* t); // replica acquired | ||
130 | void (*notify_freed)(struct ikglp_affinity* aff, struct fifo_queue* fq, struct task_struct* t); // replica freed | ||
131 | int (*replica_to_resource)(struct ikglp_affinity* aff, struct fifo_queue* fq); // convert a replica # to a GPU (includes offsets and simult user folding) | ||
132 | |||
133 | int (*notify_exit)(struct ikglp_affinity* aff, struct task_struct* t); | ||
134 | }; | ||
135 | |||
136 | struct ikglp_affinity | ||
137 | { | ||
138 | struct affinity_observer obs; | ||
139 | struct ikglp_affinity_ops *ops; | ||
140 | struct ikglp_queue_info *q_info; | ||
141 | int *nr_cur_users_on_rsrc; | ||
142 | int64_t *nr_aff_on_rsrc; | ||
143 | int offset; | ||
144 | int nr_simult; | ||
145 | int nr_rsrc; | ||
146 | int relax_max_fifo_len; | ||
147 | }; | ||
148 | |||
149 | static inline struct ikglp_affinity* ikglp_aff_obs_from_aff_obs(struct affinity_observer* aff_obs) | ||
150 | { | ||
151 | return container_of(aff_obs, struct ikglp_affinity, obs); | ||
152 | } | ||
153 | |||
154 | int ikglp_aff_obs_close(struct affinity_observer*); | ||
155 | void ikglp_aff_obs_free(struct affinity_observer*); | ||
156 | struct affinity_observer* ikglp_gpu_aff_obs_new(struct affinity_observer_ops*, | ||
157 | void* __user arg); | ||
158 | struct affinity_observer* ikglp_simple_gpu_aff_obs_new(struct affinity_observer_ops*, | ||
159 | void* __user arg); | ||
160 | #endif | ||
161 | |||
162 | |||
163 | |||
164 | #endif | ||
diff --git a/include/litmus/kexclu_affinity.h b/include/litmus/kexclu_affinity.h new file mode 100644 index 000000000000..f6355de49074 --- /dev/null +++ b/include/litmus/kexclu_affinity.h | |||
@@ -0,0 +1,35 @@ | |||
1 | #ifndef LITMUS_AFF_OBS_H | ||
2 | #define LITMUS_AFF_OBS_H | ||
3 | |||
4 | #include <litmus/locking.h> | ||
5 | |||
6 | struct affinity_observer_ops; | ||
7 | |||
8 | struct affinity_observer | ||
9 | { | ||
10 | struct affinity_observer_ops* ops; | ||
11 | int type; | ||
12 | int ident; | ||
13 | |||
14 | struct litmus_lock* lock; // the lock under observation | ||
15 | }; | ||
16 | |||
17 | typedef int (*aff_obs_open_t)(struct affinity_observer* aff_obs, | ||
18 | void* __user arg); | ||
19 | typedef int (*aff_obs_close_t)(struct affinity_observer* aff_obs); | ||
20 | typedef void (*aff_obs_free_t)(struct affinity_observer* aff_obs); | ||
21 | |||
22 | struct affinity_observer_ops | ||
23 | { | ||
24 | aff_obs_open_t open; | ||
25 | aff_obs_close_t close; | ||
26 | aff_obs_free_t deallocate; | ||
27 | }; | ||
28 | |||
29 | struct litmus_lock* get_lock_from_od(int od); | ||
30 | |||
31 | void affinity_observer_new(struct affinity_observer* aff, | ||
32 | struct affinity_observer_ops* ops, | ||
33 | struct affinity_observer_args* args); | ||
34 | |||
35 | #endif | ||
diff --git a/include/litmus/kfmlp_lock.h b/include/litmus/kfmlp_lock.h new file mode 100644 index 000000000000..5f0aae6e6f42 --- /dev/null +++ b/include/litmus/kfmlp_lock.h | |||
@@ -0,0 +1,97 @@ | |||
1 | #ifndef LITMUS_KFMLP_H | ||
2 | #define LITMUS_KFMLP_H | ||
3 | |||
4 | #include <litmus/litmus.h> | ||
5 | #include <litmus/locking.h> | ||
6 | |||
7 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING | ||
8 | #include <litmus/kexclu_affinity.h> | ||
9 | |||
10 | struct kfmlp_affinity; | ||
11 | #endif | ||
12 | |||
13 | /* struct for semaphore with priority inheritance */ | ||
14 | struct kfmlp_queue | ||
15 | { | ||
16 | wait_queue_head_t wait; | ||
17 | struct task_struct* owner; | ||
18 | struct task_struct* hp_waiter; | ||
19 | int count; /* number of waiters + holder */ | ||
20 | }; | ||
21 | |||
22 | struct kfmlp_semaphore | ||
23 | { | ||
24 | struct litmus_lock litmus_lock; | ||
25 | |||
26 | spinlock_t lock; | ||
27 | |||
28 | int num_resources; /* aka k */ | ||
29 | |||
30 | struct kfmlp_queue *queues; /* array */ | ||
31 | struct kfmlp_queue *shortest_queue; /* pointer to shortest queue */ | ||
32 | |||
33 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING | ||
34 | struct kfmlp_affinity *aff_obs; | ||
35 | #endif | ||
36 | }; | ||
37 | |||
38 | static inline struct kfmlp_semaphore* kfmlp_from_lock(struct litmus_lock* lock) | ||
39 | { | ||
40 | return container_of(lock, struct kfmlp_semaphore, litmus_lock); | ||
41 | } | ||
42 | |||
43 | int kfmlp_lock(struct litmus_lock* l); | ||
44 | int kfmlp_unlock(struct litmus_lock* l); | ||
45 | int kfmlp_close(struct litmus_lock* l); | ||
46 | void kfmlp_free(struct litmus_lock* l); | ||
47 | struct litmus_lock* kfmlp_new(struct litmus_lock_ops*, void* __user arg); | ||
48 | |||
49 | #if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA) | ||
50 | |||
51 | struct kfmlp_queue_info | ||
52 | { | ||
53 | struct kfmlp_queue* q; | ||
54 | lt_t estimated_len; | ||
55 | int *nr_cur_users; | ||
56 | }; | ||
57 | |||
58 | struct kfmlp_affinity_ops | ||
59 | { | ||
60 | struct kfmlp_queue* (*advise_enqueue)(struct kfmlp_affinity* aff, struct task_struct* t); | ||
61 | struct task_struct* (*advise_steal)(struct kfmlp_affinity* aff, wait_queue_t** to_steal, struct kfmlp_queue** to_steal_from); | ||
62 | void (*notify_enqueue)(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t); | ||
63 | void (*notify_dequeue)(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t); | ||
64 | void (*notify_acquired)(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t); | ||
65 | void (*notify_freed)(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t); | ||
66 | int (*replica_to_resource)(struct kfmlp_affinity* aff, struct kfmlp_queue* fq); | ||
67 | }; | ||
68 | |||
69 | struct kfmlp_affinity | ||
70 | { | ||
71 | struct affinity_observer obs; | ||
72 | struct kfmlp_affinity_ops *ops; | ||
73 | struct kfmlp_queue_info *q_info; | ||
74 | int *nr_cur_users_on_rsrc; | ||
75 | int offset; | ||
76 | int nr_simult; | ||
77 | int nr_rsrc; | ||
78 | }; | ||
79 | |||
80 | static inline struct kfmlp_affinity* kfmlp_aff_obs_from_aff_obs(struct affinity_observer* aff_obs) | ||
81 | { | ||
82 | return container_of(aff_obs, struct kfmlp_affinity, obs); | ||
83 | } | ||
84 | |||
85 | int kfmlp_aff_obs_close(struct affinity_observer*); | ||
86 | void kfmlp_aff_obs_free(struct affinity_observer*); | ||
87 | struct affinity_observer* kfmlp_gpu_aff_obs_new(struct affinity_observer_ops*, | ||
88 | void* __user arg); | ||
89 | struct affinity_observer* kfmlp_simple_gpu_aff_obs_new(struct affinity_observer_ops*, | ||
90 | void* __user arg); | ||
91 | |||
92 | |||
93 | #endif | ||
94 | |||
95 | #endif | ||
96 | |||
97 | |||
diff --git a/include/litmus/litmus.h b/include/litmus/litmus.h index 875783e6a67b..2da61fa58bdc 100644 --- a/include/litmus/litmus.h +++ b/include/litmus/litmus.h | |||
@@ -26,16 +26,20 @@ static inline int in_list(struct list_head* list) | |||
26 | ); | 26 | ); |
27 | } | 27 | } |
28 | 28 | ||
29 | |||
29 | struct task_struct* __waitqueue_remove_first(wait_queue_head_t *wq); | 30 | struct task_struct* __waitqueue_remove_first(wait_queue_head_t *wq); |
30 | 31 | ||
31 | #define NO_CPU 0xffffffff | 32 | #define NO_CPU 0xffffffff |
32 | 33 | ||
33 | void litmus_fork(struct task_struct *tsk); | 34 | void litmus_fork(struct task_struct *tsk); |
35 | void litmus_post_fork_thread(struct task_struct *tsk); | ||
34 | void litmus_exec(void); | 36 | void litmus_exec(void); |
35 | /* clean up real-time state of a task */ | 37 | /* clean up real-time state of a task */ |
36 | void exit_litmus(struct task_struct *dead_tsk); | 38 | void exit_litmus(struct task_struct *dead_tsk); |
37 | 39 | ||
38 | long litmus_admit_task(struct task_struct *tsk); | 40 | long litmus_admit_task(struct task_struct *tsk); |
41 | |||
42 | void litmus_pre_exit_task(struct task_struct *tsk); // called before litmus_exit_task, but without run queue locks held | ||
39 | void litmus_exit_task(struct task_struct *tsk); | 43 | void litmus_exit_task(struct task_struct *tsk); |
40 | 44 | ||
41 | #define is_realtime(t) ((t)->policy == SCHED_LITMUS) | 45 | #define is_realtime(t) ((t)->policy == SCHED_LITMUS) |
@@ -43,6 +47,7 @@ void litmus_exit_task(struct task_struct *tsk); | |||
43 | ((t)->rt_param.transition_pending) | 47 | ((t)->rt_param.transition_pending) |
44 | 48 | ||
45 | #define tsk_rt(t) (&(t)->rt_param) | 49 | #define tsk_rt(t) (&(t)->rt_param) |
50 | #define tsk_aux(t) (&(t)->aux_data) | ||
46 | 51 | ||
47 | /* Realtime utility macros */ | 52 | /* Realtime utility macros */ |
48 | #define is_priority_boosted(t) (tsk_rt(t)->priority_boosted) | 53 | #define is_priority_boosted(t) (tsk_rt(t)->priority_boosted) |
@@ -60,9 +65,13 @@ void litmus_exit_task(struct task_struct *tsk); | |||
60 | /* job_param macros */ | 65 | /* job_param macros */ |
61 | #define get_exec_time(t) (tsk_rt(t)->job_params.exec_time) | 66 | #define get_exec_time(t) (tsk_rt(t)->job_params.exec_time) |
62 | #define get_deadline(t) (tsk_rt(t)->job_params.deadline) | 67 | #define get_deadline(t) (tsk_rt(t)->job_params.deadline) |
68 | #define get_period(t) (tsk_rt(t)->task_params.period) | ||
63 | #define get_release(t) (tsk_rt(t)->job_params.release) | 69 | #define get_release(t) (tsk_rt(t)->job_params.release) |
64 | #define get_lateness(t) (tsk_rt(t)->job_params.lateness) | 70 | #define get_lateness(t) (tsk_rt(t)->job_params.lateness) |
65 | 71 | ||
72 | #define effective_priority(t) ((!(tsk_rt(t)->inh_task)) ? t : tsk_rt(t)->inh_task) | ||
73 | #define base_priority(t) (t) | ||
74 | |||
66 | #define is_hrt(t) \ | 75 | #define is_hrt(t) \ |
67 | (tsk_rt(t)->task_params.cls == RT_CLASS_HARD) | 76 | (tsk_rt(t)->task_params.cls == RT_CLASS_HARD) |
68 | #define is_srt(t) \ | 77 | #define is_srt(t) \ |
@@ -99,10 +108,12 @@ static inline lt_t litmus_clock(void) | |||
99 | #define earlier_deadline(a, b) (lt_before(\ | 108 | #define earlier_deadline(a, b) (lt_before(\ |
100 | (a)->rt_param.job_params.deadline,\ | 109 | (a)->rt_param.job_params.deadline,\ |
101 | (b)->rt_param.job_params.deadline)) | 110 | (b)->rt_param.job_params.deadline)) |
111 | #define shorter_period(a, b) (lt_before(\ | ||
112 | (a)->rt_param.task_params.period,\ | ||
113 | (b)->rt_param.task_params.period)) | ||
102 | #define earlier_release(a, b) (lt_before(\ | 114 | #define earlier_release(a, b) (lt_before(\ |
103 | (a)->rt_param.job_params.release,\ | 115 | (a)->rt_param.job_params.release,\ |
104 | (b)->rt_param.job_params.release)) | 116 | (b)->rt_param.job_params.release)) |
105 | |||
106 | void preempt_if_preemptable(struct task_struct* t, int on_cpu); | 117 | void preempt_if_preemptable(struct task_struct* t, int on_cpu); |
107 | 118 | ||
108 | #ifdef CONFIG_LITMUS_LOCKING | 119 | #ifdef CONFIG_LITMUS_LOCKING |
@@ -174,8 +185,10 @@ static inline int request_exit_np_atomic(struct task_struct *t) | |||
174 | * retry loop here since tasks might exploit that to | 185 | * retry loop here since tasks might exploit that to |
175 | * keep the kernel busy indefinitely. */ | 186 | * keep the kernel busy indefinitely. */ |
176 | } | 187 | } |
177 | } else | 188 | } |
189 | else { | ||
178 | return 0; | 190 | return 0; |
191 | } | ||
179 | } | 192 | } |
180 | 193 | ||
181 | #else | 194 | #else |
diff --git a/include/litmus/litmus_softirq.h b/include/litmus/litmus_softirq.h new file mode 100644 index 000000000000..cfef08187464 --- /dev/null +++ b/include/litmus/litmus_softirq.h | |||
@@ -0,0 +1,166 @@ | |||
1 | #ifndef __LITMUS_SOFTIRQ_H | ||
2 | #define __LITMUS_SOFTIRQ_H | ||
3 | |||
4 | #include <linux/interrupt.h> | ||
5 | #include <linux/workqueue.h> | ||
6 | |||
7 | /* | ||
8 | Threaded tasklet/workqueue handling for Litmus. | ||
9 | Items are scheduled in the following order: hi-tasklet, | ||
10 | lo-tasklet, workqueue. Items are scheduled in FIFO order | ||
11 | within each of these classes. | ||
12 | |||
13 | klmirqd assumes the priority of the owner of the | ||
14 | tasklet when the tasklet is next to execute. | ||
15 | |||
16 | The base-priority of a klimirqd thread is below all regular | ||
17 | real-time tasks, but above all other Linux scheduling | ||
18 | classes (klmirqd threads are within the SHCED_LITMUS class). | ||
19 | Regular real-time tasks may increase the priority of | ||
20 | a klmirqd thread, but klmirqd is unaware of this | ||
21 | (this was not the case in prior incarnations of klmirqd). | ||
22 | */ | ||
23 | |||
24 | |||
25 | /* Initialize klmirqd */ | ||
26 | void init_klmirqd(void); | ||
27 | |||
28 | /* Raises a flag to tell klmirqds to terminate. | ||
29 | Termination is async, so some threads may be running | ||
30 | after function return. */ | ||
31 | void kill_klmirqd(void); | ||
32 | |||
33 | void kill_klmirqd_thread(struct task_struct* klmirqd_thread); | ||
34 | |||
35 | /* Returns 1 if all NR_LITMUS_SOFTIRQD klitirqs are ready | ||
36 | to handle tasklets. 0, otherwise.*/ | ||
37 | int klmirqd_is_ready(void); | ||
38 | |||
39 | /* Returns 1 if no NR_LITMUS_SOFTIRQD klitirqs are ready | ||
40 | to handle tasklets. 0, otherwise.*/ | ||
41 | int klmirqd_is_dead(void); | ||
42 | |||
43 | |||
44 | typedef int (*klmirqd_cb_t) (void *arg); | ||
45 | |||
46 | typedef struct | ||
47 | { | ||
48 | klmirqd_cb_t func; | ||
49 | void* arg; | ||
50 | } klmirqd_callback_t; | ||
51 | |||
52 | /* Launches a klmirqd thread with the provided affinity. | ||
53 | |||
54 | Actual launch of threads is deffered to kworker's | ||
55 | workqueue, so daemons will likely not be immediately | ||
56 | running when this function returns, though the required | ||
57 | data will be initialized. | ||
58 | |||
59 | cpu == -1 for no affinity | ||
60 | |||
61 | provide a name at most 31 (32, + null terminator) characters long. | ||
62 | name == NULL for a default name. (all names are appended with | ||
63 | base-CPU affinity) | ||
64 | */ | ||
65 | #define MAX_KLMIRQD_NAME_LEN 31 | ||
66 | int launch_klmirqd_thread(char* name, int cpu, klmirqd_callback_t* cb); | ||
67 | |||
68 | |||
69 | /* Flushes all pending work out to the OS for regular | ||
70 | * tasklet/work processing. | ||
71 | */ | ||
72 | void flush_pending(struct task_struct* klmirqd_thread); | ||
73 | |||
74 | extern int __litmus_tasklet_schedule( | ||
75 | struct tasklet_struct *t, | ||
76 | struct task_struct *klmirqd_thread); | ||
77 | |||
78 | /* schedule a tasklet on klmirqd #k_id */ | ||
79 | static inline int litmus_tasklet_schedule( | ||
80 | struct tasklet_struct *t, | ||
81 | struct task_struct *klmirqd_thread) | ||
82 | { | ||
83 | int ret = 0; | ||
84 | if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) { | ||
85 | ret = __litmus_tasklet_schedule(t, klmirqd_thread); | ||
86 | } | ||
87 | return(ret); | ||
88 | } | ||
89 | |||
90 | /* for use by __tasklet_schedule() */ | ||
91 | static inline int _litmus_tasklet_schedule( | ||
92 | struct tasklet_struct *t, | ||
93 | struct task_struct *klmirqd_thread) | ||
94 | { | ||
95 | return(__litmus_tasklet_schedule(t, klmirqd_thread)); | ||
96 | } | ||
97 | |||
98 | |||
99 | |||
100 | |||
101 | extern int __litmus_tasklet_hi_schedule(struct tasklet_struct *t, | ||
102 | struct task_struct *klmirqd_thread); | ||
103 | |||
104 | /* schedule a hi tasklet on klmirqd #k_id */ | ||
105 | static inline int litmus_tasklet_hi_schedule(struct tasklet_struct *t, | ||
106 | struct task_struct *klmirqd_thread) | ||
107 | { | ||
108 | int ret = 0; | ||
109 | if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) { | ||
110 | ret = __litmus_tasklet_hi_schedule(t, klmirqd_thread); | ||
111 | } | ||
112 | return(ret); | ||
113 | } | ||
114 | |||
115 | /* for use by __tasklet_hi_schedule() */ | ||
116 | static inline int _litmus_tasklet_hi_schedule(struct tasklet_struct *t, | ||
117 | struct task_struct *klmirqd_thread) | ||
118 | { | ||
119 | return(__litmus_tasklet_hi_schedule(t, klmirqd_thread)); | ||
120 | } | ||
121 | |||
122 | |||
123 | |||
124 | |||
125 | |||
126 | extern int __litmus_tasklet_hi_schedule_first( | ||
127 | struct tasklet_struct *t, | ||
128 | struct task_struct *klmirqd_thread); | ||
129 | |||
130 | /* schedule a hi tasklet on klmirqd #k_id on next go-around */ | ||
131 | /* PRECONDITION: Interrupts must be disabled. */ | ||
132 | static inline int litmus_tasklet_hi_schedule_first( | ||
133 | struct tasklet_struct *t, | ||
134 | struct task_struct *klmirqd_thread) | ||
135 | { | ||
136 | int ret = 0; | ||
137 | if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) { | ||
138 | ret = __litmus_tasklet_hi_schedule_first(t, klmirqd_thread); | ||
139 | } | ||
140 | return(ret); | ||
141 | } | ||
142 | |||
143 | /* for use by __tasklet_hi_schedule_first() */ | ||
144 | static inline int _litmus_tasklet_hi_schedule_first( | ||
145 | struct tasklet_struct *t, | ||
146 | struct task_struct *klmirqd_thread) | ||
147 | { | ||
148 | return(__litmus_tasklet_hi_schedule_first(t, klmirqd_thread)); | ||
149 | } | ||
150 | |||
151 | |||
152 | |||
153 | ////////////// | ||
154 | |||
155 | extern int __litmus_schedule_work( | ||
156 | struct work_struct* w, | ||
157 | struct task_struct *klmirqd_thread); | ||
158 | |||
159 | static inline int litmus_schedule_work( | ||
160 | struct work_struct* w, | ||
161 | struct task_struct *klmirqd_thread) | ||
162 | { | ||
163 | return(__litmus_schedule_work(w, klmirqd_thread)); | ||
164 | } | ||
165 | |||
166 | #endif | ||
diff --git a/include/litmus/locking.h b/include/litmus/locking.h index 4d7b870cb443..4a5f198a0407 100644 --- a/include/litmus/locking.h +++ b/include/litmus/locking.h | |||
@@ -1,28 +1,163 @@ | |||
1 | #ifndef LITMUS_LOCKING_H | 1 | #ifndef LITMUS_LOCKING_H |
2 | #define LITMUS_LOCKING_H | 2 | #define LITMUS_LOCKING_H |
3 | 3 | ||
4 | #include <linux/list.h> | ||
5 | |||
4 | struct litmus_lock_ops; | 6 | struct litmus_lock_ops; |
5 | 7 | ||
8 | #ifdef CONFIG_LITMUS_NESTED_LOCKING | ||
9 | struct nested_info | ||
10 | { | ||
11 | struct litmus_lock *lock; | ||
12 | struct task_struct *hp_waiter_eff_prio; | ||
13 | struct task_struct **hp_waiter_ptr; | ||
14 | struct binheap_node hp_binheap_node; | ||
15 | }; | ||
16 | |||
17 | static inline struct task_struct* top_priority(struct binheap* handle) { | ||
18 | if(!binheap_empty(handle)) { | ||
19 | return (struct task_struct*)(binheap_top_entry(handle, struct nested_info, hp_binheap_node)->hp_waiter_eff_prio); | ||
20 | } | ||
21 | return NULL; | ||
22 | } | ||
23 | |||
24 | void print_hp_waiters(struct binheap_node* n, int depth); | ||
25 | #endif | ||
26 | |||
27 | |||
6 | /* Generic base struct for LITMUS^RT userspace semaphores. | 28 | /* Generic base struct for LITMUS^RT userspace semaphores. |
7 | * This structure should be embedded in protocol-specific semaphores. | 29 | * This structure should be embedded in protocol-specific semaphores. |
8 | */ | 30 | */ |
9 | struct litmus_lock { | 31 | struct litmus_lock { |
10 | struct litmus_lock_ops *ops; | 32 | struct litmus_lock_ops *ops; |
11 | int type; | 33 | int type; |
34 | |||
35 | int ident; | ||
36 | |||
37 | #ifdef CONFIG_LITMUS_NESTED_LOCKING | ||
38 | struct nested_info nest; | ||
39 | //#ifdef CONFIG_DEBUG_SPINLOCK | ||
40 | char cheat_lockdep[2]; | ||
41 | struct lock_class_key key; | ||
42 | //#endif | ||
43 | #endif | ||
12 | }; | 44 | }; |
13 | 45 | ||
46 | #ifdef CONFIG_LITMUS_DGL_SUPPORT | ||
47 | |||
48 | #define MAX_DGL_SIZE CONFIG_LITMUS_MAX_DGL_SIZE | ||
49 | |||
50 | typedef struct dgl_wait_state { | ||
51 | struct task_struct *task; /* task waiting on DGL */ | ||
52 | struct litmus_lock *locks[MAX_DGL_SIZE]; /* requested locks in DGL */ | ||
53 | int size; /* size of the DGL */ | ||
54 | int nr_remaining; /* nr locks remainging before DGL is complete */ | ||
55 | int last_primary; /* index lock in locks[] that has active priority */ | ||
56 | wait_queue_t wq_nodes[MAX_DGL_SIZE]; | ||
57 | } dgl_wait_state_t; | ||
58 | |||
59 | void wake_or_wait_on_next_lock(dgl_wait_state_t *dgl_wait); | ||
60 | void select_next_lock(dgl_wait_state_t* dgl_wait /*, struct litmus_lock* prev_lock*/); | ||
61 | |||
62 | void init_dgl_waitqueue_entry(wait_queue_t *wq_node, dgl_wait_state_t* dgl_wait); | ||
63 | int dgl_wake_up(wait_queue_t *wq_node, unsigned mode, int sync, void *key); | ||
64 | void __waitqueue_dgl_remove_first(wait_queue_head_t *wq, dgl_wait_state_t** dgl_wait, struct task_struct **task); | ||
65 | #endif | ||
66 | |||
67 | typedef int (*lock_op_t)(struct litmus_lock *l); | ||
68 | typedef lock_op_t lock_close_t; | ||
69 | typedef lock_op_t lock_lock_t; | ||
70 | typedef lock_op_t lock_unlock_t; | ||
71 | |||
72 | typedef int (*lock_open_t)(struct litmus_lock *l, void* __user arg); | ||
73 | typedef void (*lock_free_t)(struct litmus_lock *l); | ||
74 | |||
14 | struct litmus_lock_ops { | 75 | struct litmus_lock_ops { |
15 | /* Current task tries to obtain / drop a reference to a lock. | 76 | /* Current task tries to obtain / drop a reference to a lock. |
16 | * Optional methods, allowed by default. */ | 77 | * Optional methods, allowed by default. */ |
17 | int (*open)(struct litmus_lock*, void* __user); | 78 | lock_open_t open; |
18 | int (*close)(struct litmus_lock*); | 79 | lock_close_t close; |
19 | 80 | ||
20 | /* Current tries to lock/unlock this lock (mandatory methods). */ | 81 | /* Current tries to lock/unlock this lock (mandatory methods). */ |
21 | int (*lock)(struct litmus_lock*); | 82 | lock_lock_t lock; |
22 | int (*unlock)(struct litmus_lock*); | 83 | lock_unlock_t unlock; |
23 | 84 | ||
24 | /* The lock is no longer being referenced (mandatory method). */ | 85 | /* The lock is no longer being referenced (mandatory method). */ |
25 | void (*deallocate)(struct litmus_lock*); | 86 | lock_free_t deallocate; |
87 | |||
88 | #ifdef CONFIG_LITMUS_NESTED_LOCKING | ||
89 | void (*propagate_increase_inheritance)(struct litmus_lock* l, struct task_struct* t, raw_spinlock_t* to_unlock, unsigned long irqflags); | ||
90 | void (*propagate_decrease_inheritance)(struct litmus_lock* l, struct task_struct* t, raw_spinlock_t* to_unlock, unsigned long irqflags); | ||
91 | #endif | ||
92 | |||
93 | #ifdef CONFIG_LITMUS_DGL_SUPPORT | ||
94 | raw_spinlock_t* (*get_dgl_spin_lock)(struct litmus_lock *l); | ||
95 | int (*dgl_lock)(struct litmus_lock *l, dgl_wait_state_t* dgl_wait, wait_queue_t* wq_node); | ||
96 | int (*is_owner)(struct litmus_lock *l, struct task_struct *t); | ||
97 | void (*enable_priority)(struct litmus_lock *l, dgl_wait_state_t* dgl_wait); | ||
98 | #endif | ||
26 | }; | 99 | }; |
27 | 100 | ||
101 | |||
102 | /* | ||
103 | Nested inheritance can be achieved with fine-grain locking when there is | ||
104 | no need for DGL support, presuming locks are acquired in a partial order | ||
105 | (no cycles!). However, DGLs allow locks to be acquired in any order. This | ||
106 | makes nested inheritance very difficult (we don't yet know a solution) to | ||
107 | realize with fine-grain locks, so we use a big lock instead. | ||
108 | |||
109 | Code contains both fine-grain and coarse-grain methods together, side-by-side. | ||
110 | Each lock operation *IS NOT* surrounded by ifdef/endif to help make code more | ||
111 | readable. However, this leads to the odd situation where both code paths | ||
112 | appear together in code as if they were both active together. | ||
113 | |||
114 | THIS IS NOT REALLY THE CASE! ONLY ONE CODE PATH IS ACTUALLY ACTIVE! | ||
115 | |||
116 | Example: | ||
117 | lock_global_irqsave(coarseLock, flags); | ||
118 | lock_fine_irqsave(fineLock, flags); | ||
119 | |||
120 | Reality (coarse): | ||
121 | lock_global_irqsave(coarseLock, flags); | ||
122 | //lock_fine_irqsave(fineLock, flags); | ||
123 | |||
124 | Reality (fine): | ||
125 | //lock_global_irqsave(coarseLock, flags); | ||
126 | lock_fine_irqsave(fineLock, flags); | ||
127 | |||
128 | Be careful when you read code involving nested inheritance. | ||
129 | */ | ||
130 | #if defined(CONFIG_LITMUS_DGL_SUPPORT) | ||
131 | /* DGL requires a big lock to implement nested inheritance */ | ||
132 | #define lock_global_irqsave(lock, flags) raw_spin_lock_irqsave((lock), (flags)) | ||
133 | #define lock_global(lock) raw_spin_lock((lock)) | ||
134 | #define unlock_global_irqrestore(lock, flags) raw_spin_unlock_irqrestore((lock), (flags)) | ||
135 | #define unlock_global(lock) raw_spin_unlock((lock)) | ||
136 | |||
137 | /* fine-grain locking are no-ops with DGL support */ | ||
138 | #define lock_fine_irqsave(lock, flags) | ||
139 | #define lock_fine(lock) | ||
140 | #define unlock_fine_irqrestore(lock, flags) | ||
141 | #define unlock_fine(lock) | ||
142 | |||
143 | #elif defined(CONFIG_LITMUS_NESTED_LOCKING) | ||
144 | /* Use fine-grain locking when DGLs are disabled. */ | ||
145 | /* global locking are no-ops without DGL support */ | ||
146 | #define lock_global_irqsave(lock, flags) | ||
147 | #define lock_global(lock) | ||
148 | #define unlock_global_irqrestore(lock, flags) | ||
149 | #define unlock_global(lock) | ||
150 | |||
151 | #define lock_fine_irqsave(lock, flags) raw_spin_lock_irqsave((lock), (flags)) | ||
152 | #define lock_fine(lock) raw_spin_lock((lock)) | ||
153 | #define unlock_fine_irqrestore(lock, flags) raw_spin_unlock_irqrestore((lock), (flags)) | ||
154 | #define unlock_fine(lock) raw_spin_unlock((lock)) | ||
155 | |||
156 | #endif | ||
157 | |||
158 | |||
159 | void suspend_for_lock(void); | ||
160 | |||
161 | |||
28 | #endif | 162 | #endif |
163 | |||
diff --git a/include/litmus/nvidia_info.h b/include/litmus/nvidia_info.h new file mode 100644 index 000000000000..8c2a5524512e --- /dev/null +++ b/include/litmus/nvidia_info.h | |||
@@ -0,0 +1,51 @@ | |||
1 | #ifndef __LITMUS_NVIDIA_H | ||
2 | #define __LITMUS_NVIDIA_H | ||
3 | |||
4 | #include <linux/interrupt.h> | ||
5 | |||
6 | |||
7 | #include <litmus/litmus_softirq.h> | ||
8 | |||
9 | #define NV_DEVICE_NUM CONFIG_NV_DEVICE_NUM | ||
10 | |||
11 | /* TODO: Make this a function that checks the PCIe bus or maybe proc settings */ | ||
12 | #define num_online_gpus() (NV_DEVICE_NUM) | ||
13 | |||
14 | |||
15 | /* Functions used for decoding NVIDIA blobs. */ | ||
16 | |||
17 | int init_nvidia_info(void); | ||
18 | void shutdown_nvidia_info(void); | ||
19 | |||
20 | int is_nvidia_func(void* func_addr); | ||
21 | |||
22 | void dump_nvidia_info(const struct tasklet_struct *t); | ||
23 | |||
24 | // Returns the Nvidia device # associated with provided tasklet and work_struct. | ||
25 | u32 get_tasklet_nv_device_num(const struct tasklet_struct *t); | ||
26 | u32 get_work_nv_device_num(const struct work_struct *t); | ||
27 | |||
28 | /* Functions for figuring out the priority of GPU-using tasks */ | ||
29 | |||
30 | struct task_struct* get_nv_max_device_owner(u32 target_device_id); | ||
31 | |||
32 | #ifdef CONFIG_LITMUS_SOFTIRQD | ||
33 | struct task_struct* get_nv_klmirqd_thread(u32 target_device_id); | ||
34 | #endif | ||
35 | |||
36 | /* call when the GPU-holding task, t, blocks */ | ||
37 | long enable_gpu_owner(struct task_struct *t); | ||
38 | |||
39 | /* call when the GPU-holding task, t, resumes */ | ||
40 | long disable_gpu_owner(struct task_struct *t); | ||
41 | |||
42 | /* call when the GPU-holding task, t, increases its priority */ | ||
43 | int gpu_owner_increase_priority(struct task_struct *t); | ||
44 | |||
45 | /* call when the GPU-holding task, t, decreases its priority */ | ||
46 | int gpu_owner_decrease_priority(struct task_struct *t); | ||
47 | |||
48 | |||
49 | int reg_nv_device(int reg_device_id, int reg_action, struct task_struct *t); | ||
50 | |||
51 | #endif | ||
diff --git a/include/litmus/preempt.h b/include/litmus/preempt.h index 380b886d78ff..8f3a9ca2d4e3 100644 --- a/include/litmus/preempt.h +++ b/include/litmus/preempt.h | |||
@@ -26,12 +26,12 @@ const char* sched_state_name(int s); | |||
26 | (x), #x, __FUNCTION__); \ | 26 | (x), #x, __FUNCTION__); \ |
27 | } while (0); | 27 | } while (0); |
28 | 28 | ||
29 | //#define TRACE_SCHED_STATE_CHANGE(x, y, cpu) /* ignore */ | ||
29 | #define TRACE_SCHED_STATE_CHANGE(x, y, cpu) \ | 30 | #define TRACE_SCHED_STATE_CHANGE(x, y, cpu) \ |
30 | TRACE_STATE("[P%d] 0x%x (%s) -> 0x%x (%s)\n", \ | 31 | TRACE_STATE("[P%d] 0x%x (%s) -> 0x%x (%s)\n", \ |
31 | cpu, (x), sched_state_name(x), \ | 32 | cpu, (x), sched_state_name(x), \ |
32 | (y), sched_state_name(y)) | 33 | (y), sched_state_name(y)) |
33 | 34 | ||
34 | |||
35 | typedef enum scheduling_state { | 35 | typedef enum scheduling_state { |
36 | TASK_SCHEDULED = (1 << 0), /* The currently scheduled task is the one that | 36 | TASK_SCHEDULED = (1 << 0), /* The currently scheduled task is the one that |
37 | * should be scheduled, and the processor does not | 37 | * should be scheduled, and the processor does not |
diff --git a/include/litmus/rsm_lock.h b/include/litmus/rsm_lock.h new file mode 100644 index 000000000000..a15189683de4 --- /dev/null +++ b/include/litmus/rsm_lock.h | |||
@@ -0,0 +1,54 @@ | |||
1 | #ifndef LITMUS_RSM_H | ||
2 | #define LITMUS_RSM_H | ||
3 | |||
4 | #include <litmus/litmus.h> | ||
5 | #include <litmus/binheap.h> | ||
6 | #include <litmus/locking.h> | ||
7 | |||
8 | /* struct for semaphore with priority inheritance */ | ||
9 | struct rsm_mutex { | ||
10 | struct litmus_lock litmus_lock; | ||
11 | |||
12 | /* current resource holder */ | ||
13 | struct task_struct *owner; | ||
14 | |||
15 | /* highest-priority waiter */ | ||
16 | struct task_struct *hp_waiter; | ||
17 | |||
18 | /* FIFO queue of waiting tasks -- for now. time stamp in the future. */ | ||
19 | wait_queue_head_t wait; | ||
20 | |||
21 | /* we do some nesting within spinlocks, so we can't use the normal | ||
22 | sleeplocks found in wait_queue_head_t. */ | ||
23 | raw_spinlock_t lock; | ||
24 | }; | ||
25 | |||
26 | static inline struct rsm_mutex* rsm_mutex_from_lock(struct litmus_lock* lock) | ||
27 | { | ||
28 | return container_of(lock, struct rsm_mutex, litmus_lock); | ||
29 | } | ||
30 | |||
31 | #ifdef CONFIG_LITMUS_DGL_SUPPORT | ||
32 | int rsm_mutex_is_owner(struct litmus_lock *l, struct task_struct *t); | ||
33 | int rsm_mutex_dgl_lock(struct litmus_lock *l, dgl_wait_state_t* dgl_wait, wait_queue_t* wq_node); | ||
34 | void rsm_mutex_enable_priority(struct litmus_lock *l, dgl_wait_state_t* dgl_wait); | ||
35 | #endif | ||
36 | |||
37 | void rsm_mutex_propagate_increase_inheritance(struct litmus_lock* l, | ||
38 | struct task_struct* t, | ||
39 | raw_spinlock_t* to_unlock, | ||
40 | unsigned long irqflags); | ||
41 | |||
42 | void rsm_mutex_propagate_decrease_inheritance(struct litmus_lock* l, | ||
43 | struct task_struct* t, | ||
44 | raw_spinlock_t* to_unlock, | ||
45 | unsigned long irqflags); | ||
46 | |||
47 | int rsm_mutex_lock(struct litmus_lock* l); | ||
48 | int rsm_mutex_unlock(struct litmus_lock* l); | ||
49 | int rsm_mutex_close(struct litmus_lock* l); | ||
50 | void rsm_mutex_free(struct litmus_lock* l); | ||
51 | struct litmus_lock* rsm_mutex_new(struct litmus_lock_ops*); | ||
52 | |||
53 | |||
54 | #endif \ No newline at end of file | ||
diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h index 4cd06dd32906..39685a351cb1 100644 --- a/include/litmus/rt_param.h +++ b/include/litmus/rt_param.h | |||
@@ -1,9 +1,11 @@ | |||
1 | #ifndef _LINUX_RT_PARAM_H_ | ||
2 | #define _LINUX_RT_PARAM_H_ | ||
1 | /* | 3 | /* |
2 | * Definition of the scheduler plugin interface. | 4 | * Definition of the scheduler plugin interface. |
3 | * | 5 | * |
4 | */ | 6 | */ |
5 | #ifndef _LINUX_RT_PARAM_H_ | 7 | |
6 | #define _LINUX_RT_PARAM_H_ | 8 | #include <litmus/fpmath.h> |
7 | 9 | ||
8 | /* Litmus time type. */ | 10 | /* Litmus time type. */ |
9 | typedef unsigned long long lt_t; | 11 | typedef unsigned long long lt_t; |
@@ -30,9 +32,43 @@ typedef enum { | |||
30 | typedef enum { | 32 | typedef enum { |
31 | NO_ENFORCEMENT, /* job may overrun unhindered */ | 33 | NO_ENFORCEMENT, /* job may overrun unhindered */ |
32 | QUANTUM_ENFORCEMENT, /* budgets are only checked on quantum boundaries */ | 34 | QUANTUM_ENFORCEMENT, /* budgets are only checked on quantum boundaries */ |
33 | PRECISE_ENFORCEMENT /* budgets are enforced with hrtimers */ | 35 | PRECISE_ENFORCEMENT, /* budgets are enforced with hrtimers */ |
34 | } budget_policy_t; | 36 | } budget_policy_t; |
35 | 37 | ||
38 | typedef enum { | ||
39 | NO_SIGNALS, /* job receives no signals when it exhausts its budget */ | ||
40 | QUANTUM_SIGNALS, /* budget signals are only sent on quantum boundaries */ | ||
41 | PRECISE_SIGNALS, /* budget signals are triggered with hrtimers */ | ||
42 | } budget_signal_policy_t; | ||
43 | |||
44 | typedef enum { | ||
45 | AUX_ENABLE = 0x1, | ||
46 | AUX_CURRENT = (AUX_ENABLE<<1), | ||
47 | AUX_FUTURE = (AUX_CURRENT<<2) | ||
48 | } aux_flags_t; | ||
49 | |||
50 | /* mirror of st_event_record_type_t | ||
51 | * Assume all are UNsupported, unless otherwise stated. */ | ||
52 | typedef enum { | ||
53 | ST_INJECT_NAME = 1, /* supported */ | ||
54 | ST_INJECT_PARAM, /* supported */ | ||
55 | ST_INJECT_RELEASE, /* supported */ | ||
56 | ST_INJECT_ASSIGNED, | ||
57 | ST_INJECT_SWITCH_TO, | ||
58 | ST_INJECT_SWITCH_AWAY, | ||
59 | ST_INJECT_COMPLETION, /* supported */ | ||
60 | ST_INJECT_BLOCK, | ||
61 | ST_INJECT_RESUME, | ||
62 | ST_INJECT_ACTION, | ||
63 | ST_INJECT_SYS_RELEASE, /* supported */ | ||
64 | } sched_trace_injection_events_t; | ||
65 | |||
66 | struct st_inject_args { | ||
67 | lt_t release; | ||
68 | lt_t deadline; | ||
69 | unsigned int job_no; | ||
70 | }; | ||
71 | |||
36 | /* We use the common priority interpretation "lower index == higher priority", | 72 | /* We use the common priority interpretation "lower index == higher priority", |
37 | * which is commonly used in fixed-priority schedulability analysis papers. | 73 | * which is commonly used in fixed-priority schedulability analysis papers. |
38 | * So, a numerically lower priority value implies higher scheduling priority, | 74 | * So, a numerically lower priority value implies higher scheduling priority, |
@@ -62,6 +98,7 @@ struct rt_task { | |||
62 | unsigned int priority; | 98 | unsigned int priority; |
63 | task_class_t cls; | 99 | task_class_t cls; |
64 | budget_policy_t budget_policy; /* ignored by pfair */ | 100 | budget_policy_t budget_policy; /* ignored by pfair */ |
101 | budget_signal_policy_t budget_signal_policy; /* currently ignored by pfair */ | ||
65 | }; | 102 | }; |
66 | 103 | ||
67 | union np_flag { | 104 | union np_flag { |
@@ -74,6 +111,19 @@ union np_flag { | |||
74 | } np; | 111 | } np; |
75 | }; | 112 | }; |
76 | 113 | ||
114 | struct affinity_observer_args | ||
115 | { | ||
116 | int lock_od; | ||
117 | }; | ||
118 | |||
119 | struct gpu_affinity_observer_args | ||
120 | { | ||
121 | struct affinity_observer_args obs; | ||
122 | int replica_to_gpu_offset; | ||
123 | int nr_simult_users; | ||
124 | int relaxed_rules; | ||
125 | }; | ||
126 | |||
77 | /* The definition of the data that is shared between the kernel and real-time | 127 | /* The definition of the data that is shared between the kernel and real-time |
78 | * tasks via a shared page (see litmus/ctrldev.c). | 128 | * tasks via a shared page (see litmus/ctrldev.c). |
79 | * | 129 | * |
@@ -115,6 +165,13 @@ struct control_page { | |||
115 | /* don't export internal data structures to user space (liblitmus) */ | 165 | /* don't export internal data structures to user space (liblitmus) */ |
116 | #ifdef __KERNEL__ | 166 | #ifdef __KERNEL__ |
117 | 167 | ||
168 | #include <litmus/binheap.h> | ||
169 | #include <linux/semaphore.h> | ||
170 | |||
171 | #ifdef CONFIG_LITMUS_SOFTIRQD | ||
172 | #include <linux/interrupt.h> | ||
173 | #endif | ||
174 | |||
118 | struct _rt_domain; | 175 | struct _rt_domain; |
119 | struct bheap_node; | 176 | struct bheap_node; |
120 | struct release_heap; | 177 | struct release_heap; |
@@ -142,10 +199,82 @@ struct rt_job { | |||
142 | * Increase this sequence number when a job is released. | 199 | * Increase this sequence number when a job is released. |
143 | */ | 200 | */ |
144 | unsigned int job_no; | 201 | unsigned int job_no; |
202 | |||
203 | /* bits: | ||
204 | * 0th: Set if a budget exhaustion signal has already been sent for | ||
205 | * the current job. */ | ||
206 | unsigned long flags; | ||
145 | }; | 207 | }; |
146 | 208 | ||
209 | #define RT_JOB_SIG_BUDGET_SENT 0 | ||
210 | |||
147 | struct pfair_param; | 211 | struct pfair_param; |
148 | 212 | ||
213 | enum klmirqd_sem_status | ||
214 | { | ||
215 | NEED_TO_REACQUIRE, | ||
216 | REACQUIRING, | ||
217 | NOT_HELD, | ||
218 | HELD | ||
219 | }; | ||
220 | |||
221 | typedef enum gpu_migration_dist | ||
222 | { | ||
223 | // TODO: Make this variable against NR_NVIDIA_GPUS | ||
224 | MIG_LOCAL = 0, | ||
225 | MIG_NEAR = 1, | ||
226 | MIG_MED = 2, | ||
227 | MIG_FAR = 3, // 8 GPUs in a binary tree hierarchy | ||
228 | MIG_NONE = 4, | ||
229 | |||
230 | MIG_LAST = MIG_NONE | ||
231 | } gpu_migration_dist_t; | ||
232 | |||
233 | typedef struct feedback_est{ | ||
234 | fp_t est; | ||
235 | fp_t accum_err; | ||
236 | } feedback_est_t; | ||
237 | |||
238 | |||
239 | #define AVG_EST_WINDOW_SIZE 20 | ||
240 | |||
241 | typedef int (*notify_rsrc_exit_t)(struct task_struct* tsk); | ||
242 | |||
243 | typedef struct avg_est{ | ||
244 | lt_t history[AVG_EST_WINDOW_SIZE]; | ||
245 | uint16_t count; | ||
246 | uint16_t idx; | ||
247 | lt_t sum; | ||
248 | lt_t std; | ||
249 | lt_t avg; | ||
250 | } avg_est_t; | ||
251 | |||
252 | |||
253 | |||
254 | #ifdef CONFIG_LITMUS_SOFTIRQD | ||
255 | struct klmirqd_info | ||
256 | { | ||
257 | struct task_struct* klmirqd; | ||
258 | struct task_struct* current_owner; | ||
259 | unsigned int terminating:1; | ||
260 | |||
261 | raw_spinlock_t lock; | ||
262 | |||
263 | u32 pending; | ||
264 | atomic_t num_hi_pending; | ||
265 | atomic_t num_low_pending; | ||
266 | atomic_t num_work_pending; | ||
267 | |||
268 | /* in order of priority */ | ||
269 | struct tasklet_head pending_tasklets_hi; | ||
270 | struct tasklet_head pending_tasklets; | ||
271 | struct list_head worklist; | ||
272 | |||
273 | struct list_head klmirqd_reg; | ||
274 | }; | ||
275 | #endif | ||
276 | |||
277 | |||
149 | /* RT task parameters for scheduling extensions | 278 | /* RT task parameters for scheduling extensions |
150 | * These parameters are inherited during clone and therefore must | 279 | * These parameters are inherited during clone and therefore must |
151 | * be explicitly set up before the task set is launched. | 280 | * be explicitly set up before the task set is launched. |
@@ -163,6 +292,40 @@ struct rt_param { | |||
163 | /* has the task completed? */ | 292 | /* has the task completed? */ |
164 | unsigned int completed:1; | 293 | unsigned int completed:1; |
165 | 294 | ||
295 | #ifdef CONFIG_LITMUS_SOFTIRQD | ||
296 | /* proxy threads have minimum priority by default */ | ||
297 | unsigned int is_interrupt_thread:1; | ||
298 | |||
299 | /* pointer to data used by klmirqd thread. | ||
300 | * | ||
301 | * ptr only valid if is_interrupt_thread == 1 | ||
302 | */ | ||
303 | struct klmirqd_info* klmirqd_info; | ||
304 | #endif | ||
305 | |||
306 | #ifdef CONFIG_LITMUS_NVIDIA | ||
307 | /* number of top-half interrupts handled on behalf of current job */ | ||
308 | atomic_t nv_int_count; | ||
309 | long unsigned int held_gpus; // bitmap of held GPUs. | ||
310 | struct binheap_node gpu_owner_node; // just one GPU for now... | ||
311 | unsigned int hide_from_gpu:1; | ||
312 | |||
313 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING | ||
314 | avg_est_t gpu_migration_est[MIG_LAST+1]; | ||
315 | |||
316 | gpu_migration_dist_t gpu_migration; | ||
317 | int last_gpu; | ||
318 | |||
319 | notify_rsrc_exit_t rsrc_exit_cb; | ||
320 | void* rsrc_exit_cb_args; | ||
321 | |||
322 | lt_t accum_gpu_time; | ||
323 | lt_t gpu_time_stamp; | ||
324 | |||
325 | unsigned int suspend_gpu_tracker_on_block:1; | ||
326 | #endif | ||
327 | #endif | ||
328 | |||
166 | #ifdef CONFIG_LITMUS_LOCKING | 329 | #ifdef CONFIG_LITMUS_LOCKING |
167 | /* Is the task being priority-boosted by a locking protocol? */ | 330 | /* Is the task being priority-boosted by a locking protocol? */ |
168 | unsigned int priority_boosted:1; | 331 | unsigned int priority_boosted:1; |
@@ -182,7 +345,26 @@ struct rt_param { | |||
182 | * could point to self if PI does not result in | 345 | * could point to self if PI does not result in |
183 | * an increased task priority. | 346 | * an increased task priority. |
184 | */ | 347 | */ |
185 | struct task_struct* inh_task; | 348 | struct task_struct* inh_task; |
349 | |||
350 | #ifdef CONFIG_LITMUS_NESTED_LOCKING | ||
351 | raw_spinlock_t hp_blocked_tasks_lock; | ||
352 | struct binheap hp_blocked_tasks; | ||
353 | |||
354 | /* pointer to lock upon which is currently blocked */ | ||
355 | struct litmus_lock* blocked_lock; | ||
356 | #endif | ||
357 | |||
358 | |||
359 | #ifdef CONFIG_REALTIME_AUX_TASKS | ||
360 | unsigned int is_aux_task:1; | ||
361 | unsigned int has_aux_tasks:1; | ||
362 | unsigned int hide_from_aux_tasks:1; | ||
363 | |||
364 | struct list_head aux_task_node; | ||
365 | struct binheap_node aux_task_owner_node; | ||
366 | #endif | ||
367 | |||
186 | 368 | ||
187 | #ifdef CONFIG_NP_SECTION | 369 | #ifdef CONFIG_NP_SECTION |
188 | /* For the FMLP under PSN-EDF, it is required to make the task | 370 | /* For the FMLP under PSN-EDF, it is required to make the task |
@@ -248,6 +430,16 @@ struct rt_param { | |||
248 | struct control_page * ctrl_page; | 430 | struct control_page * ctrl_page; |
249 | }; | 431 | }; |
250 | 432 | ||
433 | #ifdef CONFIG_REALTIME_AUX_TASKS | ||
434 | struct aux_data | ||
435 | { | ||
436 | struct list_head aux_tasks; | ||
437 | struct binheap aux_task_owners; | ||
438 | unsigned int initialized:1; | ||
439 | unsigned int aux_future:1; | ||
440 | }; | ||
251 | #endif | 441 | #endif |
252 | 442 | ||
443 | #endif /* __KERNEL */ | ||
444 | |||
253 | #endif | 445 | #endif |
diff --git a/include/litmus/sched_plugin.h b/include/litmus/sched_plugin.h index 1546ab7f1d66..d0e7d74bb45e 100644 --- a/include/litmus/sched_plugin.h +++ b/include/litmus/sched_plugin.h | |||
@@ -11,6 +11,12 @@ | |||
11 | #include <litmus/locking.h> | 11 | #include <litmus/locking.h> |
12 | #endif | 12 | #endif |
13 | 13 | ||
14 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING | ||
15 | #include <litmus/kexclu_affinity.h> | ||
16 | #endif | ||
17 | |||
18 | #include <linux/interrupt.h> | ||
19 | |||
14 | /************************ setup/tear down ********************/ | 20 | /************************ setup/tear down ********************/ |
15 | 21 | ||
16 | typedef long (*activate_plugin_t) (void); | 22 | typedef long (*activate_plugin_t) (void); |
@@ -29,7 +35,6 @@ typedef struct task_struct* (*schedule_t)(struct task_struct * prev); | |||
29 | */ | 35 | */ |
30 | typedef void (*finish_switch_t)(struct task_struct *prev); | 36 | typedef void (*finish_switch_t)(struct task_struct *prev); |
31 | 37 | ||
32 | |||
33 | /********************* task state changes ********************/ | 38 | /********************* task state changes ********************/ |
34 | 39 | ||
35 | /* Called to setup a new real-time task. | 40 | /* Called to setup a new real-time task. |
@@ -60,6 +65,49 @@ typedef long (*allocate_lock_t) (struct litmus_lock **lock, int type, | |||
60 | void* __user config); | 65 | void* __user config); |
61 | #endif | 66 | #endif |
62 | 67 | ||
68 | struct affinity_observer; | ||
69 | typedef long (*allocate_affinity_observer_t) ( | ||
70 | struct affinity_observer **aff_obs, int type, | ||
71 | void* __user config); | ||
72 | |||
73 | typedef void (*increase_prio_t)(struct task_struct* t, struct task_struct* prio_inh); | ||
74 | typedef void (*decrease_prio_t)(struct task_struct* t, struct task_struct* prio_inh); | ||
75 | |||
76 | typedef int (*__increase_prio_t)(struct task_struct* t, struct task_struct* prio_inh); | ||
77 | typedef int (*__decrease_prio_t)(struct task_struct* t, struct task_struct* prio_inh); | ||
78 | |||
79 | typedef void (*nested_increase_prio_t)(struct task_struct* t, struct task_struct* prio_inh, | ||
80 | raw_spinlock_t *to_unlock, unsigned long irqflags); | ||
81 | typedef void (*nested_decrease_prio_t)(struct task_struct* t, struct task_struct* prio_inh, | ||
82 | raw_spinlock_t *to_unlock, unsigned long irqflags); | ||
83 | |||
84 | |||
85 | typedef int (*enqueue_pai_tasklet_t)(struct tasklet_struct* tasklet); | ||
86 | typedef void (*change_prio_pai_tasklet_t)(struct task_struct *old_prio, | ||
87 | struct task_struct *new_prio); | ||
88 | typedef void (*run_tasklets_t)(struct task_struct* next); | ||
89 | |||
90 | typedef raw_spinlock_t* (*get_dgl_spinlock_t) (struct task_struct *t); | ||
91 | |||
92 | |||
93 | typedef int (*higher_prio_t)(struct task_struct* a, struct task_struct* b); | ||
94 | |||
95 | #ifdef CONFIG_LITMUS_NESTED_LOCKING | ||
96 | |||
97 | typedef enum | ||
98 | { | ||
99 | BASE, | ||
100 | EFFECTIVE | ||
101 | } comparison_mode_t; | ||
102 | |||
103 | typedef int (*__higher_prio_t)(struct task_struct* a, comparison_mode_t a_mod, | ||
104 | struct task_struct* b, comparison_mode_t b_mod); | ||
105 | #endif | ||
106 | |||
107 | #if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_SOFTIRQD) | ||
108 | typedef int (*default_cpu_for_gpu_t)(int gpu); | ||
109 | #endif | ||
110 | |||
63 | 111 | ||
64 | /********************* sys call backends ********************/ | 112 | /********************* sys call backends ********************/ |
65 | /* This function causes the caller to sleep until the next release */ | 113 | /* This function causes the caller to sleep until the next release */ |
@@ -90,14 +138,42 @@ struct sched_plugin { | |||
90 | /* task state changes */ | 138 | /* task state changes */ |
91 | admit_task_t admit_task; | 139 | admit_task_t admit_task; |
92 | 140 | ||
93 | task_new_t task_new; | 141 | task_new_t task_new; |
94 | task_wake_up_t task_wake_up; | 142 | task_wake_up_t task_wake_up; |
95 | task_block_t task_block; | 143 | task_block_t task_block; |
96 | task_exit_t task_exit; | 144 | task_exit_t task_exit; |
97 | 145 | ||
146 | higher_prio_t compare; | ||
147 | |||
98 | #ifdef CONFIG_LITMUS_LOCKING | 148 | #ifdef CONFIG_LITMUS_LOCKING |
99 | /* locking protocols */ | 149 | /* locking protocols */ |
100 | allocate_lock_t allocate_lock; | 150 | allocate_lock_t allocate_lock; |
151 | increase_prio_t increase_prio; | ||
152 | decrease_prio_t decrease_prio; | ||
153 | |||
154 | __increase_prio_t __increase_prio; | ||
155 | __decrease_prio_t __decrease_prio; | ||
156 | #endif | ||
157 | #ifdef CONFIG_LITMUS_NESTED_LOCKING | ||
158 | nested_increase_prio_t nested_increase_prio; | ||
159 | nested_decrease_prio_t nested_decrease_prio; | ||
160 | __higher_prio_t __compare; | ||
161 | #endif | ||
162 | #ifdef CONFIG_LITMUS_DGL_SUPPORT | ||
163 | get_dgl_spinlock_t get_dgl_spinlock; | ||
164 | #endif | ||
165 | |||
166 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING | ||
167 | allocate_affinity_observer_t allocate_aff_obs; | ||
168 | #endif | ||
169 | #ifdef CONFIG_LITMUS_PAI_SOFTIRQD | ||
170 | enqueue_pai_tasklet_t enqueue_pai_tasklet; | ||
171 | change_prio_pai_tasklet_t change_prio_pai_tasklet; | ||
172 | run_tasklets_t run_tasklets; | ||
173 | #endif | ||
174 | |||
175 | #if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_SOFTIRQD) | ||
176 | default_cpu_for_gpu_t map_gpu_to_cpu; | ||
101 | #endif | 177 | #endif |
102 | } __attribute__ ((__aligned__(SMP_CACHE_BYTES))); | 178 | } __attribute__ ((__aligned__(SMP_CACHE_BYTES))); |
103 | 179 | ||
diff --git a/include/litmus/sched_trace.h b/include/litmus/sched_trace.h index 82bde8241298..7af12f49c600 100644 --- a/include/litmus/sched_trace.h +++ b/include/litmus/sched_trace.h | |||
@@ -10,13 +10,14 @@ struct st_trace_header { | |||
10 | u8 type; /* Of what type is this record? */ | 10 | u8 type; /* Of what type is this record? */ |
11 | u8 cpu; /* On which CPU was it recorded? */ | 11 | u8 cpu; /* On which CPU was it recorded? */ |
12 | u16 pid; /* PID of the task. */ | 12 | u16 pid; /* PID of the task. */ |
13 | u32 job; /* The job sequence number. */ | 13 | u32 job:24; /* The job sequence number. */ |
14 | }; | 14 | u8 extra; |
15 | } __attribute__((packed)); | ||
15 | 16 | ||
16 | #define ST_NAME_LEN 16 | 17 | #define ST_NAME_LEN 16 |
17 | struct st_name_data { | 18 | struct st_name_data { |
18 | char cmd[ST_NAME_LEN];/* The name of the executable of this process. */ | 19 | char cmd[ST_NAME_LEN];/* The name of the executable of this process. */ |
19 | }; | 20 | } __attribute__((packed)); |
20 | 21 | ||
21 | struct st_param_data { /* regular params */ | 22 | struct st_param_data { /* regular params */ |
22 | u32 wcet; | 23 | u32 wcet; |
@@ -25,30 +26,29 @@ struct st_param_data { /* regular params */ | |||
25 | u8 partition; | 26 | u8 partition; |
26 | u8 class; | 27 | u8 class; |
27 | u8 __unused[2]; | 28 | u8 __unused[2]; |
28 | }; | 29 | } __attribute__((packed)); |
29 | 30 | ||
30 | struct st_release_data { /* A job is was/is going to be released. */ | 31 | struct st_release_data { /* A job is was/is going to be released. */ |
31 | u64 release; /* What's the release time? */ | 32 | u64 release; /* What's the release time? */ |
32 | u64 deadline; /* By when must it finish? */ | 33 | u64 deadline; /* By when must it finish? */ |
33 | }; | 34 | } __attribute__((packed)); |
34 | 35 | ||
35 | struct st_assigned_data { /* A job was asigned to a CPU. */ | 36 | struct st_assigned_data { /* A job was asigned to a CPU. */ |
36 | u64 when; | 37 | u64 when; |
37 | u8 target; /* Where should it execute? */ | 38 | u8 target; /* Where should it execute? */ |
38 | u8 __unused[7]; | 39 | u8 __unused[7]; |
39 | }; | 40 | } __attribute__((packed)); |
40 | 41 | ||
41 | struct st_switch_to_data { /* A process was switched to on a given CPU. */ | 42 | struct st_switch_to_data { /* A process was switched to on a given CPU. */ |
42 | u64 when; /* When did this occur? */ | 43 | u64 when; /* When did this occur? */ |
43 | u32 exec_time; /* Time the current job has executed. */ | 44 | u32 exec_time; /* Time the current job has executed. */ |
44 | u8 __unused[4]; | 45 | u8 __unused[4]; |
45 | 46 | } __attribute__((packed)); | |
46 | }; | ||
47 | 47 | ||
48 | struct st_switch_away_data { /* A process was switched away from on a given CPU. */ | 48 | struct st_switch_away_data { /* A process was switched away from on a given CPU. */ |
49 | u64 when; | 49 | u64 when; |
50 | u64 exec_time; | 50 | u64 exec_time; |
51 | }; | 51 | } __attribute__((packed)); |
52 | 52 | ||
53 | struct st_completion_data { /* A job completed. */ | 53 | struct st_completion_data { /* A job completed. */ |
54 | u64 when; | 54 | u64 when; |
@@ -56,35 +56,108 @@ struct st_completion_data { /* A job completed. */ | |||
56 | * next task automatically; set to 0 otherwise. | 56 | * next task automatically; set to 0 otherwise. |
57 | */ | 57 | */ |
58 | u8 __uflags:7; | 58 | u8 __uflags:7; |
59 | u8 __unused[7]; | 59 | u16 nv_int_count; |
60 | }; | 60 | u8 __unused[5]; |
61 | } __attribute__((packed)); | ||
61 | 62 | ||
62 | struct st_block_data { /* A task blocks. */ | 63 | struct st_block_data { /* A task blocks. */ |
63 | u64 when; | 64 | u64 when; |
64 | u64 __unused; | 65 | u64 __unused; |
65 | }; | 66 | } __attribute__((packed)); |
66 | 67 | ||
67 | struct st_resume_data { /* A task resumes. */ | 68 | struct st_resume_data { /* A task resumes. */ |
68 | u64 when; | 69 | u64 when; |
69 | u64 __unused; | 70 | u64 __unused; |
70 | }; | 71 | } __attribute__((packed)); |
71 | 72 | ||
72 | struct st_action_data { | 73 | struct st_action_data { |
73 | u64 when; | 74 | u64 when; |
74 | u8 action; | 75 | u8 action; |
75 | u8 __unused[7]; | 76 | u8 __unused[7]; |
76 | }; | 77 | } __attribute__((packed)); |
77 | 78 | ||
78 | struct st_sys_release_data { | 79 | struct st_sys_release_data { |
79 | u64 when; | 80 | u64 when; |
80 | u64 release; | 81 | u64 release; |
81 | }; | 82 | } __attribute__((packed)); |
83 | |||
84 | |||
85 | struct st_tasklet_release_data { | ||
86 | u64 when; | ||
87 | u64 __unused; | ||
88 | } __attribute__((packed)); | ||
89 | |||
90 | struct st_tasklet_begin_data { | ||
91 | u64 when; | ||
92 | u16 exe_pid; | ||
93 | u8 __unused[6]; | ||
94 | } __attribute__((packed)); | ||
95 | |||
96 | struct st_tasklet_end_data { | ||
97 | u64 when; | ||
98 | u16 exe_pid; | ||
99 | u8 flushed; | ||
100 | u8 __unused[5]; | ||
101 | } __attribute__((packed)); | ||
102 | |||
103 | |||
104 | struct st_work_release_data { | ||
105 | u64 when; | ||
106 | u64 __unused; | ||
107 | } __attribute__((packed)); | ||
108 | |||
109 | struct st_work_begin_data { | ||
110 | u64 when; | ||
111 | u16 exe_pid; | ||
112 | u8 __unused[6]; | ||
113 | } __attribute__((packed)); | ||
114 | |||
115 | struct st_work_end_data { | ||
116 | u64 when; | ||
117 | u16 exe_pid; | ||
118 | u8 flushed; | ||
119 | u8 __unused[5]; | ||
120 | } __attribute__((packed)); | ||
121 | |||
122 | struct st_effective_priority_change_data { | ||
123 | u64 when; | ||
124 | u16 inh_pid; | ||
125 | u8 __unused[6]; | ||
126 | } __attribute__((packed)); | ||
127 | |||
128 | struct st_nv_interrupt_begin_data { | ||
129 | u64 when; | ||
130 | u32 device; | ||
131 | u32 serialNumber; | ||
132 | } __attribute__((packed)); | ||
133 | |||
134 | struct st_nv_interrupt_end_data { | ||
135 | u64 when; | ||
136 | u32 device; | ||
137 | u32 serialNumber; | ||
138 | } __attribute__((packed)); | ||
139 | |||
140 | struct st_prediction_err_data { | ||
141 | u64 distance; | ||
142 | u64 rel_err; | ||
143 | } __attribute__((packed)); | ||
144 | |||
145 | struct st_migration_data { | ||
146 | u64 observed; | ||
147 | u64 estimated; | ||
148 | } __attribute__((packed)); | ||
149 | |||
150 | struct migration_info { | ||
151 | u64 observed; | ||
152 | u64 estimated; | ||
153 | u8 distance; | ||
154 | } __attribute__((packed)); | ||
82 | 155 | ||
83 | #define DATA(x) struct st_ ## x ## _data x; | 156 | #define DATA(x) struct st_ ## x ## _data x; |
84 | 157 | ||
85 | typedef enum { | 158 | typedef enum { |
86 | ST_NAME = 1, /* Start at one, so that we can spot | 159 | ST_NAME = 1, /* Start at one, so that we can spot |
87 | * uninitialized records. */ | 160 | * uninitialized records. */ |
88 | ST_PARAM, | 161 | ST_PARAM, |
89 | ST_RELEASE, | 162 | ST_RELEASE, |
90 | ST_ASSIGNED, | 163 | ST_ASSIGNED, |
@@ -94,7 +167,19 @@ typedef enum { | |||
94 | ST_BLOCK, | 167 | ST_BLOCK, |
95 | ST_RESUME, | 168 | ST_RESUME, |
96 | ST_ACTION, | 169 | ST_ACTION, |
97 | ST_SYS_RELEASE | 170 | ST_SYS_RELEASE, |
171 | ST_TASKLET_RELEASE, | ||
172 | ST_TASKLET_BEGIN, | ||
173 | ST_TASKLET_END, | ||
174 | ST_WORK_RELEASE, | ||
175 | ST_WORK_BEGIN, | ||
176 | ST_WORK_END, | ||
177 | ST_EFF_PRIO_CHANGE, | ||
178 | ST_NV_INTERRUPT_BEGIN, | ||
179 | ST_NV_INTERRUPT_END, | ||
180 | |||
181 | ST_PREDICTION_ERR, | ||
182 | ST_MIGRATION, | ||
98 | } st_event_record_type_t; | 183 | } st_event_record_type_t; |
99 | 184 | ||
100 | struct st_event_record { | 185 | struct st_event_record { |
@@ -113,8 +198,20 @@ struct st_event_record { | |||
113 | DATA(resume); | 198 | DATA(resume); |
114 | DATA(action); | 199 | DATA(action); |
115 | DATA(sys_release); | 200 | DATA(sys_release); |
201 | DATA(tasklet_release); | ||
202 | DATA(tasklet_begin); | ||
203 | DATA(tasklet_end); | ||
204 | DATA(work_release); | ||
205 | DATA(work_begin); | ||
206 | DATA(work_end); | ||
207 | DATA(effective_priority_change); | ||
208 | DATA(nv_interrupt_begin); | ||
209 | DATA(nv_interrupt_end); | ||
210 | |||
211 | DATA(prediction_err); | ||
212 | DATA(migration); | ||
116 | } data; | 213 | } data; |
117 | }; | 214 | } __attribute__((packed)); |
118 | 215 | ||
119 | #undef DATA | 216 | #undef DATA |
120 | 217 | ||
@@ -129,6 +226,8 @@ struct st_event_record { | |||
129 | ft_event1(id, callback, task) | 226 | ft_event1(id, callback, task) |
130 | #define SCHED_TRACE2(id, callback, task, xtra) \ | 227 | #define SCHED_TRACE2(id, callback, task, xtra) \ |
131 | ft_event2(id, callback, task, xtra) | 228 | ft_event2(id, callback, task, xtra) |
229 | #define SCHED_TRACE3(id, callback, task, xtra1, xtra2) \ | ||
230 | ft_event3(id, callback, task, xtra1, xtra2) | ||
132 | 231 | ||
133 | /* provide prototypes; needed on sparc64 */ | 232 | /* provide prototypes; needed on sparc64 */ |
134 | #ifndef NO_TASK_TRACE_DECLS | 233 | #ifndef NO_TASK_TRACE_DECLS |
@@ -155,12 +254,58 @@ feather_callback void do_sched_trace_action(unsigned long id, | |||
155 | feather_callback void do_sched_trace_sys_release(unsigned long id, | 254 | feather_callback void do_sched_trace_sys_release(unsigned long id, |
156 | lt_t* start); | 255 | lt_t* start); |
157 | 256 | ||
257 | |||
258 | feather_callback void do_sched_trace_tasklet_release(unsigned long id, | ||
259 | struct task_struct* owner); | ||
260 | feather_callback void do_sched_trace_tasklet_begin(unsigned long id, | ||
261 | struct task_struct* owner); | ||
262 | feather_callback void do_sched_trace_tasklet_end(unsigned long id, | ||
263 | struct task_struct* owner, | ||
264 | unsigned long flushed); | ||
265 | |||
266 | feather_callback void do_sched_trace_work_release(unsigned long id, | ||
267 | struct task_struct* owner); | ||
268 | feather_callback void do_sched_trace_work_begin(unsigned long id, | ||
269 | struct task_struct* owner, | ||
270 | struct task_struct* exe); | ||
271 | feather_callback void do_sched_trace_work_end(unsigned long id, | ||
272 | struct task_struct* owner, | ||
273 | struct task_struct* exe, | ||
274 | unsigned long flushed); | ||
275 | |||
276 | feather_callback void do_sched_trace_eff_prio_change(unsigned long id, | ||
277 | struct task_struct* task, | ||
278 | struct task_struct* inh); | ||
279 | |||
280 | feather_callback void do_sched_trace_nv_interrupt_begin(unsigned long id, | ||
281 | u32 device); | ||
282 | feather_callback void do_sched_trace_nv_interrupt_end(unsigned long id, | ||
283 | unsigned long unused); | ||
284 | |||
285 | feather_callback void do_sched_trace_prediction_err(unsigned long id, | ||
286 | struct task_struct* task, | ||
287 | gpu_migration_dist_t* distance, | ||
288 | fp_t* rel_err); | ||
289 | |||
290 | |||
291 | |||
292 | |||
293 | |||
294 | feather_callback void do_sched_trace_migration(unsigned long id, | ||
295 | struct task_struct* task, | ||
296 | struct migration_info* mig_info); | ||
297 | |||
298 | |||
299 | /* returns true if we're tracing an interrupt on current CPU */ | ||
300 | /* int is_interrupt_tracing_active(void); */ | ||
301 | |||
158 | #endif | 302 | #endif |
159 | 303 | ||
160 | #else | 304 | #else |
161 | 305 | ||
162 | #define SCHED_TRACE(id, callback, task) /* no tracing */ | 306 | #define SCHED_TRACE(id, callback, task) /* no tracing */ |
163 | #define SCHED_TRACE2(id, callback, task, xtra) /* no tracing */ | 307 | #define SCHED_TRACE2(id, callback, task, xtra) /* no tracing */ |
308 | #define SCHED_TRACE3(id, callback, task, xtra1, xtra2) | ||
164 | 309 | ||
165 | #endif | 310 | #endif |
166 | 311 | ||
@@ -252,6 +397,41 @@ feather_callback void do_sched_trace_sys_release(unsigned long id, | |||
252 | trace_litmus_sys_release(when); \ | 397 | trace_litmus_sys_release(when); \ |
253 | } while (0) | 398 | } while (0) |
254 | 399 | ||
400 | #define sched_trace_tasklet_release(t) \ | ||
401 | SCHED_TRACE(SCHED_TRACE_BASE_ID + 11, do_sched_trace_tasklet_release, t) | ||
402 | |||
403 | #define sched_trace_tasklet_begin(t) \ | ||
404 | SCHED_TRACE(SCHED_TRACE_BASE_ID + 12, do_sched_trace_tasklet_begin, t) | ||
405 | |||
406 | #define sched_trace_tasklet_end(t, flushed) \ | ||
407 | SCHED_TRACE2(SCHED_TRACE_BASE_ID + 13, do_sched_trace_tasklet_end, t, flushed) | ||
408 | |||
409 | |||
410 | #define sched_trace_work_release(t) \ | ||
411 | SCHED_TRACE(SCHED_TRACE_BASE_ID + 14, do_sched_trace_work_release, t) | ||
412 | |||
413 | #define sched_trace_work_begin(t, e) \ | ||
414 | SCHED_TRACE2(SCHED_TRACE_BASE_ID + 15, do_sched_trace_work_begin, t, e) | ||
415 | |||
416 | #define sched_trace_work_end(t, e, flushed) \ | ||
417 | SCHED_TRACE3(SCHED_TRACE_BASE_ID + 16, do_sched_trace_work_end, t, e, flushed) | ||
418 | |||
419 | |||
420 | #define sched_trace_eff_prio_change(t, inh) \ | ||
421 | SCHED_TRACE2(SCHED_TRACE_BASE_ID + 17, do_sched_trace_eff_prio_change, t, inh) | ||
422 | |||
423 | |||
424 | #define sched_trace_nv_interrupt_begin(d) \ | ||
425 | SCHED_TRACE(SCHED_TRACE_BASE_ID + 18, do_sched_trace_nv_interrupt_begin, d) | ||
426 | #define sched_trace_nv_interrupt_end(d) \ | ||
427 | SCHED_TRACE(SCHED_TRACE_BASE_ID + 19, do_sched_trace_nv_interrupt_end, d) | ||
428 | |||
429 | #define sched_trace_prediction_err(t, dist, rel_err) \ | ||
430 | SCHED_TRACE3(SCHED_TRACE_BASE_ID + 20, do_sched_trace_prediction_err, t, dist, rel_err) | ||
431 | |||
432 | #define sched_trace_migration(t, mig_info) \ | ||
433 | SCHED_TRACE2(SCHED_TRACE_BASE_ID + 21, do_sched_trace_migration, t, mig_info) | ||
434 | |||
255 | #define sched_trace_quantum_boundary() /* NOT IMPLEMENTED */ | 435 | #define sched_trace_quantum_boundary() /* NOT IMPLEMENTED */ |
256 | 436 | ||
257 | #endif /* __KERNEL__ */ | 437 | #endif /* __KERNEL__ */ |
diff --git a/include/litmus/sched_trace_external.h b/include/litmus/sched_trace_external.h new file mode 100644 index 000000000000..e70e45e4cf51 --- /dev/null +++ b/include/litmus/sched_trace_external.h | |||
@@ -0,0 +1,78 @@ | |||
1 | /* | ||
2 | * sched_trace.h -- record scheduler events to a byte stream for offline analysis. | ||
3 | */ | ||
4 | #ifndef _LINUX_SCHED_TRACE_EXTERNAL_H_ | ||
5 | #define _LINUX_SCHED_TRACE_EXTERNAL_H_ | ||
6 | |||
7 | |||
8 | #ifdef CONFIG_SCHED_TASK_TRACE | ||
9 | extern void __sched_trace_tasklet_begin_external(struct task_struct* t); | ||
10 | static inline void sched_trace_tasklet_begin_external(struct task_struct* t) | ||
11 | { | ||
12 | __sched_trace_tasklet_begin_external(t); | ||
13 | } | ||
14 | |||
15 | extern void __sched_trace_tasklet_end_external(struct task_struct* t, unsigned long flushed); | ||
16 | static inline void sched_trace_tasklet_end_external(struct task_struct* t, unsigned long flushed) | ||
17 | { | ||
18 | __sched_trace_tasklet_end_external(t, flushed); | ||
19 | } | ||
20 | |||
21 | extern void __sched_trace_work_begin_external(struct task_struct* t, struct task_struct* e); | ||
22 | static inline void sched_trace_work_begin_external(struct task_struct* t, struct task_struct* e) | ||
23 | { | ||
24 | __sched_trace_work_begin_external(t, e); | ||
25 | } | ||
26 | |||
27 | extern void __sched_trace_work_end_external(struct task_struct* t, struct task_struct* e, unsigned long f); | ||
28 | static inline void sched_trace_work_end_external(struct task_struct* t, struct task_struct* e, unsigned long f) | ||
29 | { | ||
30 | __sched_trace_work_end_external(t, e, f); | ||
31 | } | ||
32 | |||
33 | #ifdef CONFIG_LITMUS_NVIDIA | ||
34 | extern void __sched_trace_nv_interrupt_begin_external(u32 device); | ||
35 | static inline void sched_trace_nv_interrupt_begin_external(u32 device) | ||
36 | { | ||
37 | __sched_trace_nv_interrupt_begin_external(device); | ||
38 | } | ||
39 | |||
40 | extern void __sched_trace_nv_interrupt_end_external(u32 device); | ||
41 | static inline void sched_trace_nv_interrupt_end_external(u32 device) | ||
42 | { | ||
43 | __sched_trace_nv_interrupt_end_external(device); | ||
44 | } | ||
45 | #endif | ||
46 | |||
47 | #else | ||
48 | |||
49 | // no tracing. | ||
50 | static inline void sched_trace_tasklet_begin_external(struct task_struct* t){} | ||
51 | static inline void sched_trace_tasklet_end_external(struct task_struct* t, unsigned long flushed){} | ||
52 | static inline void sched_trace_work_begin_external(struct task_struct* t, struct task_struct* e){} | ||
53 | static inline void sched_trace_work_end_external(struct task_struct* t, struct task_struct* e, unsigned long f){} | ||
54 | |||
55 | #ifdef CONFIG_LITMUS_NVIDIA | ||
56 | static inline void sched_trace_nv_interrupt_begin_external(u32 device){} | ||
57 | static inline void sched_trace_nv_interrupt_end_external(u32 device){} | ||
58 | #endif | ||
59 | |||
60 | #endif | ||
61 | |||
62 | |||
63 | #ifdef CONFIG_LITMUS_NVIDIA | ||
64 | |||
65 | #define EX_TS(evt) \ | ||
66 | extern void __##evt(void); \ | ||
67 | static inline void EX_##evt(void) { __##evt(); } | ||
68 | |||
69 | EX_TS(TS_NV_TOPISR_START) | ||
70 | EX_TS(TS_NV_TOPISR_END) | ||
71 | EX_TS(TS_NV_BOTISR_START) | ||
72 | EX_TS(TS_NV_BOTISR_END) | ||
73 | EX_TS(TS_NV_RELEASE_BOTISR_START) | ||
74 | EX_TS(TS_NV_RELEASE_BOTISR_END) | ||
75 | |||
76 | #endif | ||
77 | |||
78 | #endif | ||
diff --git a/include/litmus/signal.h b/include/litmus/signal.h new file mode 100644 index 000000000000..38c3207951e0 --- /dev/null +++ b/include/litmus/signal.h | |||
@@ -0,0 +1,47 @@ | |||
1 | #ifndef LITMUS_SIGNAL_H | ||
2 | #define LITMUS_SIGNAL_H | ||
3 | |||
4 | #ifdef __KERNEL__ | ||
5 | #include <linux/signal.h> | ||
6 | #else | ||
7 | #include <signal.h> | ||
8 | #endif | ||
9 | |||
10 | /* Signals used by Litmus to asynchronously communicate events | ||
11 | * to real-time tasks. | ||
12 | * | ||
13 | * Signal values overlap with [SIGRTMIN, SIGRTMAX], so beware of | ||
14 | * application-level conflicts when dealing with COTS user-level | ||
15 | * code. | ||
16 | */ | ||
17 | |||
18 | /* Sent to a Litmus task when all of the following conditions are true: | ||
19 | * (1) The task has exhausted its budget. | ||
20 | * (2) budget_signal_policy is QUANTUM_SIGNALS or PRECISE_SIGNALS. | ||
21 | * | ||
22 | * Note: If a task does not have a registered handler for SIG_BUDGET, | ||
23 | * the signal will cause the task to terminate (default action). | ||
24 | */ | ||
25 | |||
26 | /* Assigned values start at SIGRTMAX and decrease, hopefully reducing | ||
27 | * likelihood of user-level conflicts. | ||
28 | */ | ||
29 | #define SIG_BUDGET (SIGRTMAX - 0) | ||
30 | |||
31 | /* | ||
32 | Future signals could include: | ||
33 | |||
34 | #define SIG_DEADLINE_MISS (SIGRTMAX - 1) | ||
35 | #define SIG_CRIT_LEVEL_CHANGE (SIGRTMAX - 2) | ||
36 | */ | ||
37 | |||
38 | #define SIGLITMUSMIN SIG_BUDGET | ||
39 | |||
40 | #ifdef __KERNEL__ | ||
41 | #if (SIGLITMUSMIN < SIGRTMIN) | ||
42 | /* no compile-time check in user-space since SIGRTMIN may be a variable. */ | ||
43 | #error "Too many LITMUS^RT signals!" | ||
44 | #endif | ||
45 | #endif | ||
46 | |||
47 | #endif | ||
diff --git a/include/litmus/trace.h b/include/litmus/trace.h index 8ad4966c602e..15bd645d2466 100644 --- a/include/litmus/trace.h +++ b/include/litmus/trace.h | |||
@@ -137,9 +137,34 @@ feather_callback void save_timestamp_hide_irq(unsigned long event); | |||
137 | #define TS_EXIT_NP_START TIMESTAMP(150) | 137 | #define TS_EXIT_NP_START TIMESTAMP(150) |
138 | #define TS_EXIT_NP_END TIMESTAMP(151) | 138 | #define TS_EXIT_NP_END TIMESTAMP(151) |
139 | 139 | ||
140 | #ifdef CONFIG_LITMUS_DGL_SUPPORT | ||
141 | #define TS_DGL_LOCK_START TIMESTAMP(175) | ||
142 | #define TS_DGL_LOCK_SUSPEND TIMESTAMP(176) | ||
143 | #define TS_DGL_LOCK_RESUME TIMESTAMP(177) | ||
144 | #define TS_DGL_LOCK_END TIMESTAMP(178) | ||
145 | #define TS_DGL_UNLOCK_START TIMESTAMP(185) | ||
146 | #define TS_DGL_UNLOCK_END TIMESTAMP(186) | ||
147 | #endif | ||
148 | |||
140 | #define TS_SEND_RESCHED_START(c) CTIMESTAMP(190, c) | 149 | #define TS_SEND_RESCHED_START(c) CTIMESTAMP(190, c) |
141 | #define TS_SEND_RESCHED_END TIMESTAMP_IN_IRQ(191) | 150 | #define TS_SEND_RESCHED_END TIMESTAMP_IN_IRQ(191) |
142 | 151 | ||
152 | #ifdef CONFIG_LITMUS_NVIDIA | ||
153 | #define TS_NV_TOPISR_START TIMESTAMP(200) | ||
154 | #define TS_NV_TOPISR_END TIMESTAMP(201) | ||
155 | |||
156 | #define TS_NV_BOTISR_START TIMESTAMP(202) | ||
157 | #define TS_NV_BOTISR_END TIMESTAMP(203) | ||
158 | |||
159 | #define TS_NV_RELEASE_BOTISR_START TIMESTAMP(204) | ||
160 | #define TS_NV_RELEASE_BOTISR_END TIMESTAMP(205) | ||
161 | #endif | ||
162 | |||
163 | #ifdef CONFIG_LITMUS_PAI_SOFTIRQD | ||
164 | #define TS_NV_SCHED_BOTISR_START TIMESTAMP(206) | ||
165 | #define TS_NV_SCHED_BOTISR_END TIMESTAMP(207) | ||
166 | #endif | ||
167 | |||
143 | #define TS_RELEASE_LATENCY(when) LTIMESTAMP(208, &(when)) | 168 | #define TS_RELEASE_LATENCY(when) LTIMESTAMP(208, &(when)) |
144 | 169 | ||
145 | #endif /* !_SYS_TRACE_H_ */ | 170 | #endif /* !_SYS_TRACE_H_ */ |
diff --git a/include/litmus/unistd_32.h b/include/litmus/unistd_32.h index 94264c27d9ac..d1fe84a5d574 100644 --- a/include/litmus/unistd_32.h +++ b/include/litmus/unistd_32.h | |||
@@ -17,5 +17,11 @@ | |||
17 | #define __NR_wait_for_ts_release __LSC(9) | 17 | #define __NR_wait_for_ts_release __LSC(9) |
18 | #define __NR_release_ts __LSC(10) | 18 | #define __NR_release_ts __LSC(10) |
19 | #define __NR_null_call __LSC(11) | 19 | #define __NR_null_call __LSC(11) |
20 | #define __NR_litmus_dgl_lock __LSC(12) | ||
21 | #define __NR_litmus_dgl_unlock __LSC(13) | ||
20 | 22 | ||
21 | #define NR_litmus_syscalls 12 | 23 | #define __NR_set_aux_tasks __LSC(14) |
24 | |||
25 | #define __NR_sched_trace_event __LSC(15) | ||
26 | |||
27 | #define NR_litmus_syscalls 16 | ||
diff --git a/include/litmus/unistd_64.h b/include/litmus/unistd_64.h index d5ced0d2642c..75f9fcb897f5 100644 --- a/include/litmus/unistd_64.h +++ b/include/litmus/unistd_64.h | |||
@@ -29,5 +29,15 @@ __SYSCALL(__NR_wait_for_ts_release, sys_wait_for_ts_release) | |||
29 | __SYSCALL(__NR_release_ts, sys_release_ts) | 29 | __SYSCALL(__NR_release_ts, sys_release_ts) |
30 | #define __NR_null_call __LSC(11) | 30 | #define __NR_null_call __LSC(11) |
31 | __SYSCALL(__NR_null_call, sys_null_call) | 31 | __SYSCALL(__NR_null_call, sys_null_call) |
32 | #define __NR_litmus_dgl_lock __LSC(12) | ||
33 | __SYSCALL(__NR_litmus_dgl_lock, sys_litmus_dgl_lock) | ||
34 | #define __NR_litmus_dgl_unlock __LSC(13) | ||
35 | __SYSCALL(__NR_litmus_dgl_unlock, sys_litmus_dgl_unlock) | ||
32 | 36 | ||
33 | #define NR_litmus_syscalls 12 | 37 | #define __NR_set_aux_tasks __LSC(14) |
38 | __SYSCALL(__NR_set_aux_tasks, sys_set_aux_tasks) | ||
39 | |||
40 | #define __NR_sched_trace_event __LSC(15) | ||
41 | __SYSCALL(__NR_sched_trace_event, sys_sched_trace_event) | ||
42 | |||
43 | #define NR_litmus_syscalls 16 | ||