diff options
author | Glenn Elliott <gelliott@cs.unc.edu> | 2012-04-18 21:33:21 -0400 |
---|---|---|
committer | Glenn Elliott <gelliott@cs.unc.edu> | 2012-04-18 21:33:21 -0400 |
commit | 149ef3b424a49e6b928c5e23fea83380ed95ea38 (patch) | |
tree | 38b9a7397875be56f31f9f04f86fcf1f9e4966ac | |
parent | f916cdb8e6a9ee2c917fddb7351e6bb39f6c953e (diff) |
Zap line-endings
29 files changed, 877 insertions, 877 deletions
diff --git a/include/litmus/fdso.h b/include/litmus/fdso.h index 552a1e731672..1f5d3bd1a1db 100644 --- a/include/litmus/fdso.h +++ b/include/litmus/fdso.h | |||
@@ -19,7 +19,7 @@ typedef enum { | |||
19 | 19 | ||
20 | FMLP_SEM = 0, | 20 | FMLP_SEM = 0, |
21 | SRP_SEM = 1, | 21 | SRP_SEM = 1, |
22 | 22 | ||
23 | RSM_MUTEX = 2, | 23 | RSM_MUTEX = 2, |
24 | IKGLP_SEM = 3, | 24 | IKGLP_SEM = 3, |
25 | KFMLP_SEM = 4, | 25 | KFMLP_SEM = 4, |
@@ -28,7 +28,7 @@ typedef enum { | |||
28 | IKGLP_GPU_AFF_OBS = 6, | 28 | IKGLP_GPU_AFF_OBS = 6, |
29 | KFMLP_SIMPLE_GPU_AFF_OBS = 7, | 29 | KFMLP_SIMPLE_GPU_AFF_OBS = 7, |
30 | KFMLP_GPU_AFF_OBS = 8, | 30 | KFMLP_GPU_AFF_OBS = 8, |
31 | 31 | ||
32 | MAX_OBJ_TYPE = 8 | 32 | MAX_OBJ_TYPE = 8 |
33 | } obj_type_t; | 33 | } obj_type_t; |
34 | 34 | ||
diff --git a/include/litmus/fpmath.h b/include/litmus/fpmath.h index d062b5ab5dc2..ba4121eaa1bf 100644 --- a/include/litmus/fpmath.h +++ b/include/litmus/fpmath.h | |||
@@ -33,11 +33,11 @@ static inline fp_t _frac(fpbuf_t a, fpbuf_t b) | |||
33 | return _fp(FP(a).val / (b)); | 33 | return _fp(FP(a).val / (b)); |
34 | } | 34 | } |
35 | 35 | ||
36 | static inline fpbuf_t _point(fp_t x) | 36 | static inline fpbuf_t _point(fp_t x) |
37 | { | 37 | { |
38 | return (x.val % (1 << FP_SHIFT)); | 38 | return (x.val % (1 << FP_SHIFT)); |
39 | 39 | ||
40 | } | 40 | } |
41 | 41 | ||
42 | #define fp2str(x) x.val | 42 | #define fp2str(x) x.val |
43 | /*(x.val >> FP_SHIFT), (x.val % (1 << FP_SHIFT)) */ | 43 | /*(x.val >> FP_SHIFT), (x.val % (1 << FP_SHIFT)) */ |
@@ -52,11 +52,11 @@ static inline fpbuf_t _floor(fp_t x) | |||
52 | static inline fpbuf_t _round(fp_t x) | 52 | static inline fpbuf_t _round(fp_t x) |
53 | { | 53 | { |
54 | return _floor(x) + ((x.val >> ROUND_BIT) & 1); | 54 | return _floor(x) + ((x.val >> ROUND_BIT) & 1); |
55 | } | 55 | } |
56 | 56 | ||
57 | /* multiply two fixed point values */ | 57 | /* multiply two fixed point values */ |
58 | static inline fp_t _mul(fp_t a, fp_t b) | 58 | static inline fp_t _mul(fp_t a, fp_t b) |
59 | { | 59 | { |
60 | return _fp((a.val * b.val) >> FP_SHIFT); | 60 | return _fp((a.val * b.val) >> FP_SHIFT); |
61 | } | 61 | } |
62 | 62 | ||
@@ -66,7 +66,7 @@ static inline fp_t _div(fp_t a, fp_t b) | |||
66 | #define unlikely(x) (x) | 66 | #define unlikely(x) (x) |
67 | #define DO_UNDEF_UNLIKELY | 67 | #define DO_UNDEF_UNLIKELY |
68 | #endif | 68 | #endif |
69 | /* try not to overflow */ | 69 | /* try not to overflow */ |
70 | if (unlikely( a.val > (2l << ((sizeof(fpbuf_t)*8) - FP_SHIFT)) )) | 70 | if (unlikely( a.val > (2l << ((sizeof(fpbuf_t)*8) - FP_SHIFT)) )) |
71 | return _fp((a.val / b.val) << FP_SHIFT); | 71 | return _fp((a.val / b.val) << FP_SHIFT); |
72 | else | 72 | else |
diff --git a/include/litmus/gpu_affinity.h b/include/litmus/gpu_affinity.h index ca4d10b93203..d4db2003ad86 100644 --- a/include/litmus/gpu_affinity.h +++ b/include/litmus/gpu_affinity.h | |||
@@ -34,15 +34,15 @@ static inline lt_t get_gpu_estimate(struct task_struct* t, gpu_migration_dist_t | |||
34 | int i; | 34 | int i; |
35 | fpbuf_t temp = _fp_to_integer(t->rt_param.gpu_migration_est[dist].est); | 35 | fpbuf_t temp = _fp_to_integer(t->rt_param.gpu_migration_est[dist].est); |
36 | lt_t val = (temp >= 0) ? temp : 0; // never allow negative estimates... | 36 | lt_t val = (temp >= 0) ? temp : 0; // never allow negative estimates... |
37 | 37 | ||
38 | WARN_ON(temp < 0); | 38 | WARN_ON(temp < 0); |
39 | 39 | ||
40 | // lower-bound a distant migration to be at least equal to the level | 40 | // lower-bound a distant migration to be at least equal to the level |
41 | // below it. | 41 | // below it. |
42 | for(i = dist-1; (val == 0) && (i >= MIG_LOCAL); --i) { | 42 | for(i = dist-1; (val == 0) && (i >= MIG_LOCAL); --i) { |
43 | val = _fp_to_integer(t->rt_param.gpu_migration_est[i].est); | 43 | val = _fp_to_integer(t->rt_param.gpu_migration_est[i].est); |
44 | } | 44 | } |
45 | 45 | ||
46 | // minimum value is 1 (val is 0 if we haven't run with local affinity yet) | 46 | // minimum value is 1 (val is 0 if we haven't run with local affinity yet) |
47 | // TODO: pick a better default min-value. 1 is too small. perhaps | 47 | // TODO: pick a better default min-value. 1 is too small. perhaps |
48 | // task execution time? | 48 | // task execution time? |
diff --git a/include/litmus/ikglp_lock.h b/include/litmus/ikglp_lock.h index 2a75a1719815..08e73332c3d4 100644 --- a/include/litmus/ikglp_lock.h +++ b/include/litmus/ikglp_lock.h | |||
@@ -87,10 +87,10 @@ struct ikglp_semaphore | |||
87 | struct fifo_queue *fifo_queues; // array nr_replicas in length | 87 | struct fifo_queue *fifo_queues; // array nr_replicas in length |
88 | struct binheap_handle priority_queue; // max-heap, base prio | 88 | struct binheap_handle priority_queue; // max-heap, base prio |
89 | struct binheap_handle donors; // max-heap, base prio | 89 | struct binheap_handle donors; // max-heap, base prio |
90 | 90 | ||
91 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING | 91 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING |
92 | struct ikglp_affinity *aff_obs; | 92 | struct ikglp_affinity *aff_obs; |
93 | #endif | 93 | #endif |
94 | }; | 94 | }; |
95 | 95 | ||
96 | static inline struct ikglp_semaphore* ikglp_from_lock(struct litmus_lock* lock) | 96 | static inline struct ikglp_semaphore* ikglp_from_lock(struct litmus_lock* lock) |
@@ -121,7 +121,7 @@ struct ikglp_affinity_ops | |||
121 | struct task_struct* (*advise_steal)(struct ikglp_affinity* aff, wait_queue_t** to_steal, struct fifo_queue** to_steal_from); // select steal from FIFO | 121 | struct task_struct* (*advise_steal)(struct ikglp_affinity* aff, wait_queue_t** to_steal, struct fifo_queue** to_steal_from); // select steal from FIFO |
122 | struct task_struct* (*advise_donee_selection)(struct ikglp_affinity* aff, wait_queue_t** donee, struct fifo_queue** donee_queue); // select a donee | 122 | struct task_struct* (*advise_donee_selection)(struct ikglp_affinity* aff, wait_queue_t** donee, struct fifo_queue** donee_queue); // select a donee |
123 | struct task_struct* (*advise_doner_to_fq)(struct ikglp_affinity* aff, ikglp_wait_state_t** donor); // select a donor to move to PQ | 123 | struct task_struct* (*advise_doner_to_fq)(struct ikglp_affinity* aff, ikglp_wait_state_t** donor); // select a donor to move to PQ |
124 | 124 | ||
125 | void (*notify_enqueue)(struct ikglp_affinity* aff, struct fifo_queue* fq, struct task_struct* t); // fifo enqueue | 125 | void (*notify_enqueue)(struct ikglp_affinity* aff, struct fifo_queue* fq, struct task_struct* t); // fifo enqueue |
126 | void (*notify_dequeue)(struct ikglp_affinity* aff, struct fifo_queue* fq, struct task_struct* t); // fifo dequeue | 126 | void (*notify_dequeue)(struct ikglp_affinity* aff, struct fifo_queue* fq, struct task_struct* t); // fifo dequeue |
127 | void (*notify_acquired)(struct ikglp_affinity* aff, struct fifo_queue* fq, struct task_struct* t); // replica acquired | 127 | void (*notify_acquired)(struct ikglp_affinity* aff, struct fifo_queue* fq, struct task_struct* t); // replica acquired |
@@ -132,7 +132,7 @@ struct ikglp_affinity_ops | |||
132 | struct ikglp_affinity | 132 | struct ikglp_affinity |
133 | { | 133 | { |
134 | struct affinity_observer obs; | 134 | struct affinity_observer obs; |
135 | struct ikglp_affinity_ops *ops; | 135 | struct ikglp_affinity_ops *ops; |
136 | struct fifo_queue *q_info; | 136 | struct fifo_queue *q_info; |
137 | int *nr_cur_users_on_rsrc; | 137 | int *nr_cur_users_on_rsrc; |
138 | int offset; | 138 | int offset; |
diff --git a/include/litmus/kexclu_affinity.h b/include/litmus/kexclu_affinity.h index 51e097f8ec54..f6355de49074 100644 --- a/include/litmus/kexclu_affinity.h +++ b/include/litmus/kexclu_affinity.h | |||
@@ -10,7 +10,7 @@ struct affinity_observer | |||
10 | struct affinity_observer_ops* ops; | 10 | struct affinity_observer_ops* ops; |
11 | int type; | 11 | int type; |
12 | int ident; | 12 | int ident; |
13 | 13 | ||
14 | struct litmus_lock* lock; // the lock under observation | 14 | struct litmus_lock* lock; // the lock under observation |
15 | }; | 15 | }; |
16 | 16 | ||
diff --git a/include/litmus/kfmlp_lock.h b/include/litmus/kfmlp_lock.h index 6d7e24b2a3ad..5f0aae6e6f42 100644 --- a/include/litmus/kfmlp_lock.h +++ b/include/litmus/kfmlp_lock.h | |||
@@ -22,14 +22,14 @@ struct kfmlp_queue | |||
22 | struct kfmlp_semaphore | 22 | struct kfmlp_semaphore |
23 | { | 23 | { |
24 | struct litmus_lock litmus_lock; | 24 | struct litmus_lock litmus_lock; |
25 | 25 | ||
26 | spinlock_t lock; | 26 | spinlock_t lock; |
27 | 27 | ||
28 | int num_resources; /* aka k */ | 28 | int num_resources; /* aka k */ |
29 | 29 | ||
30 | struct kfmlp_queue *queues; /* array */ | 30 | struct kfmlp_queue *queues; /* array */ |
31 | struct kfmlp_queue *shortest_queue; /* pointer to shortest queue */ | 31 | struct kfmlp_queue *shortest_queue; /* pointer to shortest queue */ |
32 | 32 | ||
33 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING | 33 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING |
34 | struct kfmlp_affinity *aff_obs; | 34 | struct kfmlp_affinity *aff_obs; |
35 | #endif | 35 | #endif |
@@ -59,7 +59,7 @@ struct kfmlp_affinity_ops | |||
59 | { | 59 | { |
60 | struct kfmlp_queue* (*advise_enqueue)(struct kfmlp_affinity* aff, struct task_struct* t); | 60 | struct kfmlp_queue* (*advise_enqueue)(struct kfmlp_affinity* aff, struct task_struct* t); |
61 | struct task_struct* (*advise_steal)(struct kfmlp_affinity* aff, wait_queue_t** to_steal, struct kfmlp_queue** to_steal_from); | 61 | struct task_struct* (*advise_steal)(struct kfmlp_affinity* aff, wait_queue_t** to_steal, struct kfmlp_queue** to_steal_from); |
62 | void (*notify_enqueue)(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t); | 62 | void (*notify_enqueue)(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t); |
63 | void (*notify_dequeue)(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t); | 63 | void (*notify_dequeue)(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t); |
64 | void (*notify_acquired)(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t); | 64 | void (*notify_acquired)(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t); |
65 | void (*notify_freed)(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t); | 65 | void (*notify_freed)(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t); |
@@ -69,7 +69,7 @@ struct kfmlp_affinity_ops | |||
69 | struct kfmlp_affinity | 69 | struct kfmlp_affinity |
70 | { | 70 | { |
71 | struct affinity_observer obs; | 71 | struct affinity_observer obs; |
72 | struct kfmlp_affinity_ops *ops; | 72 | struct kfmlp_affinity_ops *ops; |
73 | struct kfmlp_queue_info *q_info; | 73 | struct kfmlp_queue_info *q_info; |
74 | int *nr_cur_users_on_rsrc; | 74 | int *nr_cur_users_on_rsrc; |
75 | int offset; | 75 | int offset; |
diff --git a/include/litmus/litmus_softirq.h b/include/litmus/litmus_softirq.h index 34287f3cbb8d..1eb5ea1a6c4b 100644 --- a/include/litmus/litmus_softirq.h +++ b/include/litmus/litmus_softirq.h | |||
@@ -9,13 +9,13 @@ | |||
9 | are scheduled with the priority of the tasklet's | 9 | are scheduled with the priority of the tasklet's |
10 | owner---that is, the RT task on behalf the tasklet | 10 | owner---that is, the RT task on behalf the tasklet |
11 | runs. | 11 | runs. |
12 | 12 | ||
13 | Tasklets are current scheduled in FIFO order with | 13 | Tasklets are current scheduled in FIFO order with |
14 | NO priority inheritance for "blocked" tasklets. | 14 | NO priority inheritance for "blocked" tasklets. |
15 | 15 | ||
16 | klitirqd assumes the priority of the owner of the | 16 | klitirqd assumes the priority of the owner of the |
17 | tasklet when the tasklet is next to execute. | 17 | tasklet when the tasklet is next to execute. |
18 | 18 | ||
19 | Currently, hi-tasklets are scheduled before | 19 | Currently, hi-tasklets are scheduled before |
20 | low-tasklets, regardless of priority of low-tasklets. | 20 | low-tasklets, regardless of priority of low-tasklets. |
21 | And likewise, low-tasklets are scheduled before work | 21 | And likewise, low-tasklets are scheduled before work |
@@ -35,22 +35,22 @@ | |||
35 | workqueue, so daemons will likely not be immediately | 35 | workqueue, so daemons will likely not be immediately |
36 | running when this function returns, though the required | 36 | running when this function returns, though the required |
37 | data will be initialized. | 37 | data will be initialized. |
38 | 38 | ||
39 | @affinity_set: an array expressing the processor affinity | 39 | @affinity_set: an array expressing the processor affinity |
40 | for each of the NR_LITMUS_SOFTIRQD daemons. May be set | 40 | for each of the NR_LITMUS_SOFTIRQD daemons. May be set |
41 | to NULL for global scheduling. | 41 | to NULL for global scheduling. |
42 | 42 | ||
43 | - Examples - | 43 | - Examples - |
44 | 8-CPU system with two CPU clusters: | 44 | 8-CPU system with two CPU clusters: |
45 | affinity[] = {0, 0, 0, 0, 3, 3, 3, 3} | 45 | affinity[] = {0, 0, 0, 0, 3, 3, 3, 3} |
46 | NOTE: Daemons not actually bound to specified CPU, but rather | 46 | NOTE: Daemons not actually bound to specified CPU, but rather |
47 | cluster in which the CPU resides. | 47 | cluster in which the CPU resides. |
48 | 48 | ||
49 | 8-CPU system, partitioned: | 49 | 8-CPU system, partitioned: |
50 | affinity[] = {0, 1, 2, 3, 4, 5, 6, 7} | 50 | affinity[] = {0, 1, 2, 3, 4, 5, 6, 7} |
51 | 51 | ||
52 | FIXME: change array to a CPU topology or array of cpumasks | 52 | FIXME: change array to a CPU topology or array of cpumasks |
53 | 53 | ||
54 | */ | 54 | */ |
55 | void spawn_klitirqd(int* affinity); | 55 | void spawn_klitirqd(int* affinity); |
56 | 56 | ||
@@ -176,7 +176,7 @@ static inline int litmus_schedule_work( | |||
176 | 176 | ||
177 | 177 | ||
178 | ///////////// mutex operations for client threads. | 178 | ///////////// mutex operations for client threads. |
179 | 179 | ||
180 | void down_and_set_stat(struct task_struct* t, | 180 | void down_and_set_stat(struct task_struct* t, |
181 | enum klitirqd_sem_status to_set, | 181 | enum klitirqd_sem_status to_set, |
182 | struct mutex* sem); | 182 | struct mutex* sem); |
diff --git a/include/litmus/locking.h b/include/litmus/locking.h index c2324c4ccb8a..36647fee03e4 100644 --- a/include/litmus/locking.h +++ b/include/litmus/locking.h | |||
@@ -11,7 +11,7 @@ struct nested_info | |||
11 | struct litmus_lock *lock; | 11 | struct litmus_lock *lock; |
12 | struct task_struct *hp_waiter_eff_prio; | 12 | struct task_struct *hp_waiter_eff_prio; |
13 | struct task_struct **hp_waiter_ptr; | 13 | struct task_struct **hp_waiter_ptr; |
14 | struct binheap_node hp_binheap_node; | 14 | struct binheap_node hp_binheap_node; |
15 | }; | 15 | }; |
16 | 16 | ||
17 | static inline struct task_struct* top_priority(struct binheap_handle* handle) { | 17 | static inline struct task_struct* top_priority(struct binheap_handle* handle) { |
@@ -31,12 +31,12 @@ void print_hp_waiters(struct binheap_node* n, int depth); | |||
31 | struct litmus_lock { | 31 | struct litmus_lock { |
32 | struct litmus_lock_ops *ops; | 32 | struct litmus_lock_ops *ops; |
33 | int type; | 33 | int type; |
34 | 34 | ||
35 | int ident; | 35 | int ident; |
36 | 36 | ||
37 | #ifdef CONFIG_LITMUS_NESTED_LOCKING | 37 | #ifdef CONFIG_LITMUS_NESTED_LOCKING |
38 | struct nested_info nest; | 38 | struct nested_info nest; |
39 | //#ifdef CONFIG_DEBUG_SPINLOCK | 39 | //#ifdef CONFIG_DEBUG_SPINLOCK |
40 | char cheat_lockdep[2]; | 40 | char cheat_lockdep[2]; |
41 | struct lock_class_key key; | 41 | struct lock_class_key key; |
42 | //#endif | 42 | //#endif |
@@ -81,15 +81,15 @@ struct litmus_lock_ops { | |||
81 | /* Current tries to lock/unlock this lock (mandatory methods). */ | 81 | /* Current tries to lock/unlock this lock (mandatory methods). */ |
82 | lock_lock_t lock; | 82 | lock_lock_t lock; |
83 | lock_unlock_t unlock; | 83 | lock_unlock_t unlock; |
84 | 84 | ||
85 | /* The lock is no longer being referenced (mandatory method). */ | 85 | /* The lock is no longer being referenced (mandatory method). */ |
86 | lock_free_t deallocate; | 86 | lock_free_t deallocate; |
87 | 87 | ||
88 | #ifdef CONFIG_LITMUS_NESTED_LOCKING | 88 | #ifdef CONFIG_LITMUS_NESTED_LOCKING |
89 | void (*propagate_increase_inheritance)(struct litmus_lock* l, struct task_struct* t, raw_spinlock_t* to_unlock, unsigned long irqflags); | 89 | void (*propagate_increase_inheritance)(struct litmus_lock* l, struct task_struct* t, raw_spinlock_t* to_unlock, unsigned long irqflags); |
90 | void (*propagate_decrease_inheritance)(struct litmus_lock* l, struct task_struct* t, raw_spinlock_t* to_unlock, unsigned long irqflags); | 90 | void (*propagate_decrease_inheritance)(struct litmus_lock* l, struct task_struct* t, raw_spinlock_t* to_unlock, unsigned long irqflags); |
91 | #endif | 91 | #endif |
92 | 92 | ||
93 | #ifdef CONFIG_LITMUS_DGL_SUPPORT | 93 | #ifdef CONFIG_LITMUS_DGL_SUPPORT |
94 | raw_spinlock_t* (*get_dgl_spin_lock)(struct litmus_lock *l); | 94 | raw_spinlock_t* (*get_dgl_spin_lock)(struct litmus_lock *l); |
95 | int (*dgl_lock)(struct litmus_lock *l, dgl_wait_state_t* dgl_wait, wait_queue_t* wq_node); | 95 | int (*dgl_lock)(struct litmus_lock *l, dgl_wait_state_t* dgl_wait, wait_queue_t* wq_node); |
@@ -105,26 +105,26 @@ struct litmus_lock_ops { | |||
105 | (no cycles!). However, DGLs allow locks to be acquired in any order. This | 105 | (no cycles!). However, DGLs allow locks to be acquired in any order. This |
106 | makes nested inheritance very difficult (we don't yet know a solution) to | 106 | makes nested inheritance very difficult (we don't yet know a solution) to |
107 | realize with fine-grain locks, so we use a big lock instead. | 107 | realize with fine-grain locks, so we use a big lock instead. |
108 | 108 | ||
109 | Code contains both fine-grain and coarse-grain methods together, side-by-side. | 109 | Code contains both fine-grain and coarse-grain methods together, side-by-side. |
110 | Each lock operation *IS NOT* surrounded by ifdef/endif to help make code more | 110 | Each lock operation *IS NOT* surrounded by ifdef/endif to help make code more |
111 | readable. However, this leads to the odd situation where both code paths | 111 | readable. However, this leads to the odd situation where both code paths |
112 | appear together in code as if they were both active together. | 112 | appear together in code as if they were both active together. |
113 | 113 | ||
114 | THIS IS NOT REALLY THE CASE! ONLY ONE CODE PATH IS ACTUALLY ACTIVE! | 114 | THIS IS NOT REALLY THE CASE! ONLY ONE CODE PATH IS ACTUALLY ACTIVE! |
115 | 115 | ||
116 | Example: | 116 | Example: |
117 | lock_global_irqsave(coarseLock, flags); | 117 | lock_global_irqsave(coarseLock, flags); |
118 | lock_fine_irqsave(fineLock, flags); | 118 | lock_fine_irqsave(fineLock, flags); |
119 | 119 | ||
120 | Reality (coarse): | 120 | Reality (coarse): |
121 | lock_global_irqsave(coarseLock, flags); | 121 | lock_global_irqsave(coarseLock, flags); |
122 | //lock_fine_irqsave(fineLock, flags); | 122 | //lock_fine_irqsave(fineLock, flags); |
123 | 123 | ||
124 | Reality (fine): | 124 | Reality (fine): |
125 | //lock_global_irqsave(coarseLock, flags); | 125 | //lock_global_irqsave(coarseLock, flags); |
126 | lock_fine_irqsave(fineLock, flags); | 126 | lock_fine_irqsave(fineLock, flags); |
127 | 127 | ||
128 | Be careful when you read code involving nested inheritance. | 128 | Be careful when you read code involving nested inheritance. |
129 | */ | 129 | */ |
130 | #if defined(CONFIG_LITMUS_DGL_SUPPORT) | 130 | #if defined(CONFIG_LITMUS_DGL_SUPPORT) |
diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h index e832ffcba17c..f2f3bfb6a6db 100644 --- a/include/litmus/rt_param.h +++ b/include/litmus/rt_param.h | |||
@@ -126,14 +126,14 @@ enum klitirqd_sem_status | |||
126 | }; | 126 | }; |
127 | 127 | ||
128 | typedef enum gpu_migration_dist | 128 | typedef enum gpu_migration_dist |
129 | { | 129 | { |
130 | // TODO: Make this variable against NR_NVIDIA_GPUS | 130 | // TODO: Make this variable against NR_NVIDIA_GPUS |
131 | MIG_LOCAL = 0, | 131 | MIG_LOCAL = 0, |
132 | MIG_NEAR = 1, | 132 | MIG_NEAR = 1, |
133 | MIG_MED = 2, | 133 | MIG_MED = 2, |
134 | MIG_FAR = 3, // 8 GPUs in a binary tree hierarchy | 134 | MIG_FAR = 3, // 8 GPUs in a binary tree hierarchy |
135 | MIG_NONE = 4, | 135 | MIG_NONE = 4, |
136 | 136 | ||
137 | MIG_LAST = MIG_NONE | 137 | MIG_LAST = MIG_NONE |
138 | } gpu_migration_dist_t; | 138 | } gpu_migration_dist_t; |
139 | 139 | ||
@@ -159,11 +159,11 @@ struct rt_param { | |||
159 | #ifdef CONFIG_LITMUS_SOFTIRQD | 159 | #ifdef CONFIG_LITMUS_SOFTIRQD |
160 | /* proxy threads have minimum priority by default */ | 160 | /* proxy threads have minimum priority by default */ |
161 | unsigned int is_proxy_thread:1; | 161 | unsigned int is_proxy_thread:1; |
162 | 162 | ||
163 | /* pointer to klitirqd currently working on this | 163 | /* pointer to klitirqd currently working on this |
164 | task_struct's behalf. only set by the task pointed | 164 | task_struct's behalf. only set by the task pointed |
165 | to by klitirqd. | 165 | to by klitirqd. |
166 | 166 | ||
167 | ptr only valid if is_proxy_thread == 0 | 167 | ptr only valid if is_proxy_thread == 0 |
168 | */ | 168 | */ |
169 | struct task_struct* cur_klitirqd; | 169 | struct task_struct* cur_klitirqd; |
@@ -190,14 +190,14 @@ struct rt_param { | |||
190 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING | 190 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING |
191 | fp_t gpu_fb_param_a; | 191 | fp_t gpu_fb_param_a; |
192 | fp_t gpu_fb_param_b; | 192 | fp_t gpu_fb_param_b; |
193 | 193 | ||
194 | gpu_migration_dist_t gpu_migration; | 194 | gpu_migration_dist_t gpu_migration; |
195 | int last_gpu; | 195 | int last_gpu; |
196 | feedback_est_t gpu_migration_est[MIG_LAST+1]; // local, near, med, far | 196 | feedback_est_t gpu_migration_est[MIG_LAST+1]; // local, near, med, far |
197 | 197 | ||
198 | lt_t accum_gpu_time; | 198 | lt_t accum_gpu_time; |
199 | lt_t gpu_time_stamp; | 199 | lt_t gpu_time_stamp; |
200 | 200 | ||
201 | unsigned int suspend_gpu_tracker_on_block:1; | 201 | unsigned int suspend_gpu_tracker_on_block:1; |
202 | #endif | 202 | #endif |
203 | #endif | 203 | #endif |
@@ -222,15 +222,15 @@ struct rt_param { | |||
222 | * an increased task priority. | 222 | * an increased task priority. |
223 | */ | 223 | */ |
224 | struct task_struct* inh_task; | 224 | struct task_struct* inh_task; |
225 | 225 | ||
226 | #ifdef CONFIG_LITMUS_NESTED_LOCKING | 226 | #ifdef CONFIG_LITMUS_NESTED_LOCKING |
227 | raw_spinlock_t hp_blocked_tasks_lock; | 227 | raw_spinlock_t hp_blocked_tasks_lock; |
228 | struct binheap_handle hp_blocked_tasks; | 228 | struct binheap_handle hp_blocked_tasks; |
229 | 229 | ||
230 | /* pointer to lock upon which is currently blocked */ | 230 | /* pointer to lock upon which is currently blocked */ |
231 | struct litmus_lock* blocked_lock; | 231 | struct litmus_lock* blocked_lock; |
232 | #endif | 232 | #endif |
233 | 233 | ||
234 | #ifdef CONFIG_NP_SECTION | 234 | #ifdef CONFIG_NP_SECTION |
235 | /* For the FMLP under PSN-EDF, it is required to make the task | 235 | /* For the FMLP under PSN-EDF, it is required to make the task |
236 | * non-preemptive from kernel space. In order not to interfere with | 236 | * non-preemptive from kernel space. In order not to interfere with |
diff --git a/include/litmus/sched_plugin.h b/include/litmus/sched_plugin.h index d14f705ef414..f0b464207e14 100644 --- a/include/litmus/sched_plugin.h +++ b/include/litmus/sched_plugin.h | |||
@@ -139,7 +139,7 @@ struct sched_plugin { | |||
139 | task_exit_t task_exit; | 139 | task_exit_t task_exit; |
140 | 140 | ||
141 | higher_prio_t compare; | 141 | higher_prio_t compare; |
142 | 142 | ||
143 | #ifdef CONFIG_LITMUS_LOCKING | 143 | #ifdef CONFIG_LITMUS_LOCKING |
144 | /* locking protocols */ | 144 | /* locking protocols */ |
145 | allocate_lock_t allocate_lock; | 145 | allocate_lock_t allocate_lock; |
@@ -158,7 +158,7 @@ struct sched_plugin { | |||
158 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING | 158 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING |
159 | allocate_affinity_observer_t allocate_aff_obs; | 159 | allocate_affinity_observer_t allocate_aff_obs; |
160 | #endif | 160 | #endif |
161 | 161 | ||
162 | #ifdef CONFIG_LITMUS_SOFTIRQD | 162 | #ifdef CONFIG_LITMUS_SOFTIRQD |
163 | increase_prio_klitirq_t increase_prio_klitirqd; | 163 | increase_prio_klitirq_t increase_prio_klitirqd; |
164 | decrease_prio_klitirqd_t decrease_prio_klitirqd; | 164 | decrease_prio_klitirqd_t decrease_prio_klitirqd; |
diff --git a/litmus/affinity.c b/litmus/affinity.c index 3fa6dd789400..cd93249b5506 100644 --- a/litmus/affinity.c +++ b/litmus/affinity.c | |||
@@ -26,7 +26,7 @@ void init_topology(void) { | |||
26 | cpumask_weight((struct cpumask *)&neigh_info[cpu].neighbors[i]); | 26 | cpumask_weight((struct cpumask *)&neigh_info[cpu].neighbors[i]); |
27 | } | 27 | } |
28 | printk("CPU %d has %d neighbors at level %d. (mask = %lx)\n", | 28 | printk("CPU %d has %d neighbors at level %d. (mask = %lx)\n", |
29 | cpu, neigh_info[cpu].size[i], i, | 29 | cpu, neigh_info[cpu].size[i], i, |
30 | *cpumask_bits(neigh_info[cpu].neighbors[i])); | 30 | *cpumask_bits(neigh_info[cpu].neighbors[i])); |
31 | } | 31 | } |
32 | 32 | ||
diff --git a/litmus/binheap.c b/litmus/binheap.c index 22feea614e50..8d42403ad52c 100644 --- a/litmus/binheap.c +++ b/litmus/binheap.c | |||
@@ -9,11 +9,11 @@ int binheap_is_in_this_heap(struct binheap_node *node, | |||
9 | if(!binheap_is_in_heap(node)) { | 9 | if(!binheap_is_in_heap(node)) { |
10 | return 0; | 10 | return 0; |
11 | } | 11 | } |
12 | 12 | ||
13 | while(node->parent != NULL) { | 13 | while(node->parent != NULL) { |
14 | node = node->parent; | 14 | node = node->parent; |
15 | } | 15 | } |
16 | 16 | ||
17 | return (node == heap->root); | 17 | return (node == heap->root); |
18 | } | 18 | } |
19 | 19 | ||
@@ -23,7 +23,7 @@ static void __update_ref(struct binheap_node *parent, | |||
23 | { | 23 | { |
24 | *(parent->ref_ptr) = child; | 24 | *(parent->ref_ptr) = child; |
25 | *(child->ref_ptr) = parent; | 25 | *(child->ref_ptr) = parent; |
26 | 26 | ||
27 | swap(parent->ref_ptr, child->ref_ptr); | 27 | swap(parent->ref_ptr, child->ref_ptr); |
28 | } | 28 | } |
29 | 29 | ||
@@ -35,7 +35,7 @@ static void __binheap_swap(struct binheap_node *parent, | |||
35 | // dump_node_data(parent, child); | 35 | // dump_node_data(parent, child); |
36 | // BUG(); | 36 | // BUG(); |
37 | // } | 37 | // } |
38 | 38 | ||
39 | swap(parent->data, child->data); | 39 | swap(parent->data, child->data); |
40 | __update_ref(parent, child); | 40 | __update_ref(parent, child); |
41 | } | 41 | } |
@@ -50,14 +50,14 @@ static void __binheap_swap_safe(struct binheap_handle *handle, | |||
50 | { | 50 | { |
51 | swap(a->data, b->data); | 51 | swap(a->data, b->data); |
52 | __update_ref(a, b); | 52 | __update_ref(a, b); |
53 | 53 | ||
54 | if((a->parent != NULL) && (a->parent == b->parent)) { | 54 | if((a->parent != NULL) && (a->parent == b->parent)) { |
55 | /* special case: shared parent */ | 55 | /* special case: shared parent */ |
56 | swap(a->parent->left, a->parent->right); | 56 | swap(a->parent->left, a->parent->right); |
57 | } | 57 | } |
58 | else { | 58 | else { |
59 | /* Update pointers to swap parents. */ | 59 | /* Update pointers to swap parents. */ |
60 | 60 | ||
61 | if(a->parent) { | 61 | if(a->parent) { |
62 | if(a == a->parent->left) { | 62 | if(a == a->parent->left) { |
63 | a->parent->left = b; | 63 | a->parent->left = b; |
@@ -66,7 +66,7 @@ static void __binheap_swap_safe(struct binheap_handle *handle, | |||
66 | a->parent->right = b; | 66 | a->parent->right = b; |
67 | } | 67 | } |
68 | } | 68 | } |
69 | 69 | ||
70 | if(b->parent) { | 70 | if(b->parent) { |
71 | if(b == b->parent->left) { | 71 | if(b == b->parent->left) { |
72 | b->parent->left = a; | 72 | b->parent->left = a; |
@@ -75,48 +75,48 @@ static void __binheap_swap_safe(struct binheap_handle *handle, | |||
75 | b->parent->right = a; | 75 | b->parent->right = a; |
76 | } | 76 | } |
77 | } | 77 | } |
78 | 78 | ||
79 | swap(a->parent, b->parent); | 79 | swap(a->parent, b->parent); |
80 | } | 80 | } |
81 | 81 | ||
82 | /* swap children */ | 82 | /* swap children */ |
83 | 83 | ||
84 | if(a->left) { | 84 | if(a->left) { |
85 | a->left->parent = b; | 85 | a->left->parent = b; |
86 | 86 | ||
87 | if(a->right) { | 87 | if(a->right) { |
88 | a->right->parent = b; | 88 | a->right->parent = b; |
89 | } | 89 | } |
90 | } | 90 | } |
91 | 91 | ||
92 | if(b->left) { | 92 | if(b->left) { |
93 | b->left->parent = a; | 93 | b->left->parent = a; |
94 | 94 | ||
95 | if(b->right) { | 95 | if(b->right) { |
96 | b->right->parent = a; | 96 | b->right->parent = a; |
97 | } | 97 | } |
98 | } | 98 | } |
99 | 99 | ||
100 | swap(a->left, b->left); | 100 | swap(a->left, b->left); |
101 | swap(a->right, b->right); | 101 | swap(a->right, b->right); |
102 | 102 | ||
103 | 103 | ||
104 | /* update next/last/root pointers */ | 104 | /* update next/last/root pointers */ |
105 | 105 | ||
106 | if(a == handle->next) { | 106 | if(a == handle->next) { |
107 | handle->next = b; | 107 | handle->next = b; |
108 | } | 108 | } |
109 | else if(b == handle->next) { | 109 | else if(b == handle->next) { |
110 | handle->next = a; | 110 | handle->next = a; |
111 | } | 111 | } |
112 | 112 | ||
113 | if(a == handle->last) { | 113 | if(a == handle->last) { |
114 | handle->last = b; | 114 | handle->last = b; |
115 | } | 115 | } |
116 | else if(b == handle->last) { | 116 | else if(b == handle->last) { |
117 | handle->last = a; | 117 | handle->last = a; |
118 | } | 118 | } |
119 | 119 | ||
120 | if(a == handle->root) { | 120 | if(a == handle->root) { |
121 | handle->root = b; | 121 | handle->root = b; |
122 | } | 122 | } |
@@ -133,29 +133,29 @@ static void __binheap_swap_safe(struct binheap_handle *handle, | |||
133 | static void __binheap_update_last(struct binheap_handle *handle) | 133 | static void __binheap_update_last(struct binheap_handle *handle) |
134 | { | 134 | { |
135 | struct binheap_node *temp = handle->last; | 135 | struct binheap_node *temp = handle->last; |
136 | 136 | ||
137 | /* find a "bend" in the tree. */ | 137 | /* find a "bend" in the tree. */ |
138 | while(temp->parent && (temp == temp->parent->left)) { | 138 | while(temp->parent && (temp == temp->parent->left)) { |
139 | temp = temp->parent; | 139 | temp = temp->parent; |
140 | } | 140 | } |
141 | 141 | ||
142 | /* step over to sibling if we're not at root */ | 142 | /* step over to sibling if we're not at root */ |
143 | if(temp->parent != NULL) { | 143 | if(temp->parent != NULL) { |
144 | temp = temp->parent->left; | 144 | temp = temp->parent->left; |
145 | } | 145 | } |
146 | 146 | ||
147 | /* now travel right as far as possible. */ | 147 | /* now travel right as far as possible. */ |
148 | while(temp->right != NULL) { | 148 | while(temp->right != NULL) { |
149 | temp = temp->right; | 149 | temp = temp->right; |
150 | } | 150 | } |
151 | 151 | ||
152 | /* take one step to the left if we're not at the bottom-most level. */ | 152 | /* take one step to the left if we're not at the bottom-most level. */ |
153 | if(temp->left != NULL) { | 153 | if(temp->left != NULL) { |
154 | temp = temp->left; | 154 | temp = temp->left; |
155 | } | 155 | } |
156 | 156 | ||
157 | //BUG_ON(!(temp->left == NULL && temp->right == NULL)); | 157 | //BUG_ON(!(temp->left == NULL && temp->right == NULL)); |
158 | 158 | ||
159 | handle->last = temp; | 159 | handle->last = temp; |
160 | } | 160 | } |
161 | 161 | ||
@@ -166,22 +166,22 @@ static void __binheap_update_last(struct binheap_handle *handle) | |||
166 | static void __binheap_update_next(struct binheap_handle *handle) | 166 | static void __binheap_update_next(struct binheap_handle *handle) |
167 | { | 167 | { |
168 | struct binheap_node *temp = handle->next; | 168 | struct binheap_node *temp = handle->next; |
169 | 169 | ||
170 | /* find a "bend" in the tree. */ | 170 | /* find a "bend" in the tree. */ |
171 | while(temp->parent && (temp == temp->parent->right)) { | 171 | while(temp->parent && (temp == temp->parent->right)) { |
172 | temp = temp->parent; | 172 | temp = temp->parent; |
173 | } | 173 | } |
174 | 174 | ||
175 | /* step over to sibling if we're not at root */ | 175 | /* step over to sibling if we're not at root */ |
176 | if(temp->parent != NULL) { | 176 | if(temp->parent != NULL) { |
177 | temp = temp->parent->right; | 177 | temp = temp->parent->right; |
178 | } | 178 | } |
179 | 179 | ||
180 | /* now travel left as far as possible. */ | 180 | /* now travel left as far as possible. */ |
181 | while(temp->left != NULL) { | 181 | while(temp->left != NULL) { |
182 | temp = temp->left; | 182 | temp = temp->left; |
183 | } | 183 | } |
184 | 184 | ||
185 | handle->next = temp; | 185 | handle->next = temp; |
186 | } | 186 | } |
187 | 187 | ||
@@ -198,13 +198,13 @@ static void __binheap_bubble_up( | |||
198 | // dump_node_data2(handle, node); | 198 | // dump_node_data2(handle, node); |
199 | // BUG(); | 199 | // BUG(); |
200 | // } | 200 | // } |
201 | 201 | ||
202 | while((node->parent != NULL) && | 202 | while((node->parent != NULL) && |
203 | ((node->data == BINHEAP_POISON) /* let BINHEAP_POISON data bubble to the top */ || | 203 | ((node->data == BINHEAP_POISON) /* let BINHEAP_POISON data bubble to the top */ || |
204 | handle->compare(node, node->parent))) { | 204 | handle->compare(node, node->parent))) { |
205 | __binheap_swap(node->parent, node); | 205 | __binheap_swap(node->parent, node); |
206 | node = node->parent; | 206 | node = node->parent; |
207 | 207 | ||
208 | // if(!binheap_is_in_heap(node)) | 208 | // if(!binheap_is_in_heap(node)) |
209 | // { | 209 | // { |
210 | // dump_node_data2(handle, node); | 210 | // dump_node_data2(handle, node); |
@@ -218,7 +218,7 @@ static void __binheap_bubble_up( | |||
218 | static void __binheap_bubble_down(struct binheap_handle *handle) | 218 | static void __binheap_bubble_down(struct binheap_handle *handle) |
219 | { | 219 | { |
220 | struct binheap_node *node = handle->root; | 220 | struct binheap_node *node = handle->root; |
221 | 221 | ||
222 | while(node->left != NULL) { | 222 | while(node->left != NULL) { |
223 | if(node->right && handle->compare(node->right, node->left)) { | 223 | if(node->right && handle->compare(node->right, node->left)) { |
224 | if(handle->compare(node->right, node)) { | 224 | if(handle->compare(node->right, node)) { |
@@ -252,11 +252,11 @@ void __binheap_add(struct binheap_node *new_node, | |||
252 | // dump_node_data2(handle, new_node); | 252 | // dump_node_data2(handle, new_node); |
253 | // BUG(); | 253 | // BUG(); |
254 | // } | 254 | // } |
255 | 255 | ||
256 | new_node->data = data; | 256 | new_node->data = data; |
257 | new_node->ref = new_node; | 257 | new_node->ref = new_node; |
258 | new_node->ref_ptr = &(new_node->ref); | 258 | new_node->ref_ptr = &(new_node->ref); |
259 | 259 | ||
260 | if(!binheap_empty(handle)) { | 260 | if(!binheap_empty(handle)) { |
261 | /* insert left side first */ | 261 | /* insert left side first */ |
262 | if(handle->next->left == NULL) { | 262 | if(handle->next->left == NULL) { |
@@ -264,9 +264,9 @@ void __binheap_add(struct binheap_node *new_node, | |||
264 | new_node->parent = handle->next; | 264 | new_node->parent = handle->next; |
265 | new_node->left = NULL; | 265 | new_node->left = NULL; |
266 | new_node->right = NULL; | 266 | new_node->right = NULL; |
267 | 267 | ||
268 | handle->last = new_node; | 268 | handle->last = new_node; |
269 | 269 | ||
270 | __binheap_bubble_up(handle, new_node); | 270 | __binheap_bubble_up(handle, new_node); |
271 | } | 271 | } |
272 | else { | 272 | else { |
@@ -275,20 +275,20 @@ void __binheap_add(struct binheap_node *new_node, | |||
275 | new_node->parent = handle->next; | 275 | new_node->parent = handle->next; |
276 | new_node->left = NULL; | 276 | new_node->left = NULL; |
277 | new_node->right = NULL; | 277 | new_node->right = NULL; |
278 | 278 | ||
279 | handle->last = new_node; | 279 | handle->last = new_node; |
280 | 280 | ||
281 | __binheap_update_next(handle); | 281 | __binheap_update_next(handle); |
282 | __binheap_bubble_up(handle, new_node); | 282 | __binheap_bubble_up(handle, new_node); |
283 | } | 283 | } |
284 | } | 284 | } |
285 | else { | 285 | else { |
286 | /* first node in heap */ | 286 | /* first node in heap */ |
287 | 287 | ||
288 | new_node->parent = NULL; | 288 | new_node->parent = NULL; |
289 | new_node->left = NULL; | 289 | new_node->left = NULL; |
290 | new_node->right = NULL; | 290 | new_node->right = NULL; |
291 | 291 | ||
292 | handle->root = new_node; | 292 | handle->root = new_node; |
293 | handle->next = new_node; | 293 | handle->next = new_node; |
294 | handle->last = new_node; | 294 | handle->last = new_node; |
@@ -308,27 +308,27 @@ void __binheap_delete_root(struct binheap_handle *handle, | |||
308 | struct binheap_node *container) | 308 | struct binheap_node *container) |
309 | { | 309 | { |
310 | struct binheap_node *root = handle->root; | 310 | struct binheap_node *root = handle->root; |
311 | 311 | ||
312 | // if(!binheap_is_in_heap(container)) | 312 | // if(!binheap_is_in_heap(container)) |
313 | // { | 313 | // { |
314 | // dump_node_data2(handle, container); | 314 | // dump_node_data2(handle, container); |
315 | // BUG(); | 315 | // BUG(); |
316 | // } | 316 | // } |
317 | 317 | ||
318 | if(root != container) { | 318 | if(root != container) { |
319 | /* coalesce */ | 319 | /* coalesce */ |
320 | __binheap_swap_safe(handle, root, container); | 320 | __binheap_swap_safe(handle, root, container); |
321 | root = container; | 321 | root = container; |
322 | } | 322 | } |
323 | 323 | ||
324 | if(handle->last != root) { | 324 | if(handle->last != root) { |
325 | /* swap 'last' node up to root and bubble it down. */ | 325 | /* swap 'last' node up to root and bubble it down. */ |
326 | 326 | ||
327 | struct binheap_node *to_move = handle->last; | 327 | struct binheap_node *to_move = handle->last; |
328 | 328 | ||
329 | if(to_move->parent != root) { | 329 | if(to_move->parent != root) { |
330 | handle->next = to_move->parent; | 330 | handle->next = to_move->parent; |
331 | 331 | ||
332 | if(handle->next->right == to_move) { | 332 | if(handle->next->right == to_move) { |
333 | /* disconnect from parent */ | 333 | /* disconnect from parent */ |
334 | to_move->parent->right = NULL; | 334 | to_move->parent->right = NULL; |
@@ -337,16 +337,16 @@ void __binheap_delete_root(struct binheap_handle *handle, | |||
337 | else { | 337 | else { |
338 | /* find new 'last' before we disconnect */ | 338 | /* find new 'last' before we disconnect */ |
339 | __binheap_update_last(handle); | 339 | __binheap_update_last(handle); |
340 | 340 | ||
341 | /* disconnect from parent */ | 341 | /* disconnect from parent */ |
342 | to_move->parent->left = NULL; | 342 | to_move->parent->left = NULL; |
343 | } | 343 | } |
344 | } | 344 | } |
345 | else { | 345 | else { |
346 | /* 'last' is direct child of root */ | 346 | /* 'last' is direct child of root */ |
347 | 347 | ||
348 | handle->next = to_move; | 348 | handle->next = to_move; |
349 | 349 | ||
350 | if(to_move == to_move->parent->right) { | 350 | if(to_move == to_move->parent->right) { |
351 | to_move->parent->right = NULL; | 351 | to_move->parent->right = NULL; |
352 | handle->last = to_move->parent->left; | 352 | handle->last = to_move->parent->left; |
@@ -357,7 +357,7 @@ void __binheap_delete_root(struct binheap_handle *handle, | |||
357 | } | 357 | } |
358 | } | 358 | } |
359 | to_move->parent = NULL; | 359 | to_move->parent = NULL; |
360 | 360 | ||
361 | /* reconnect as root. We can't just swap data ptrs since root node | 361 | /* reconnect as root. We can't just swap data ptrs since root node |
362 | * may be freed after this function returns. | 362 | * may be freed after this function returns. |
363 | */ | 363 | */ |
@@ -369,9 +369,9 @@ void __binheap_delete_root(struct binheap_handle *handle, | |||
369 | if(to_move->right != NULL) { | 369 | if(to_move->right != NULL) { |
370 | to_move->right->parent = to_move; | 370 | to_move->right->parent = to_move; |
371 | } | 371 | } |
372 | 372 | ||
373 | handle->root = to_move; | 373 | handle->root = to_move; |
374 | 374 | ||
375 | /* bubble down */ | 375 | /* bubble down */ |
376 | __binheap_bubble_down(handle); | 376 | __binheap_bubble_down(handle); |
377 | } | 377 | } |
@@ -381,7 +381,7 @@ void __binheap_delete_root(struct binheap_handle *handle, | |||
381 | handle->next = NULL; | 381 | handle->next = NULL; |
382 | handle->last = NULL; | 382 | handle->last = NULL; |
383 | } | 383 | } |
384 | 384 | ||
385 | /* mark as removed */ | 385 | /* mark as removed */ |
386 | container->parent = BINHEAP_POISON; | 386 | container->parent = BINHEAP_POISON; |
387 | } | 387 | } |
@@ -396,25 +396,25 @@ void __binheap_delete(struct binheap_node *node_to_delete, | |||
396 | { | 396 | { |
397 | struct binheap_node *target = node_to_delete->ref; | 397 | struct binheap_node *target = node_to_delete->ref; |
398 | void *temp_data = target->data; | 398 | void *temp_data = target->data; |
399 | 399 | ||
400 | // if(!binheap_is_in_heap(node_to_delete)) | 400 | // if(!binheap_is_in_heap(node_to_delete)) |
401 | // { | 401 | // { |
402 | // dump_node_data2(handle, node_to_delete); | 402 | // dump_node_data2(handle, node_to_delete); |
403 | // BUG(); | 403 | // BUG(); |
404 | // } | 404 | // } |
405 | // | 405 | // |
406 | // if(!binheap_is_in_heap(target)) | 406 | // if(!binheap_is_in_heap(target)) |
407 | // { | 407 | // { |
408 | // dump_node_data2(handle, target); | 408 | // dump_node_data2(handle, target); |
409 | // BUG(); | 409 | // BUG(); |
410 | // } | 410 | // } |
411 | 411 | ||
412 | /* temporarily set data to null to allow node to bubble up to the top. */ | 412 | /* temporarily set data to null to allow node to bubble up to the top. */ |
413 | target->data = BINHEAP_POISON; | 413 | target->data = BINHEAP_POISON; |
414 | 414 | ||
415 | __binheap_bubble_up(handle, target); | 415 | __binheap_bubble_up(handle, target); |
416 | __binheap_delete_root(handle, node_to_delete); | 416 | __binheap_delete_root(handle, node_to_delete); |
417 | 417 | ||
418 | node_to_delete->data = temp_data; /* restore node data pointer */ | 418 | node_to_delete->data = temp_data; /* restore node data pointer */ |
419 | //node_to_delete->parent = BINHEAP_POISON; /* poison the node */ | 419 | //node_to_delete->parent = BINHEAP_POISON; /* poison the node */ |
420 | } | 420 | } |
@@ -426,18 +426,18 @@ void __binheap_decrease(struct binheap_node *orig_node, | |||
426 | struct binheap_handle *handle) | 426 | struct binheap_handle *handle) |
427 | { | 427 | { |
428 | struct binheap_node *target = orig_node->ref; | 428 | struct binheap_node *target = orig_node->ref; |
429 | 429 | ||
430 | // if(!binheap_is_in_heap(orig_node)) | 430 | // if(!binheap_is_in_heap(orig_node)) |
431 | // { | 431 | // { |
432 | // dump_node_data2(handle, orig_node); | 432 | // dump_node_data2(handle, orig_node); |
433 | // BUG(); | 433 | // BUG(); |
434 | // } | 434 | // } |
435 | // | 435 | // |
436 | // if(!binheap_is_in_heap(target)) | 436 | // if(!binheap_is_in_heap(target)) |
437 | // { | 437 | // { |
438 | // dump_node_data2(handle, target); | 438 | // dump_node_data2(handle, target); |
439 | // BUG(); | 439 | // BUG(); |
440 | // } | 440 | // } |
441 | // | 441 | // |
442 | __binheap_bubble_up(handle, target); | 442 | __binheap_bubble_up(handle, target); |
443 | } | 443 | } |
diff --git a/litmus/edf_common.c b/litmus/edf_common.c index 250808e934a6..b346bdd65b3b 100644 --- a/litmus/edf_common.c +++ b/litmus/edf_common.c | |||
@@ -92,13 +92,13 @@ int edf_higher_prio(struct task_struct* first, struct task_struct* second) | |||
92 | // if (!is_realtime(second_task)) { | 92 | // if (!is_realtime(second_task)) { |
93 | // return true; | 93 | // return true; |
94 | // } | 94 | // } |
95 | // | 95 | // |
96 | // if (shorter_period(first_task, second_task)) { | 96 | // if (shorter_period(first_task, second_task)) { |
97 | // return true; | 97 | // return true; |
98 | // } | 98 | // } |
99 | // | 99 | // |
100 | // if (get_period(first_task) == get_period(second_task)) { | 100 | // if (get_period(first_task) == get_period(second_task)) { |
101 | // if (first_task->pid < second_task->pid) { | 101 | // if (first_task->pid < second_task->pid) { |
102 | // return true; | 102 | // return true; |
103 | // } | 103 | // } |
104 | // else if (first_task->pid == second_task->pid) { | 104 | // else if (first_task->pid == second_task->pid) { |
@@ -114,12 +114,12 @@ int edf_higher_prio(struct task_struct* first, struct task_struct* second) | |||
114 | return true; | 114 | return true; |
115 | } | 115 | } |
116 | if (get_deadline(first_task) == get_deadline(second_task)) { | 116 | if (get_deadline(first_task) == get_deadline(second_task)) { |
117 | 117 | ||
118 | if (shorter_period(first_task, second_task)) { | 118 | if (shorter_period(first_task, second_task)) { |
119 | return true; | 119 | return true; |
120 | } | 120 | } |
121 | if (get_rt_period(first_task) == get_rt_period(second_task)) { | 121 | if (get_rt_period(first_task) == get_rt_period(second_task)) { |
122 | if (first_task->pid < second_task->pid) { | 122 | if (first_task->pid < second_task->pid) { |
123 | return true; | 123 | return true; |
124 | } | 124 | } |
125 | if (first_task->pid == second_task->pid) { | 125 | if (first_task->pid == second_task->pid) { |
@@ -131,14 +131,14 @@ int edf_higher_prio(struct task_struct* first, struct task_struct* second) | |||
131 | if(first_task->rt_param.is_proxy_thread == second_task->rt_param.is_proxy_thread) { | 131 | if(first_task->rt_param.is_proxy_thread == second_task->rt_param.is_proxy_thread) { |
132 | return !second->rt_param.inh_task; | 132 | return !second->rt_param.inh_task; |
133 | } | 133 | } |
134 | #else | 134 | #else |
135 | return !second->rt_param.inh_task; | 135 | return !second->rt_param.inh_task; |
136 | #endif | 136 | #endif |
137 | } | 137 | } |
138 | 138 | ||
139 | } | 139 | } |
140 | } | 140 | } |
141 | 141 | ||
142 | return false; | 142 | return false; |
143 | } | 143 | } |
144 | 144 | ||
diff --git a/litmus/fdso.c b/litmus/fdso.c index fb328db77dec..0fc74be7f5ee 100644 --- a/litmus/fdso.c +++ b/litmus/fdso.c | |||
@@ -29,7 +29,7 @@ static const struct fdso_ops* fdso_ops[] = { | |||
29 | &generic_lock_ops, /* IKGLP_SEM */ | 29 | &generic_lock_ops, /* IKGLP_SEM */ |
30 | &generic_lock_ops, /* KFMLP_SEM */ | 30 | &generic_lock_ops, /* KFMLP_SEM */ |
31 | &generic_affinity_ops, /* IKGLP_SIMPLE_GPU_AFF_OBS */ | 31 | &generic_affinity_ops, /* IKGLP_SIMPLE_GPU_AFF_OBS */ |
32 | &generic_affinity_ops, /* IKGLP_GPU_AFF_OBS */ | 32 | &generic_affinity_ops, /* IKGLP_GPU_AFF_OBS */ |
33 | &generic_affinity_ops, /* KFMLP_SIMPLE_GPU_AFF_OBS */ | 33 | &generic_affinity_ops, /* KFMLP_SIMPLE_GPU_AFF_OBS */ |
34 | &generic_affinity_ops, /* KFMLP_GPU_AFF_OBS */ | 34 | &generic_affinity_ops, /* KFMLP_GPU_AFF_OBS */ |
35 | }; | 35 | }; |
diff --git a/litmus/gpu_affinity.c b/litmus/gpu_affinity.c index 87349fe10a9b..70a86bdd9aec 100644 --- a/litmus/gpu_affinity.c +++ b/litmus/gpu_affinity.c | |||
@@ -11,7 +11,7 @@ static void update_estimate(feedback_est_t* fb, fp_t a, fp_t b, lt_t observed) | |||
11 | { | 11 | { |
12 | fp_t err, new; | 12 | fp_t err, new; |
13 | fp_t actual = _integer_to_fp(observed); | 13 | fp_t actual = _integer_to_fp(observed); |
14 | 14 | ||
15 | err = _sub(actual, fb->est); | 15 | err = _sub(actual, fb->est); |
16 | new = _add(_mul(a, err), _mul(b, fb->accum_err)); | 16 | new = _add(_mul(a, err), _mul(b, fb->accum_err)); |
17 | 17 | ||
@@ -22,9 +22,9 @@ static void update_estimate(feedback_est_t* fb, fp_t a, fp_t b, lt_t observed) | |||
22 | void update_gpu_estimate(struct task_struct *t, lt_t observed) | 22 | void update_gpu_estimate(struct task_struct *t, lt_t observed) |
23 | { | 23 | { |
24 | feedback_est_t *fb = &(tsk_rt(t)->gpu_migration_est[tsk_rt(t)->gpu_migration]); | 24 | feedback_est_t *fb = &(tsk_rt(t)->gpu_migration_est[tsk_rt(t)->gpu_migration]); |
25 | 25 | ||
26 | WARN_ON(tsk_rt(t)->gpu_migration > MIG_LAST); | 26 | WARN_ON(tsk_rt(t)->gpu_migration > MIG_LAST); |
27 | 27 | ||
28 | if(unlikely(fb->est.val == 0)) { | 28 | if(unlikely(fb->est.val == 0)) { |
29 | // kludge-- cap observed values to prevent whacky estimations. | 29 | // kludge-- cap observed values to prevent whacky estimations. |
30 | // whacky stuff happens during the first few jobs. | 30 | // whacky stuff happens during the first few jobs. |
@@ -32,8 +32,8 @@ void update_gpu_estimate(struct task_struct *t, lt_t observed) | |||
32 | TRACE_TASK(t, "Crazy observation was capped: %llu -> %llu\n", | 32 | TRACE_TASK(t, "Crazy observation was capped: %llu -> %llu\n", |
33 | observed, OBSERVATION_CAP); | 33 | observed, OBSERVATION_CAP); |
34 | observed = OBSERVATION_CAP; | 34 | observed = OBSERVATION_CAP; |
35 | } | 35 | } |
36 | 36 | ||
37 | // take the first observation as our estimate | 37 | // take the first observation as our estimate |
38 | // (initial value of 0 was bogus anyhow) | 38 | // (initial value of 0 was bogus anyhow) |
39 | fb->est = _integer_to_fp(observed); | 39 | fb->est = _integer_to_fp(observed); |
@@ -44,21 +44,21 @@ void update_gpu_estimate(struct task_struct *t, lt_t observed) | |||
44 | tsk_rt(t)->gpu_fb_param_a, | 44 | tsk_rt(t)->gpu_fb_param_a, |
45 | tsk_rt(t)->gpu_fb_param_b, | 45 | tsk_rt(t)->gpu_fb_param_b, |
46 | observed); | 46 | observed); |
47 | 47 | ||
48 | if(_fp_to_integer(fb->est) <= 0) { | 48 | if(_fp_to_integer(fb->est) <= 0) { |
49 | // TODO: talk to Jonathan about how well this works. | 49 | // TODO: talk to Jonathan about how well this works. |
50 | // Maybe we should average the observed and est instead? | 50 | // Maybe we should average the observed and est instead? |
51 | TRACE_TASK(t, "Invalid estimate. Patching.\n"); | 51 | TRACE_TASK(t, "Invalid estimate. Patching.\n"); |
52 | fb->est = _integer_to_fp(observed); | 52 | fb->est = _integer_to_fp(observed); |
53 | fb->accum_err = _div(fb->est, _integer_to_fp(2)); // ...seems to work. | 53 | fb->accum_err = _div(fb->est, _integer_to_fp(2)); // ...seems to work. |
54 | } | 54 | } |
55 | } | 55 | } |
56 | 56 | ||
57 | TRACE_TASK(t, "GPU est update after (dist = %d, obs = %llu): %d.%d\n", | 57 | TRACE_TASK(t, "GPU est update after (dist = %d, obs = %llu): %d.%d\n", |
58 | tsk_rt(t)->gpu_migration, | 58 | tsk_rt(t)->gpu_migration, |
59 | observed, | 59 | observed, |
60 | _fp_to_integer(fb->est), | 60 | _fp_to_integer(fb->est), |
61 | _point(fb->est)); | 61 | _point(fb->est)); |
62 | } | 62 | } |
63 | 63 | ||
64 | gpu_migration_dist_t gpu_migration_distance(int a, int b) | 64 | gpu_migration_dist_t gpu_migration_distance(int a, int b) |
@@ -66,7 +66,7 @@ gpu_migration_dist_t gpu_migration_distance(int a, int b) | |||
66 | // GPUs organized in a binary hierarchy, no more than 2^MIG_FAR GPUs | 66 | // GPUs organized in a binary hierarchy, no more than 2^MIG_FAR GPUs |
67 | int i; | 67 | int i; |
68 | int dist; | 68 | int dist; |
69 | 69 | ||
70 | if(likely(a >= 0 && b >= 0)) { | 70 | if(likely(a >= 0 && b >= 0)) { |
71 | for(i = 0; i <= MIG_FAR; ++i) { | 71 | for(i = 0; i <= MIG_FAR; ++i) { |
72 | if(a>>i == b>>i) { | 72 | if(a>>i == b>>i) { |
@@ -80,11 +80,11 @@ gpu_migration_dist_t gpu_migration_distance(int a, int b) | |||
80 | else { | 80 | else { |
81 | dist = MIG_NONE; | 81 | dist = MIG_NONE; |
82 | } | 82 | } |
83 | 83 | ||
84 | out: | 84 | out: |
85 | TRACE_CUR("Distance %d -> %d is %d\n", | 85 | TRACE_CUR("Distance %d -> %d is %d\n", |
86 | a, b, dist); | 86 | a, b, dist); |
87 | 87 | ||
88 | return dist; | 88 | return dist; |
89 | } | 89 | } |
90 | 90 | ||
diff --git a/litmus/ikglp_lock.c b/litmus/ikglp_lock.c index a41a9d9a3627..94c954464a96 100644 --- a/litmus/ikglp_lock.c +++ b/litmus/ikglp_lock.c | |||
@@ -368,7 +368,7 @@ static void ikglp_refresh_owners_prio_increase(struct task_struct *t, | |||
368 | { | 368 | { |
369 | // priority of 't' has increased (note: 't' might already be hp_waiter). | 369 | // priority of 't' has increased (note: 't' might already be hp_waiter). |
370 | // if ((t == fq->hp_waiter) || edf_higher_prio(t, fq->hp_waiter)) { | 370 | // if ((t == fq->hp_waiter) || edf_higher_prio(t, fq->hp_waiter)) { |
371 | if ((t == fq->hp_waiter) || litmus->compare(t, fq->hp_waiter)) { | 371 | if ((t == fq->hp_waiter) || litmus->compare(t, fq->hp_waiter)) { |
372 | struct task_struct *old_max_eff_prio; | 372 | struct task_struct *old_max_eff_prio; |
373 | struct task_struct *new_max_eff_prio; | 373 | struct task_struct *new_max_eff_prio; |
374 | struct task_struct *new_prio = NULL; | 374 | struct task_struct *new_prio = NULL; |
diff --git a/litmus/kexclu_affinity.c b/litmus/kexclu_affinity.c index 552179bf797d..5ef5e54d600d 100644 --- a/litmus/kexclu_affinity.c +++ b/litmus/kexclu_affinity.c | |||
@@ -35,7 +35,7 @@ static int create_generic_aff_obs(void** obj_ref, obj_type_t type, void* __user | |||
35 | { | 35 | { |
36 | struct affinity_observer* aff_obs; | 36 | struct affinity_observer* aff_obs; |
37 | int err; | 37 | int err; |
38 | 38 | ||
39 | err = litmus->allocate_aff_obs(&aff_obs, type, arg); | 39 | err = litmus->allocate_aff_obs(&aff_obs, type, arg); |
40 | if (err == 0) { | 40 | if (err == 0) { |
41 | BUG_ON(!aff_obs->lock); | 41 | BUG_ON(!aff_obs->lock); |
@@ -73,9 +73,9 @@ static void destroy_generic_aff_obs(obj_type_t type, void* obj) | |||
73 | struct litmus_lock* get_lock_from_od(int od) | 73 | struct litmus_lock* get_lock_from_od(int od) |
74 | { | 74 | { |
75 | extern struct fdso_ops generic_lock_ops; | 75 | extern struct fdso_ops generic_lock_ops; |
76 | 76 | ||
77 | struct od_table_entry *entry = get_entry_for_od(od); | 77 | struct od_table_entry *entry = get_entry_for_od(od); |
78 | 78 | ||
79 | if(entry && entry->class == &generic_lock_ops) { | 79 | if(entry && entry->class == &generic_lock_ops) { |
80 | return (struct litmus_lock*) entry->obj->obj; | 80 | return (struct litmus_lock*) entry->obj->obj; |
81 | } | 81 | } |
diff --git a/litmus/kfmlp_lock.c b/litmus/kfmlp_lock.c index 9bbe31a05b97..d0a6bd364c43 100644 --- a/litmus/kfmlp_lock.c +++ b/litmus/kfmlp_lock.c | |||
@@ -34,11 +34,11 @@ static struct task_struct* kfmlp_find_hp_waiter(struct kfmlp_queue *kqueue, | |||
34 | { | 34 | { |
35 | struct list_head *pos; | 35 | struct list_head *pos; |
36 | struct task_struct *queued, *found = NULL; | 36 | struct task_struct *queued, *found = NULL; |
37 | 37 | ||
38 | list_for_each(pos, &kqueue->wait.task_list) { | 38 | list_for_each(pos, &kqueue->wait.task_list) { |
39 | queued = (struct task_struct*) list_entry(pos, wait_queue_t, | 39 | queued = (struct task_struct*) list_entry(pos, wait_queue_t, |
40 | task_list)->private; | 40 | task_list)->private; |
41 | 41 | ||
42 | /* Compare task prios, find high prio task. */ | 42 | /* Compare task prios, find high prio task. */ |
43 | //if (queued != skip && edf_higher_prio(queued, found)) | 43 | //if (queued != skip && edf_higher_prio(queued, found)) |
44 | if (queued != skip && litmus->compare(queued, found)) | 44 | if (queued != skip && litmus->compare(queued, found)) |
@@ -54,21 +54,21 @@ static inline struct kfmlp_queue* kfmlp_find_shortest(struct kfmlp_semaphore* se | |||
54 | // queue list to load-balance across all resources. | 54 | // queue list to load-balance across all resources. |
55 | struct kfmlp_queue* step = search_start; | 55 | struct kfmlp_queue* step = search_start; |
56 | struct kfmlp_queue* shortest = sem->shortest_queue; | 56 | struct kfmlp_queue* shortest = sem->shortest_queue; |
57 | 57 | ||
58 | do | 58 | do |
59 | { | 59 | { |
60 | step = (step+1 != &sem->queues[sem->num_resources]) ? | 60 | step = (step+1 != &sem->queues[sem->num_resources]) ? |
61 | step+1 : &sem->queues[0]; | 61 | step+1 : &sem->queues[0]; |
62 | 62 | ||
63 | if(step->count < shortest->count) | 63 | if(step->count < shortest->count) |
64 | { | 64 | { |
65 | shortest = step; | 65 | shortest = step; |
66 | if(step->count == 0) | 66 | if(step->count == 0) |
67 | break; /* can't get any shorter */ | 67 | break; /* can't get any shorter */ |
68 | } | 68 | } |
69 | 69 | ||
70 | }while(step != search_start); | 70 | }while(step != search_start); |
71 | 71 | ||
72 | return(shortest); | 72 | return(shortest); |
73 | } | 73 | } |
74 | 74 | ||
@@ -77,13 +77,13 @@ static struct task_struct* kfmlp_select_hp_steal(struct kfmlp_semaphore* sem, | |||
77 | wait_queue_t** to_steal, | 77 | wait_queue_t** to_steal, |
78 | struct kfmlp_queue** to_steal_from) | 78 | struct kfmlp_queue** to_steal_from) |
79 | { | 79 | { |
80 | /* must hold sem->lock */ | 80 | /* must hold sem->lock */ |
81 | 81 | ||
82 | int i; | 82 | int i; |
83 | 83 | ||
84 | *to_steal = NULL; | 84 | *to_steal = NULL; |
85 | *to_steal_from = NULL; | 85 | *to_steal_from = NULL; |
86 | 86 | ||
87 | for(i = 0; i < sem->num_resources; ++i) | 87 | for(i = 0; i < sem->num_resources; ++i) |
88 | { | 88 | { |
89 | if( (sem->queues[i].count > 1) && | 89 | if( (sem->queues[i].count > 1) && |
@@ -94,17 +94,17 @@ static struct task_struct* kfmlp_select_hp_steal(struct kfmlp_semaphore* sem, | |||
94 | *to_steal_from = &sem->queues[i]; | 94 | *to_steal_from = &sem->queues[i]; |
95 | } | 95 | } |
96 | } | 96 | } |
97 | 97 | ||
98 | if(*to_steal_from) | 98 | if(*to_steal_from) |
99 | { | 99 | { |
100 | struct list_head *pos; | 100 | struct list_head *pos; |
101 | struct task_struct *target = (*to_steal_from)->hp_waiter; | 101 | struct task_struct *target = (*to_steal_from)->hp_waiter; |
102 | 102 | ||
103 | TRACE_CUR("want to steal hp_waiter (%s/%d) from queue %d\n", | 103 | TRACE_CUR("want to steal hp_waiter (%s/%d) from queue %d\n", |
104 | target->comm, | 104 | target->comm, |
105 | target->pid, | 105 | target->pid, |
106 | kfmlp_get_idx(sem, *to_steal_from)); | 106 | kfmlp_get_idx(sem, *to_steal_from)); |
107 | 107 | ||
108 | list_for_each(pos, &(*to_steal_from)->wait.task_list) | 108 | list_for_each(pos, &(*to_steal_from)->wait.task_list) |
109 | { | 109 | { |
110 | wait_queue_t *node = list_entry(pos, wait_queue_t, task_list); | 110 | wait_queue_t *node = list_entry(pos, wait_queue_t, task_list); |
@@ -113,21 +113,21 @@ static struct task_struct* kfmlp_select_hp_steal(struct kfmlp_semaphore* sem, | |||
113 | if (queued == target) | 113 | if (queued == target) |
114 | { | 114 | { |
115 | *to_steal = node; | 115 | *to_steal = node; |
116 | 116 | ||
117 | TRACE_CUR("steal: selected %s/%d from queue %d\n", | 117 | TRACE_CUR("steal: selected %s/%d from queue %d\n", |
118 | queued->comm, queued->pid, | 118 | queued->comm, queued->pid, |
119 | kfmlp_get_idx(sem, *to_steal_from)); | 119 | kfmlp_get_idx(sem, *to_steal_from)); |
120 | 120 | ||
121 | return queued; | 121 | return queued; |
122 | } | 122 | } |
123 | } | 123 | } |
124 | 124 | ||
125 | TRACE_CUR("Could not find %s/%d in queue %d!!! THIS IS A BUG!\n", | 125 | TRACE_CUR("Could not find %s/%d in queue %d!!! THIS IS A BUG!\n", |
126 | target->comm, | 126 | target->comm, |
127 | target->pid, | 127 | target->pid, |
128 | kfmlp_get_idx(sem, *to_steal_from)); | 128 | kfmlp_get_idx(sem, *to_steal_from)); |
129 | } | 129 | } |
130 | 130 | ||
131 | return NULL; | 131 | return NULL; |
132 | } | 132 | } |
133 | 133 | ||
@@ -137,13 +137,13 @@ static void kfmlp_steal_node(struct kfmlp_semaphore *sem, | |||
137 | struct kfmlp_queue *src) | 137 | struct kfmlp_queue *src) |
138 | { | 138 | { |
139 | struct task_struct* t = (struct task_struct*) wait->private; | 139 | struct task_struct* t = (struct task_struct*) wait->private; |
140 | 140 | ||
141 | __remove_wait_queue(&src->wait, wait); | 141 | __remove_wait_queue(&src->wait, wait); |
142 | --(src->count); | 142 | --(src->count); |
143 | 143 | ||
144 | if(t == src->hp_waiter) { | 144 | if(t == src->hp_waiter) { |
145 | src->hp_waiter = kfmlp_find_hp_waiter(src, NULL); | 145 | src->hp_waiter = kfmlp_find_hp_waiter(src, NULL); |
146 | 146 | ||
147 | TRACE_CUR("queue %d: %s/%d is new hp_waiter\n", | 147 | TRACE_CUR("queue %d: %s/%d is new hp_waiter\n", |
148 | kfmlp_get_idx(sem, src), | 148 | kfmlp_get_idx(sem, src), |
149 | (src->hp_waiter) ? src->hp_waiter->comm : "nil", | 149 | (src->hp_waiter) ? src->hp_waiter->comm : "nil", |
@@ -153,40 +153,40 @@ static void kfmlp_steal_node(struct kfmlp_semaphore *sem, | |||
153 | litmus->decrease_prio(src->owner, src->hp_waiter); | 153 | litmus->decrease_prio(src->owner, src->hp_waiter); |
154 | } | 154 | } |
155 | } | 155 | } |
156 | 156 | ||
157 | if(sem->shortest_queue->count > src->count) { | 157 | if(sem->shortest_queue->count > src->count) { |
158 | sem->shortest_queue = src; | 158 | sem->shortest_queue = src; |
159 | TRACE_CUR("queue %d is the shortest\n", kfmlp_get_idx(sem, sem->shortest_queue)); | 159 | TRACE_CUR("queue %d is the shortest\n", kfmlp_get_idx(sem, sem->shortest_queue)); |
160 | } | 160 | } |
161 | 161 | ||
162 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING | 162 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING |
163 | if(sem->aff_obs) { | 163 | if(sem->aff_obs) { |
164 | sem->aff_obs->ops->notify_dequeue(sem->aff_obs, src, t); | 164 | sem->aff_obs->ops->notify_dequeue(sem->aff_obs, src, t); |
165 | } | 165 | } |
166 | #endif | 166 | #endif |
167 | 167 | ||
168 | init_waitqueue_entry(wait, t); | 168 | init_waitqueue_entry(wait, t); |
169 | __add_wait_queue_tail_exclusive(&dst->wait, wait); | 169 | __add_wait_queue_tail_exclusive(&dst->wait, wait); |
170 | ++(dst->count); | 170 | ++(dst->count); |
171 | 171 | ||
172 | if(litmus->compare(t, dst->hp_waiter)) { | 172 | if(litmus->compare(t, dst->hp_waiter)) { |
173 | dst->hp_waiter = t; | 173 | dst->hp_waiter = t; |
174 | 174 | ||
175 | TRACE_CUR("queue %d: %s/%d is new hp_waiter\n", | 175 | TRACE_CUR("queue %d: %s/%d is new hp_waiter\n", |
176 | kfmlp_get_idx(sem, dst), | 176 | kfmlp_get_idx(sem, dst), |
177 | t->comm, t->pid); | 177 | t->comm, t->pid); |
178 | 178 | ||
179 | if(dst->owner && litmus->compare(t, dst->owner)) | 179 | if(dst->owner && litmus->compare(t, dst->owner)) |
180 | { | 180 | { |
181 | litmus->increase_prio(dst->owner, t); | 181 | litmus->increase_prio(dst->owner, t); |
182 | } | 182 | } |
183 | } | 183 | } |
184 | 184 | ||
185 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING | 185 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING |
186 | if(sem->aff_obs) { | 186 | if(sem->aff_obs) { |
187 | sem->aff_obs->ops->notify_enqueue(sem->aff_obs, dst, t); | 187 | sem->aff_obs->ops->notify_enqueue(sem->aff_obs, dst, t); |
188 | } | 188 | } |
189 | #endif | 189 | #endif |
190 | } | 190 | } |
191 | 191 | ||
192 | 192 | ||
@@ -197,13 +197,13 @@ int kfmlp_lock(struct litmus_lock* l) | |||
197 | struct kfmlp_queue* my_queue = NULL; | 197 | struct kfmlp_queue* my_queue = NULL; |
198 | wait_queue_t wait; | 198 | wait_queue_t wait; |
199 | unsigned long flags; | 199 | unsigned long flags; |
200 | 200 | ||
201 | if (!is_realtime(t)) | 201 | if (!is_realtime(t)) |
202 | return -EPERM; | 202 | return -EPERM; |
203 | 203 | ||
204 | spin_lock_irqsave(&sem->lock, flags); | 204 | spin_lock_irqsave(&sem->lock, flags); |
205 | 205 | ||
206 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING | 206 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING |
207 | if(sem->aff_obs) { | 207 | if(sem->aff_obs) { |
208 | my_queue = sem->aff_obs->ops->advise_enqueue(sem->aff_obs, t); | 208 | my_queue = sem->aff_obs->ops->advise_enqueue(sem->aff_obs, t); |
209 | } | 209 | } |
@@ -213,25 +213,25 @@ int kfmlp_lock(struct litmus_lock* l) | |||
213 | #else | 213 | #else |
214 | my_queue = sem->shortest_queue; | 214 | my_queue = sem->shortest_queue; |
215 | #endif | 215 | #endif |
216 | 216 | ||
217 | if (my_queue->owner) { | 217 | if (my_queue->owner) { |
218 | /* resource is not free => must suspend and wait */ | 218 | /* resource is not free => must suspend and wait */ |
219 | TRACE_CUR("queue %d: Resource is not free => must suspend and wait. (queue size = %d)\n", | 219 | TRACE_CUR("queue %d: Resource is not free => must suspend and wait. (queue size = %d)\n", |
220 | kfmlp_get_idx(sem, my_queue), | 220 | kfmlp_get_idx(sem, my_queue), |
221 | my_queue->count); | 221 | my_queue->count); |
222 | 222 | ||
223 | init_waitqueue_entry(&wait, t); | 223 | init_waitqueue_entry(&wait, t); |
224 | 224 | ||
225 | /* FIXME: interruptible would be nice some day */ | 225 | /* FIXME: interruptible would be nice some day */ |
226 | set_task_state(t, TASK_UNINTERRUPTIBLE); | 226 | set_task_state(t, TASK_UNINTERRUPTIBLE); |
227 | 227 | ||
228 | __add_wait_queue_tail_exclusive(&my_queue->wait, &wait); | 228 | __add_wait_queue_tail_exclusive(&my_queue->wait, &wait); |
229 | 229 | ||
230 | TRACE_CUR("queue %d: hp_waiter is currently %s/%d\n", | 230 | TRACE_CUR("queue %d: hp_waiter is currently %s/%d\n", |
231 | kfmlp_get_idx(sem, my_queue), | 231 | kfmlp_get_idx(sem, my_queue), |
232 | (my_queue->hp_waiter) ? my_queue->hp_waiter->comm : "nil", | 232 | (my_queue->hp_waiter) ? my_queue->hp_waiter->comm : "nil", |
233 | (my_queue->hp_waiter) ? my_queue->hp_waiter->pid : -1); | 233 | (my_queue->hp_waiter) ? my_queue->hp_waiter->pid : -1); |
234 | 234 | ||
235 | /* check if we need to activate priority inheritance */ | 235 | /* check if we need to activate priority inheritance */ |
236 | //if (edf_higher_prio(t, my_queue->hp_waiter)) | 236 | //if (edf_higher_prio(t, my_queue->hp_waiter)) |
237 | if (litmus->compare(t, my_queue->hp_waiter)) { | 237 | if (litmus->compare(t, my_queue->hp_waiter)) { |
@@ -239,37 +239,37 @@ int kfmlp_lock(struct litmus_lock* l) | |||
239 | TRACE_CUR("queue %d: %s/%d is new hp_waiter\n", | 239 | TRACE_CUR("queue %d: %s/%d is new hp_waiter\n", |
240 | kfmlp_get_idx(sem, my_queue), | 240 | kfmlp_get_idx(sem, my_queue), |
241 | t->comm, t->pid); | 241 | t->comm, t->pid); |
242 | 242 | ||
243 | //if (edf_higher_prio(t, my_queue->owner)) | 243 | //if (edf_higher_prio(t, my_queue->owner)) |
244 | if (litmus->compare(t, my_queue->owner)) { | 244 | if (litmus->compare(t, my_queue->owner)) { |
245 | litmus->increase_prio(my_queue->owner, my_queue->hp_waiter); | 245 | litmus->increase_prio(my_queue->owner, my_queue->hp_waiter); |
246 | } | 246 | } |
247 | } | 247 | } |
248 | 248 | ||
249 | ++(my_queue->count); | 249 | ++(my_queue->count); |
250 | 250 | ||
251 | if(my_queue == sem->shortest_queue) { | 251 | if(my_queue == sem->shortest_queue) { |
252 | sem->shortest_queue = kfmlp_find_shortest(sem, my_queue); | 252 | sem->shortest_queue = kfmlp_find_shortest(sem, my_queue); |
253 | TRACE_CUR("queue %d is the shortest\n", | 253 | TRACE_CUR("queue %d is the shortest\n", |
254 | kfmlp_get_idx(sem, sem->shortest_queue)); | 254 | kfmlp_get_idx(sem, sem->shortest_queue)); |
255 | } | 255 | } |
256 | 256 | ||
257 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING | 257 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING |
258 | if(sem->aff_obs) { | 258 | if(sem->aff_obs) { |
259 | sem->aff_obs->ops->notify_enqueue(sem->aff_obs, my_queue, t); | 259 | sem->aff_obs->ops->notify_enqueue(sem->aff_obs, my_queue, t); |
260 | } | 260 | } |
261 | #endif | 261 | #endif |
262 | 262 | ||
263 | /* release lock before sleeping */ | 263 | /* release lock before sleeping */ |
264 | spin_unlock_irqrestore(&sem->lock, flags); | 264 | spin_unlock_irqrestore(&sem->lock, flags); |
265 | 265 | ||
266 | /* We depend on the FIFO order. Thus, we don't need to recheck | 266 | /* We depend on the FIFO order. Thus, we don't need to recheck |
267 | * when we wake up; we are guaranteed to have the lock since | 267 | * when we wake up; we are guaranteed to have the lock since |
268 | * there is only one wake up per release (or steal). | 268 | * there is only one wake up per release (or steal). |
269 | */ | 269 | */ |
270 | schedule(); | 270 | schedule(); |
271 | 271 | ||
272 | 272 | ||
273 | if(my_queue->owner == t) { | 273 | if(my_queue->owner == t) { |
274 | TRACE_CUR("queue %d: acquired through waiting\n", | 274 | TRACE_CUR("queue %d: acquired through waiting\n", |
275 | kfmlp_get_idx(sem, my_queue)); | 275 | kfmlp_get_idx(sem, my_queue)); |
@@ -278,7 +278,7 @@ int kfmlp_lock(struct litmus_lock* l) | |||
278 | /* this case may happen if our wait entry was stolen | 278 | /* this case may happen if our wait entry was stolen |
279 | between queues. record where we went. */ | 279 | between queues. record where we went. */ |
280 | my_queue = kfmlp_get_queue(sem, t); | 280 | my_queue = kfmlp_get_queue(sem, t); |
281 | 281 | ||
282 | BUG_ON(!my_queue); | 282 | BUG_ON(!my_queue); |
283 | TRACE_CUR("queue %d: acquired through stealing\n", | 283 | TRACE_CUR("queue %d: acquired through stealing\n", |
284 | kfmlp_get_idx(sem, my_queue)); | 284 | kfmlp_get_idx(sem, my_queue)); |
@@ -287,28 +287,28 @@ int kfmlp_lock(struct litmus_lock* l) | |||
287 | else { | 287 | else { |
288 | TRACE_CUR("queue %d: acquired immediately\n", | 288 | TRACE_CUR("queue %d: acquired immediately\n", |
289 | kfmlp_get_idx(sem, my_queue)); | 289 | kfmlp_get_idx(sem, my_queue)); |
290 | 290 | ||
291 | my_queue->owner = t; | 291 | my_queue->owner = t; |
292 | 292 | ||
293 | ++(my_queue->count); | 293 | ++(my_queue->count); |
294 | 294 | ||
295 | if(my_queue == sem->shortest_queue) { | 295 | if(my_queue == sem->shortest_queue) { |
296 | sem->shortest_queue = kfmlp_find_shortest(sem, my_queue); | 296 | sem->shortest_queue = kfmlp_find_shortest(sem, my_queue); |
297 | TRACE_CUR("queue %d is the shortest\n", | 297 | TRACE_CUR("queue %d is the shortest\n", |
298 | kfmlp_get_idx(sem, sem->shortest_queue)); | 298 | kfmlp_get_idx(sem, sem->shortest_queue)); |
299 | } | 299 | } |
300 | 300 | ||
301 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING | 301 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING |
302 | if(sem->aff_obs) { | 302 | if(sem->aff_obs) { |
303 | sem->aff_obs->ops->notify_enqueue(sem->aff_obs, my_queue, t); | 303 | sem->aff_obs->ops->notify_enqueue(sem->aff_obs, my_queue, t); |
304 | sem->aff_obs->ops->notify_acquired(sem->aff_obs, my_queue, t); | 304 | sem->aff_obs->ops->notify_acquired(sem->aff_obs, my_queue, t); |
305 | } | 305 | } |
306 | #endif | 306 | #endif |
307 | 307 | ||
308 | spin_unlock_irqrestore(&sem->lock, flags); | 308 | spin_unlock_irqrestore(&sem->lock, flags); |
309 | } | 309 | } |
310 | 310 | ||
311 | 311 | ||
312 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING | 312 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING |
313 | if(sem->aff_obs) { | 313 | if(sem->aff_obs) { |
314 | return sem->aff_obs->ops->replica_to_resource(sem->aff_obs, my_queue); | 314 | return sem->aff_obs->ops->replica_to_resource(sem->aff_obs, my_queue); |
@@ -325,56 +325,56 @@ int kfmlp_unlock(struct litmus_lock* l) | |||
325 | struct kfmlp_queue *my_queue, *to_steal_from; | 325 | struct kfmlp_queue *my_queue, *to_steal_from; |
326 | unsigned long flags; | 326 | unsigned long flags; |
327 | int err = 0; | 327 | int err = 0; |
328 | 328 | ||
329 | my_queue = kfmlp_get_queue(sem, t); | 329 | my_queue = kfmlp_get_queue(sem, t); |
330 | 330 | ||
331 | if (!my_queue) { | 331 | if (!my_queue) { |
332 | err = -EINVAL; | 332 | err = -EINVAL; |
333 | goto out; | 333 | goto out; |
334 | } | 334 | } |
335 | 335 | ||
336 | spin_lock_irqsave(&sem->lock, flags); | 336 | spin_lock_irqsave(&sem->lock, flags); |
337 | 337 | ||
338 | TRACE_CUR("queue %d: unlocking\n", kfmlp_get_idx(sem, my_queue)); | 338 | TRACE_CUR("queue %d: unlocking\n", kfmlp_get_idx(sem, my_queue)); |
339 | 339 | ||
340 | my_queue->owner = NULL; // clear ownership | 340 | my_queue->owner = NULL; // clear ownership |
341 | --(my_queue->count); | 341 | --(my_queue->count); |
342 | 342 | ||
343 | if(my_queue->count < sem->shortest_queue->count) | 343 | if(my_queue->count < sem->shortest_queue->count) |
344 | { | 344 | { |
345 | sem->shortest_queue = my_queue; | 345 | sem->shortest_queue = my_queue; |
346 | TRACE_CUR("queue %d is the shortest\n", | 346 | TRACE_CUR("queue %d is the shortest\n", |
347 | kfmlp_get_idx(sem, sem->shortest_queue)); | 347 | kfmlp_get_idx(sem, sem->shortest_queue)); |
348 | } | 348 | } |
349 | 349 | ||
350 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING | 350 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING |
351 | if(sem->aff_obs) { | 351 | if(sem->aff_obs) { |
352 | sem->aff_obs->ops->notify_dequeue(sem->aff_obs, my_queue, t); | 352 | sem->aff_obs->ops->notify_dequeue(sem->aff_obs, my_queue, t); |
353 | sem->aff_obs->ops->notify_freed(sem->aff_obs, my_queue, t); | 353 | sem->aff_obs->ops->notify_freed(sem->aff_obs, my_queue, t); |
354 | } | 354 | } |
355 | #endif | 355 | #endif |
356 | 356 | ||
357 | /* we lose the benefit of priority inheritance (if any) */ | 357 | /* we lose the benefit of priority inheritance (if any) */ |
358 | if (tsk_rt(t)->inh_task) | 358 | if (tsk_rt(t)->inh_task) |
359 | litmus->decrease_prio(t, NULL); | 359 | litmus->decrease_prio(t, NULL); |
360 | 360 | ||
361 | 361 | ||
362 | /* check if there are jobs waiting for this resource */ | 362 | /* check if there are jobs waiting for this resource */ |
363 | RETRY: | 363 | RETRY: |
364 | next = __waitqueue_remove_first(&my_queue->wait); | 364 | next = __waitqueue_remove_first(&my_queue->wait); |
365 | if (next) { | 365 | if (next) { |
366 | /* next becomes the resouce holder */ | 366 | /* next becomes the resouce holder */ |
367 | my_queue->owner = next; | 367 | my_queue->owner = next; |
368 | 368 | ||
369 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING | 369 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING |
370 | if(sem->aff_obs) { | 370 | if(sem->aff_obs) { |
371 | sem->aff_obs->ops->notify_acquired(sem->aff_obs, my_queue, next); | 371 | sem->aff_obs->ops->notify_acquired(sem->aff_obs, my_queue, next); |
372 | } | 372 | } |
373 | #endif | 373 | #endif |
374 | 374 | ||
375 | TRACE_CUR("queue %d: lock ownership passed to %s/%d\n", | 375 | TRACE_CUR("queue %d: lock ownership passed to %s/%d\n", |
376 | kfmlp_get_idx(sem, my_queue), next->comm, next->pid); | 376 | kfmlp_get_idx(sem, my_queue), next->comm, next->pid); |
377 | 377 | ||
378 | /* determine new hp_waiter if necessary */ | 378 | /* determine new hp_waiter if necessary */ |
379 | if (next == my_queue->hp_waiter) { | 379 | if (next == my_queue->hp_waiter) { |
380 | TRACE_TASK(next, "was highest-prio waiter\n"); | 380 | TRACE_TASK(next, "was highest-prio waiter\n"); |
@@ -389,7 +389,7 @@ RETRY: | |||
389 | * waiter's priority. */ | 389 | * waiter's priority. */ |
390 | litmus->increase_prio(next, my_queue->hp_waiter); | 390 | litmus->increase_prio(next, my_queue->hp_waiter); |
391 | } | 391 | } |
392 | 392 | ||
393 | /* wake up next */ | 393 | /* wake up next */ |
394 | wake_up_process(next); | 394 | wake_up_process(next); |
395 | } | 395 | } |
@@ -397,7 +397,7 @@ RETRY: | |||
397 | // TODO: put this stealing logic before we attempt to release | 397 | // TODO: put this stealing logic before we attempt to release |
398 | // our resource. (simplifies code and gets rid of ugly goto RETRY. | 398 | // our resource. (simplifies code and gets rid of ugly goto RETRY. |
399 | wait_queue_t *wait; | 399 | wait_queue_t *wait; |
400 | 400 | ||
401 | TRACE_CUR("queue %d: looking to steal someone...\n", | 401 | TRACE_CUR("queue %d: looking to steal someone...\n", |
402 | kfmlp_get_idx(sem, my_queue)); | 402 | kfmlp_get_idx(sem, my_queue)); |
403 | 403 | ||
@@ -408,26 +408,26 @@ RETRY: | |||
408 | #else | 408 | #else |
409 | next = kfmlp_select_hp_steal(sem, &wait, &to_steal_from); | 409 | next = kfmlp_select_hp_steal(sem, &wait, &to_steal_from); |
410 | #endif | 410 | #endif |
411 | 411 | ||
412 | if(next) { | 412 | if(next) { |
413 | TRACE_CUR("queue %d: stealing %s/%d from queue %d\n", | 413 | TRACE_CUR("queue %d: stealing %s/%d from queue %d\n", |
414 | kfmlp_get_idx(sem, my_queue), | 414 | kfmlp_get_idx(sem, my_queue), |
415 | next->comm, next->pid, | 415 | next->comm, next->pid, |
416 | kfmlp_get_idx(sem, to_steal_from)); | 416 | kfmlp_get_idx(sem, to_steal_from)); |
417 | 417 | ||
418 | kfmlp_steal_node(sem, my_queue, wait, to_steal_from); | 418 | kfmlp_steal_node(sem, my_queue, wait, to_steal_from); |
419 | 419 | ||
420 | goto RETRY; // will succeed this time. | 420 | goto RETRY; // will succeed this time. |
421 | } | 421 | } |
422 | else { | 422 | else { |
423 | TRACE_CUR("queue %d: no one to steal.\n", | 423 | TRACE_CUR("queue %d: no one to steal.\n", |
424 | kfmlp_get_idx(sem, my_queue)); | 424 | kfmlp_get_idx(sem, my_queue)); |
425 | } | 425 | } |
426 | } | 426 | } |
427 | 427 | ||
428 | spin_unlock_irqrestore(&sem->lock, flags); | 428 | spin_unlock_irqrestore(&sem->lock, flags); |
429 | 429 | ||
430 | out: | 430 | out: |
431 | return err; | 431 | return err; |
432 | } | 432 | } |
433 | 433 | ||
@@ -437,19 +437,19 @@ int kfmlp_close(struct litmus_lock* l) | |||
437 | struct kfmlp_semaphore *sem = kfmlp_from_lock(l); | 437 | struct kfmlp_semaphore *sem = kfmlp_from_lock(l); |
438 | struct kfmlp_queue *my_queue; | 438 | struct kfmlp_queue *my_queue; |
439 | unsigned long flags; | 439 | unsigned long flags; |
440 | 440 | ||
441 | int owner; | 441 | int owner; |
442 | 442 | ||
443 | spin_lock_irqsave(&sem->lock, flags); | 443 | spin_lock_irqsave(&sem->lock, flags); |
444 | 444 | ||
445 | my_queue = kfmlp_get_queue(sem, t); | 445 | my_queue = kfmlp_get_queue(sem, t); |
446 | owner = (my_queue) ? (my_queue->owner == t) : 0; | 446 | owner = (my_queue) ? (my_queue->owner == t) : 0; |
447 | 447 | ||
448 | spin_unlock_irqrestore(&sem->lock, flags); | 448 | spin_unlock_irqrestore(&sem->lock, flags); |
449 | 449 | ||
450 | if (owner) | 450 | if (owner) |
451 | kfmlp_unlock(l); | 451 | kfmlp_unlock(l); |
452 | 452 | ||
453 | return 0; | 453 | return 0; |
454 | } | 454 | } |
455 | 455 | ||
@@ -467,7 +467,7 @@ struct litmus_lock* kfmlp_new(struct litmus_lock_ops* ops, void* __user args) | |||
467 | struct kfmlp_semaphore* sem; | 467 | struct kfmlp_semaphore* sem; |
468 | int num_resources = 0; | 468 | int num_resources = 0; |
469 | int i; | 469 | int i; |
470 | 470 | ||
471 | if(!access_ok(VERIFY_READ, args, sizeof(num_resources))) | 471 | if(!access_ok(VERIFY_READ, args, sizeof(num_resources))) |
472 | { | 472 | { |
473 | return(NULL); | 473 | return(NULL); |
@@ -478,26 +478,26 @@ struct litmus_lock* kfmlp_new(struct litmus_lock_ops* ops, void* __user args) | |||
478 | } | 478 | } |
479 | if(num_resources < 1) | 479 | if(num_resources < 1) |
480 | { | 480 | { |
481 | return(NULL); | 481 | return(NULL); |
482 | } | 482 | } |
483 | 483 | ||
484 | sem = kmalloc(sizeof(*sem), GFP_KERNEL); | 484 | sem = kmalloc(sizeof(*sem), GFP_KERNEL); |
485 | if(!sem) | 485 | if(!sem) |
486 | { | 486 | { |
487 | return(NULL); | 487 | return(NULL); |
488 | } | 488 | } |
489 | 489 | ||
490 | sem->queues = kmalloc(sizeof(struct kfmlp_queue)*num_resources, GFP_KERNEL); | 490 | sem->queues = kmalloc(sizeof(struct kfmlp_queue)*num_resources, GFP_KERNEL); |
491 | if(!sem->queues) | 491 | if(!sem->queues) |
492 | { | 492 | { |
493 | kfree(sem); | 493 | kfree(sem); |
494 | return(NULL); | 494 | return(NULL); |
495 | } | 495 | } |
496 | 496 | ||
497 | sem->litmus_lock.ops = ops; | 497 | sem->litmus_lock.ops = ops; |
498 | spin_lock_init(&sem->lock); | 498 | spin_lock_init(&sem->lock); |
499 | sem->num_resources = num_resources; | 499 | sem->num_resources = num_resources; |
500 | 500 | ||
501 | for(i = 0; i < num_resources; ++i) | 501 | for(i = 0; i < num_resources; ++i) |
502 | { | 502 | { |
503 | sem->queues[i].owner = NULL; | 503 | sem->queues[i].owner = NULL; |
@@ -505,7 +505,7 @@ struct litmus_lock* kfmlp_new(struct litmus_lock_ops* ops, void* __user args) | |||
505 | init_waitqueue_head(&sem->queues[i].wait); | 505 | init_waitqueue_head(&sem->queues[i].wait); |
506 | sem->queues[i].count = 0; | 506 | sem->queues[i].count = 0; |
507 | } | 507 | } |
508 | 508 | ||
509 | sem->shortest_queue = &sem->queues[0]; | 509 | sem->shortest_queue = &sem->queues[0]; |
510 | 510 | ||
511 | return &sem->litmus_lock; | 511 | return &sem->litmus_lock; |
@@ -517,13 +517,13 @@ struct litmus_lock* kfmlp_new(struct litmus_lock_ops* ops, void* __user args) | |||
517 | #if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA) | 517 | #if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA) |
518 | 518 | ||
519 | static inline int __replica_to_gpu(struct kfmlp_affinity* aff, int replica) | 519 | static inline int __replica_to_gpu(struct kfmlp_affinity* aff, int replica) |
520 | { | 520 | { |
521 | int gpu = replica % aff->nr_rsrc; | 521 | int gpu = replica % aff->nr_rsrc; |
522 | return gpu; | 522 | return gpu; |
523 | } | 523 | } |
524 | 524 | ||
525 | static inline int replica_to_gpu(struct kfmlp_affinity* aff, int replica) | 525 | static inline int replica_to_gpu(struct kfmlp_affinity* aff, int replica) |
526 | { | 526 | { |
527 | int gpu = __replica_to_gpu(aff, replica) + aff->offset; | 527 | int gpu = __replica_to_gpu(aff, replica) + aff->offset; |
528 | return gpu; | 528 | return gpu; |
529 | } | 529 | } |
@@ -557,21 +557,21 @@ static struct affinity_observer* kfmlp_aff_obs_new(struct affinity_observer_ops* | |||
557 | struct kfmlp_semaphore* sem; | 557 | struct kfmlp_semaphore* sem; |
558 | int i; | 558 | int i; |
559 | unsigned long flags; | 559 | unsigned long flags; |
560 | 560 | ||
561 | if(!access_ok(VERIFY_READ, args, sizeof(aff_args))) { | 561 | if(!access_ok(VERIFY_READ, args, sizeof(aff_args))) { |
562 | return(NULL); | 562 | return(NULL); |
563 | } | 563 | } |
564 | if(__copy_from_user(&aff_args, args, sizeof(aff_args))) { | 564 | if(__copy_from_user(&aff_args, args, sizeof(aff_args))) { |
565 | return(NULL); | 565 | return(NULL); |
566 | } | 566 | } |
567 | 567 | ||
568 | sem = (struct kfmlp_semaphore*) get_lock_from_od(aff_args.obs.lock_od); | 568 | sem = (struct kfmlp_semaphore*) get_lock_from_od(aff_args.obs.lock_od); |
569 | 569 | ||
570 | if(sem->litmus_lock.type != KFMLP_SEM) { | 570 | if(sem->litmus_lock.type != KFMLP_SEM) { |
571 | TRACE_CUR("Lock type not supported. Type = %d\n", sem->litmus_lock.type); | 571 | TRACE_CUR("Lock type not supported. Type = %d\n", sem->litmus_lock.type); |
572 | return(NULL); | 572 | return(NULL); |
573 | } | 573 | } |
574 | 574 | ||
575 | if((aff_args.nr_simult_users <= 0) || | 575 | if((aff_args.nr_simult_users <= 0) || |
576 | (sem->num_resources%aff_args.nr_simult_users != 0)) { | 576 | (sem->num_resources%aff_args.nr_simult_users != 0)) { |
577 | TRACE_CUR("Lock %d does not support #replicas (%d) for #simult_users " | 577 | TRACE_CUR("Lock %d does not support #replicas (%d) for #simult_users " |
@@ -582,62 +582,62 @@ static struct affinity_observer* kfmlp_aff_obs_new(struct affinity_observer_ops* | |||
582 | aff_args.nr_simult_users); | 582 | aff_args.nr_simult_users); |
583 | return(NULL); | 583 | return(NULL); |
584 | } | 584 | } |
585 | 585 | ||
586 | if(aff_args.nr_simult_users > NV_MAX_SIMULT_USERS) { | 586 | if(aff_args.nr_simult_users > NV_MAX_SIMULT_USERS) { |
587 | TRACE_CUR("System does not support #simult_users >%d. %d requested.\n", | 587 | TRACE_CUR("System does not support #simult_users >%d. %d requested.\n", |
588 | NV_MAX_SIMULT_USERS, aff_args.nr_simult_users); | 588 | NV_MAX_SIMULT_USERS, aff_args.nr_simult_users); |
589 | return(NULL); | 589 | return(NULL); |
590 | } | 590 | } |
591 | 591 | ||
592 | kfmlp_aff = kmalloc(sizeof(*kfmlp_aff), GFP_KERNEL); | 592 | kfmlp_aff = kmalloc(sizeof(*kfmlp_aff), GFP_KERNEL); |
593 | if(!kfmlp_aff) { | 593 | if(!kfmlp_aff) { |
594 | return(NULL); | 594 | return(NULL); |
595 | } | 595 | } |
596 | 596 | ||
597 | kfmlp_aff->q_info = kmalloc(sizeof(struct kfmlp_queue_info)*sem->num_resources, GFP_KERNEL); | 597 | kfmlp_aff->q_info = kmalloc(sizeof(struct kfmlp_queue_info)*sem->num_resources, GFP_KERNEL); |
598 | if(!kfmlp_aff->q_info) { | 598 | if(!kfmlp_aff->q_info) { |
599 | kfree(kfmlp_aff); | 599 | kfree(kfmlp_aff); |
600 | return(NULL); | 600 | return(NULL); |
601 | } | 601 | } |
602 | 602 | ||
603 | kfmlp_aff->nr_cur_users_on_rsrc = kmalloc(sizeof(int)*(sem->num_resources / aff_args.nr_simult_users), GFP_KERNEL); | 603 | kfmlp_aff->nr_cur_users_on_rsrc = kmalloc(sizeof(int)*(sem->num_resources / aff_args.nr_simult_users), GFP_KERNEL); |
604 | if(!kfmlp_aff->nr_cur_users_on_rsrc) { | 604 | if(!kfmlp_aff->nr_cur_users_on_rsrc) { |
605 | kfree(kfmlp_aff->q_info); | 605 | kfree(kfmlp_aff->q_info); |
606 | kfree(kfmlp_aff); | 606 | kfree(kfmlp_aff); |
607 | return(NULL); | 607 | return(NULL); |
608 | } | 608 | } |
609 | 609 | ||
610 | affinity_observer_new(&kfmlp_aff->obs, ops, &aff_args.obs); | 610 | affinity_observer_new(&kfmlp_aff->obs, ops, &aff_args.obs); |
611 | 611 | ||
612 | kfmlp_aff->ops = kfmlp_ops; | 612 | kfmlp_aff->ops = kfmlp_ops; |
613 | kfmlp_aff->offset = aff_args.replica_to_gpu_offset; | 613 | kfmlp_aff->offset = aff_args.replica_to_gpu_offset; |
614 | kfmlp_aff->nr_simult = aff_args.nr_simult_users; | 614 | kfmlp_aff->nr_simult = aff_args.nr_simult_users; |
615 | kfmlp_aff->nr_rsrc = sem->num_resources / kfmlp_aff->nr_simult; | 615 | kfmlp_aff->nr_rsrc = sem->num_resources / kfmlp_aff->nr_simult; |
616 | 616 | ||
617 | memset(kfmlp_aff->nr_cur_users_on_rsrc, 0, sizeof(int)*(sem->num_resources / kfmlp_aff->nr_rsrc)); | 617 | memset(kfmlp_aff->nr_cur_users_on_rsrc, 0, sizeof(int)*(sem->num_resources / kfmlp_aff->nr_rsrc)); |
618 | 618 | ||
619 | for(i = 0; i < sem->num_resources; ++i) { | 619 | for(i = 0; i < sem->num_resources; ++i) { |
620 | kfmlp_aff->q_info[i].q = &sem->queues[i]; | 620 | kfmlp_aff->q_info[i].q = &sem->queues[i]; |
621 | kfmlp_aff->q_info[i].estimated_len = 0; | 621 | kfmlp_aff->q_info[i].estimated_len = 0; |
622 | 622 | ||
623 | // multiple q_info's will point to the same resource (aka GPU) if | 623 | // multiple q_info's will point to the same resource (aka GPU) if |
624 | // aff_args.nr_simult_users > 1 | 624 | // aff_args.nr_simult_users > 1 |
625 | kfmlp_aff->q_info[i].nr_cur_users = &kfmlp_aff->nr_cur_users_on_rsrc[__replica_to_gpu(kfmlp_aff,i)]; | 625 | kfmlp_aff->q_info[i].nr_cur_users = &kfmlp_aff->nr_cur_users_on_rsrc[__replica_to_gpu(kfmlp_aff,i)]; |
626 | } | 626 | } |
627 | 627 | ||
628 | // attach observer to the lock | 628 | // attach observer to the lock |
629 | spin_lock_irqsave(&sem->lock, flags); | 629 | spin_lock_irqsave(&sem->lock, flags); |
630 | sem->aff_obs = kfmlp_aff; | 630 | sem->aff_obs = kfmlp_aff; |
631 | //kfmlp_aff->shortest_queue = &kfmlp_aff->q_info[kfmlp_get_idx(sem, sem->shortest_queue)]; | 631 | //kfmlp_aff->shortest_queue = &kfmlp_aff->q_info[kfmlp_get_idx(sem, sem->shortest_queue)]; |
632 | spin_unlock_irqrestore(&sem->lock, flags); | 632 | spin_unlock_irqrestore(&sem->lock, flags); |
633 | 633 | ||
634 | return &kfmlp_aff->obs; | 634 | return &kfmlp_aff->obs; |
635 | } | 635 | } |
636 | 636 | ||
637 | 637 | ||
638 | 638 | ||
639 | 639 | ||
640 | static int gpu_replica_to_resource(struct kfmlp_affinity* aff, | 640 | static int gpu_replica_to_resource(struct kfmlp_affinity* aff, |
641 | struct kfmlp_queue* fq) { | 641 | struct kfmlp_queue* fq) { |
642 | struct kfmlp_semaphore *sem = kfmlp_from_lock(aff->obs.lock); | 642 | struct kfmlp_semaphore *sem = kfmlp_from_lock(aff->obs.lock); |
643 | return(replica_to_gpu(aff, kfmlp_get_idx(sem, fq))); | 643 | return(replica_to_gpu(aff, kfmlp_get_idx(sem, fq))); |
@@ -651,13 +651,13 @@ static inline struct kfmlp_queue_info* kfmlp_aff_find_shortest(struct kfmlp_affi | |||
651 | struct kfmlp_semaphore *sem = kfmlp_from_lock(aff->obs.lock); | 651 | struct kfmlp_semaphore *sem = kfmlp_from_lock(aff->obs.lock); |
652 | struct kfmlp_queue_info *shortest = &aff->q_info[0]; | 652 | struct kfmlp_queue_info *shortest = &aff->q_info[0]; |
653 | int i; | 653 | int i; |
654 | 654 | ||
655 | for(i = 1; i < sem->num_resources; ++i) { | 655 | for(i = 1; i < sem->num_resources; ++i) { |
656 | if(aff->q_info[i].estimated_len < shortest->estimated_len) { | 656 | if(aff->q_info[i].estimated_len < shortest->estimated_len) { |
657 | shortest = &aff->q_info[i]; | 657 | shortest = &aff->q_info[i]; |
658 | } | 658 | } |
659 | } | 659 | } |
660 | 660 | ||
661 | return(shortest); | 661 | return(shortest); |
662 | } | 662 | } |
663 | 663 | ||
@@ -670,7 +670,7 @@ struct kfmlp_queue* gpu_kfmlp_advise_enqueue(struct kfmlp_affinity* aff, struct | |||
670 | struct kfmlp_queue *to_enqueue; | 670 | struct kfmlp_queue *to_enqueue; |
671 | int i; | 671 | int i; |
672 | int affinity_gpu; | 672 | int affinity_gpu; |
673 | 673 | ||
674 | // simply pick the shortest queue if, we have no affinity, or we have | 674 | // simply pick the shortest queue if, we have no affinity, or we have |
675 | // affinity with the shortest | 675 | // affinity with the shortest |
676 | if(unlikely(tsk_rt(t)->last_gpu < 0)) { | 676 | if(unlikely(tsk_rt(t)->last_gpu < 0)) { |
@@ -680,20 +680,20 @@ struct kfmlp_queue* gpu_kfmlp_advise_enqueue(struct kfmlp_affinity* aff, struct | |||
680 | else { | 680 | else { |
681 | affinity_gpu = tsk_rt(t)->last_gpu; | 681 | affinity_gpu = tsk_rt(t)->last_gpu; |
682 | } | 682 | } |
683 | 683 | ||
684 | // all things being equal, let's start with the queue with which we have | 684 | // all things being equal, let's start with the queue with which we have |
685 | // affinity. this helps us maintain affinity even when we don't have | 685 | // affinity. this helps us maintain affinity even when we don't have |
686 | // an estiamte for local-affinity execution time (i.e., 2nd time on GPU) | 686 | // an estiamte for local-affinity execution time (i.e., 2nd time on GPU) |
687 | shortest = &aff->q_info[gpu_to_base_replica(aff, affinity_gpu)]; | 687 | shortest = &aff->q_info[gpu_to_base_replica(aff, affinity_gpu)]; |
688 | 688 | ||
689 | // if(shortest == aff->shortest_queue) { | 689 | // if(shortest == aff->shortest_queue) { |
690 | // TRACE_CUR("special case: have affinity with shortest queue\n"); | 690 | // TRACE_CUR("special case: have affinity with shortest queue\n"); |
691 | // goto out; | 691 | // goto out; |
692 | // } | 692 | // } |
693 | 693 | ||
694 | min_len = shortest->estimated_len + get_gpu_estimate(t, MIG_LOCAL); | 694 | min_len = shortest->estimated_len + get_gpu_estimate(t, MIG_LOCAL); |
695 | min_nr_users = *(shortest->nr_cur_users); | 695 | min_nr_users = *(shortest->nr_cur_users); |
696 | 696 | ||
697 | TRACE_CUR("cs is %llu on queue %d: est len = %llu\n", | 697 | TRACE_CUR("cs is %llu on queue %d: est len = %llu\n", |
698 | get_gpu_estimate(t, MIG_LOCAL), | 698 | get_gpu_estimate(t, MIG_LOCAL), |
699 | kfmlp_get_idx(sem, shortest->q), | 699 | kfmlp_get_idx(sem, shortest->q), |
@@ -701,11 +701,11 @@ struct kfmlp_queue* gpu_kfmlp_advise_enqueue(struct kfmlp_affinity* aff, struct | |||
701 | 701 | ||
702 | for(i = 0; i < sem->num_resources; ++i) { | 702 | for(i = 0; i < sem->num_resources; ++i) { |
703 | if(&aff->q_info[i] != shortest) { | 703 | if(&aff->q_info[i] != shortest) { |
704 | 704 | ||
705 | lt_t est_len = | 705 | lt_t est_len = |
706 | aff->q_info[i].estimated_len + | 706 | aff->q_info[i].estimated_len + |
707 | get_gpu_estimate(t, gpu_migration_distance(tsk_rt(t)->last_gpu, replica_to_gpu(aff, i))); | 707 | get_gpu_estimate(t, gpu_migration_distance(tsk_rt(t)->last_gpu, replica_to_gpu(aff, i))); |
708 | 708 | ||
709 | // queue is smaller, or they're equal and the other has a smaller number | 709 | // queue is smaller, or they're equal and the other has a smaller number |
710 | // of total users. | 710 | // of total users. |
711 | // | 711 | // |
@@ -717,29 +717,29 @@ struct kfmlp_queue* gpu_kfmlp_advise_enqueue(struct kfmlp_affinity* aff, struct | |||
717 | min_len = est_len; | 717 | min_len = est_len; |
718 | min_nr_users = *(aff->q_info[i].nr_cur_users); | 718 | min_nr_users = *(aff->q_info[i].nr_cur_users); |
719 | } | 719 | } |
720 | 720 | ||
721 | TRACE_CUR("cs is %llu on queue %d: est len = %llu\n", | 721 | TRACE_CUR("cs is %llu on queue %d: est len = %llu\n", |
722 | get_gpu_estimate(t, gpu_migration_distance(tsk_rt(t)->last_gpu, replica_to_gpu(aff, i))), | 722 | get_gpu_estimate(t, gpu_migration_distance(tsk_rt(t)->last_gpu, replica_to_gpu(aff, i))), |
723 | kfmlp_get_idx(sem, aff->q_info[i].q), | 723 | kfmlp_get_idx(sem, aff->q_info[i].q), |
724 | est_len); | 724 | est_len); |
725 | } | 725 | } |
726 | } | 726 | } |
727 | 727 | ||
728 | to_enqueue = shortest->q; | 728 | to_enqueue = shortest->q; |
729 | TRACE_CUR("enqueue on fq %d (non-aff wanted fq %d)\n", | 729 | TRACE_CUR("enqueue on fq %d (non-aff wanted fq %d)\n", |
730 | kfmlp_get_idx(sem, to_enqueue), | 730 | kfmlp_get_idx(sem, to_enqueue), |
731 | kfmlp_get_idx(sem, sem->shortest_queue)); | 731 | kfmlp_get_idx(sem, sem->shortest_queue)); |
732 | 732 | ||
733 | return to_enqueue; | 733 | return to_enqueue; |
734 | } | 734 | } |
735 | 735 | ||
736 | struct task_struct* gpu_kfmlp_advise_steal(struct kfmlp_affinity* aff, wait_queue_t** to_steal, struct kfmlp_queue** to_steal_from) | 736 | struct task_struct* gpu_kfmlp_advise_steal(struct kfmlp_affinity* aff, wait_queue_t** to_steal, struct kfmlp_queue** to_steal_from) |
737 | { | 737 | { |
738 | struct kfmlp_semaphore *sem = kfmlp_from_lock(aff->obs.lock); | 738 | struct kfmlp_semaphore *sem = kfmlp_from_lock(aff->obs.lock); |
739 | 739 | ||
740 | // For now, just steal highest priority waiter | 740 | // For now, just steal highest priority waiter |
741 | // TODO: Implement affinity-aware stealing. | 741 | // TODO: Implement affinity-aware stealing. |
742 | 742 | ||
743 | return kfmlp_select_hp_steal(sem, to_steal, to_steal_from); | 743 | return kfmlp_select_hp_steal(sem, to_steal, to_steal_from); |
744 | } | 744 | } |
745 | 745 | ||
@@ -752,24 +752,24 @@ void gpu_kfmlp_notify_enqueue(struct kfmlp_affinity* aff, struct kfmlp_queue* fq | |||
752 | struct kfmlp_queue_info *info = &aff->q_info[replica]; | 752 | struct kfmlp_queue_info *info = &aff->q_info[replica]; |
753 | lt_t est_time; | 753 | lt_t est_time; |
754 | lt_t est_len_before; | 754 | lt_t est_len_before; |
755 | 755 | ||
756 | if(current == t) { | 756 | if(current == t) { |
757 | tsk_rt(t)->suspend_gpu_tracker_on_block = 1; | 757 | tsk_rt(t)->suspend_gpu_tracker_on_block = 1; |
758 | } | 758 | } |
759 | 759 | ||
760 | est_len_before = info->estimated_len; | 760 | est_len_before = info->estimated_len; |
761 | est_time = get_gpu_estimate(t, gpu_migration_distance(tsk_rt(t)->last_gpu, gpu)); | 761 | est_time = get_gpu_estimate(t, gpu_migration_distance(tsk_rt(t)->last_gpu, gpu)); |
762 | info->estimated_len += est_time; | 762 | info->estimated_len += est_time; |
763 | 763 | ||
764 | TRACE_CUR("fq %d: q_len (%llu) + est_cs (%llu) = %llu\n", | 764 | TRACE_CUR("fq %d: q_len (%llu) + est_cs (%llu) = %llu\n", |
765 | kfmlp_get_idx(sem, info->q), | 765 | kfmlp_get_idx(sem, info->q), |
766 | est_len_before, est_time, | 766 | est_len_before, est_time, |
767 | info->estimated_len); | 767 | info->estimated_len); |
768 | 768 | ||
769 | // if(aff->shortest_queue == info) { | 769 | // if(aff->shortest_queue == info) { |
770 | // // we may no longer be the shortest | 770 | // // we may no longer be the shortest |
771 | // aff->shortest_queue = kfmlp_aff_find_shortest(aff); | 771 | // aff->shortest_queue = kfmlp_aff_find_shortest(aff); |
772 | // | 772 | // |
773 | // TRACE_CUR("shortest queue is fq %d (with %d in queue) has est len %llu\n", | 773 | // TRACE_CUR("shortest queue is fq %d (with %d in queue) has est len %llu\n", |
774 | // kfmlp_get_idx(sem, aff->shortest_queue->q), | 774 | // kfmlp_get_idx(sem, aff->shortest_queue->q), |
775 | // aff->shortest_queue->q->count, | 775 | // aff->shortest_queue->q->count, |
@@ -784,7 +784,7 @@ void gpu_kfmlp_notify_dequeue(struct kfmlp_affinity* aff, struct kfmlp_queue* fq | |||
784 | int gpu = replica_to_gpu(aff, replica); | 784 | int gpu = replica_to_gpu(aff, replica); |
785 | struct kfmlp_queue_info *info = &aff->q_info[replica]; | 785 | struct kfmlp_queue_info *info = &aff->q_info[replica]; |
786 | lt_t est_time = get_gpu_estimate(t, gpu_migration_distance(tsk_rt(t)->last_gpu, gpu)); | 786 | lt_t est_time = get_gpu_estimate(t, gpu_migration_distance(tsk_rt(t)->last_gpu, gpu)); |
787 | 787 | ||
788 | if(est_time > info->estimated_len) { | 788 | if(est_time > info->estimated_len) { |
789 | WARN_ON(1); | 789 | WARN_ON(1); |
790 | info->estimated_len = 0; | 790 | info->estimated_len = 0; |
@@ -792,21 +792,21 @@ void gpu_kfmlp_notify_dequeue(struct kfmlp_affinity* aff, struct kfmlp_queue* fq | |||
792 | else { | 792 | else { |
793 | info->estimated_len -= est_time; | 793 | info->estimated_len -= est_time; |
794 | } | 794 | } |
795 | 795 | ||
796 | TRACE_CUR("fq %d est len is now %llu\n", | 796 | TRACE_CUR("fq %d est len is now %llu\n", |
797 | kfmlp_get_idx(sem, info->q), | 797 | kfmlp_get_idx(sem, info->q), |
798 | info->estimated_len); | 798 | info->estimated_len); |
799 | 799 | ||
800 | // check to see if we're the shortest queue now. | 800 | // check to see if we're the shortest queue now. |
801 | // if((aff->shortest_queue != info) && | 801 | // if((aff->shortest_queue != info) && |
802 | // (aff->shortest_queue->estimated_len > info->estimated_len)) { | 802 | // (aff->shortest_queue->estimated_len > info->estimated_len)) { |
803 | // | 803 | // |
804 | // aff->shortest_queue = info; | 804 | // aff->shortest_queue = info; |
805 | // | 805 | // |
806 | // TRACE_CUR("shortest queue is fq %d (with %d in queue) has est len %llu\n", | 806 | // TRACE_CUR("shortest queue is fq %d (with %d in queue) has est len %llu\n", |
807 | // kfmlp_get_idx(sem, info->q), | 807 | // kfmlp_get_idx(sem, info->q), |
808 | // info->q->count, | 808 | // info->q->count, |
809 | // info->estimated_len); | 809 | // info->estimated_len); |
810 | // } | 810 | // } |
811 | } | 811 | } |
812 | 812 | ||
@@ -815,17 +815,17 @@ void gpu_kfmlp_notify_acquired(struct kfmlp_affinity* aff, struct kfmlp_queue* f | |||
815 | struct kfmlp_semaphore *sem = kfmlp_from_lock(aff->obs.lock); | 815 | struct kfmlp_semaphore *sem = kfmlp_from_lock(aff->obs.lock); |
816 | int replica = kfmlp_get_idx(sem, fq); | 816 | int replica = kfmlp_get_idx(sem, fq); |
817 | int gpu = replica_to_gpu(aff, replica); | 817 | int gpu = replica_to_gpu(aff, replica); |
818 | 818 | ||
819 | tsk_rt(t)->gpu_migration = gpu_migration_distance(tsk_rt(t)->last_gpu, gpu); // record the type of migration | 819 | tsk_rt(t)->gpu_migration = gpu_migration_distance(tsk_rt(t)->last_gpu, gpu); // record the type of migration |
820 | 820 | ||
821 | TRACE_CUR("%s/%d acquired gpu %d. migration type = %d\n", | 821 | TRACE_CUR("%s/%d acquired gpu %d. migration type = %d\n", |
822 | t->comm, t->pid, gpu, tsk_rt(t)->gpu_migration); | 822 | t->comm, t->pid, gpu, tsk_rt(t)->gpu_migration); |
823 | 823 | ||
824 | // count the number or resource holders | 824 | // count the number or resource holders |
825 | ++(*(aff->q_info[replica].nr_cur_users)); | 825 | ++(*(aff->q_info[replica].nr_cur_users)); |
826 | 826 | ||
827 | reg_nv_device(gpu, 1, t); // register | 827 | reg_nv_device(gpu, 1, t); // register |
828 | 828 | ||
829 | tsk_rt(t)->suspend_gpu_tracker_on_block = 0; | 829 | tsk_rt(t)->suspend_gpu_tracker_on_block = 0; |
830 | reset_gpu_tracker(t); | 830 | reset_gpu_tracker(t); |
831 | start_gpu_tracker(t); | 831 | start_gpu_tracker(t); |
@@ -837,21 +837,21 @@ void gpu_kfmlp_notify_freed(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, | |||
837 | int replica = kfmlp_get_idx(sem, fq); | 837 | int replica = kfmlp_get_idx(sem, fq); |
838 | int gpu = replica_to_gpu(aff, replica); | 838 | int gpu = replica_to_gpu(aff, replica); |
839 | lt_t est_time; | 839 | lt_t est_time; |
840 | 840 | ||
841 | stop_gpu_tracker(t); // stop the tracker before we do anything else. | 841 | stop_gpu_tracker(t); // stop the tracker before we do anything else. |
842 | 842 | ||
843 | est_time = get_gpu_estimate(t, gpu_migration_distance(tsk_rt(t)->last_gpu, gpu)); | 843 | est_time = get_gpu_estimate(t, gpu_migration_distance(tsk_rt(t)->last_gpu, gpu)); |
844 | 844 | ||
845 | tsk_rt(t)->last_gpu = gpu; | 845 | tsk_rt(t)->last_gpu = gpu; |
846 | 846 | ||
847 | // count the number or resource holders | 847 | // count the number or resource holders |
848 | --(*(aff->q_info[replica].nr_cur_users)); | 848 | --(*(aff->q_info[replica].nr_cur_users)); |
849 | 849 | ||
850 | reg_nv_device(gpu, 0, t); // unregister | 850 | reg_nv_device(gpu, 0, t); // unregister |
851 | 851 | ||
852 | // update estimates | 852 | // update estimates |
853 | update_gpu_estimate(t, get_gpu_time(t)); | 853 | update_gpu_estimate(t, get_gpu_time(t)); |
854 | 854 | ||
855 | TRACE_CUR("%s/%d freed gpu %d. actual time was %llu. estimated was %llu. diff is %d\n", | 855 | TRACE_CUR("%s/%d freed gpu %d. actual time was %llu. estimated was %llu. diff is %d\n", |
856 | t->comm, t->pid, gpu, | 856 | t->comm, t->pid, gpu, |
857 | get_gpu_time(t), | 857 | get_gpu_time(t), |
@@ -893,9 +893,9 @@ struct kfmlp_queue* simple_gpu_kfmlp_advise_enqueue(struct kfmlp_affinity* aff, | |||
893 | struct kfmlp_queue_info *shortest; | 893 | struct kfmlp_queue_info *shortest; |
894 | struct kfmlp_queue *to_enqueue; | 894 | struct kfmlp_queue *to_enqueue; |
895 | int i; | 895 | int i; |
896 | 896 | ||
897 | // TRACE_CUR("Simple GPU KFMLP advise_enqueue invoked\n"); | 897 | // TRACE_CUR("Simple GPU KFMLP advise_enqueue invoked\n"); |
898 | 898 | ||
899 | shortest = &aff->q_info[0]; | 899 | shortest = &aff->q_info[0]; |
900 | min_count = shortest->q->count; | 900 | min_count = shortest->q->count; |
901 | min_nr_users = *(shortest->nr_cur_users); | 901 | min_nr_users = *(shortest->nr_cur_users); |
@@ -904,10 +904,10 @@ struct kfmlp_queue* simple_gpu_kfmlp_advise_enqueue(struct kfmlp_affinity* aff, | |||
904 | kfmlp_get_idx(sem, shortest->q), | 904 | kfmlp_get_idx(sem, shortest->q), |
905 | shortest->q->count, | 905 | shortest->q->count, |
906 | min_nr_users); | 906 | min_nr_users); |
907 | 907 | ||
908 | for(i = 1; i < sem->num_resources; ++i) { | 908 | for(i = 1; i < sem->num_resources; ++i) { |
909 | int len = aff->q_info[i].q->count; | 909 | int len = aff->q_info[i].q->count; |
910 | 910 | ||
911 | // queue is smaller, or they're equal and the other has a smaller number | 911 | // queue is smaller, or they're equal and the other has a smaller number |
912 | // of total users. | 912 | // of total users. |
913 | // | 913 | // |
@@ -919,18 +919,18 @@ struct kfmlp_queue* simple_gpu_kfmlp_advise_enqueue(struct kfmlp_affinity* aff, | |||
919 | min_count = shortest->q->count; | 919 | min_count = shortest->q->count; |
920 | min_nr_users = *(aff->q_info[i].nr_cur_users); | 920 | min_nr_users = *(aff->q_info[i].nr_cur_users); |
921 | } | 921 | } |
922 | 922 | ||
923 | TRACE_CUR("queue %d: waiters = %d, total holders = %d\n", | 923 | TRACE_CUR("queue %d: waiters = %d, total holders = %d\n", |
924 | kfmlp_get_idx(sem, aff->q_info[i].q), | 924 | kfmlp_get_idx(sem, aff->q_info[i].q), |
925 | aff->q_info[i].q->count, | 925 | aff->q_info[i].q->count, |
926 | *(aff->q_info[i].nr_cur_users)); | 926 | *(aff->q_info[i].nr_cur_users)); |
927 | } | 927 | } |
928 | 928 | ||
929 | to_enqueue = shortest->q; | 929 | to_enqueue = shortest->q; |
930 | TRACE_CUR("enqueue on fq %d (non-aff wanted fq %d)\n", | 930 | TRACE_CUR("enqueue on fq %d (non-aff wanted fq %d)\n", |
931 | kfmlp_get_idx(sem, to_enqueue), | 931 | kfmlp_get_idx(sem, to_enqueue), |
932 | kfmlp_get_idx(sem, sem->shortest_queue)); | 932 | kfmlp_get_idx(sem, sem->shortest_queue)); |
933 | 933 | ||
934 | return to_enqueue; | 934 | return to_enqueue; |
935 | } | 935 | } |
936 | 936 | ||
@@ -956,12 +956,12 @@ void simple_gpu_kfmlp_notify_acquired(struct kfmlp_affinity* aff, struct kfmlp_q | |||
956 | struct kfmlp_semaphore *sem = kfmlp_from_lock(aff->obs.lock); | 956 | struct kfmlp_semaphore *sem = kfmlp_from_lock(aff->obs.lock); |
957 | int replica = kfmlp_get_idx(sem, fq); | 957 | int replica = kfmlp_get_idx(sem, fq); |
958 | int gpu = replica_to_gpu(aff, replica); | 958 | int gpu = replica_to_gpu(aff, replica); |
959 | 959 | ||
960 | // TRACE_CUR("Simple GPU KFMLP notify_acquired invoked\n"); | 960 | // TRACE_CUR("Simple GPU KFMLP notify_acquired invoked\n"); |
961 | 961 | ||
962 | // count the number or resource holders | 962 | // count the number or resource holders |
963 | ++(*(aff->q_info[replica].nr_cur_users)); | 963 | ++(*(aff->q_info[replica].nr_cur_users)); |
964 | 964 | ||
965 | reg_nv_device(gpu, 1, t); // register | 965 | reg_nv_device(gpu, 1, t); // register |
966 | } | 966 | } |
967 | 967 | ||
@@ -970,11 +970,11 @@ void simple_gpu_kfmlp_notify_freed(struct kfmlp_affinity* aff, struct kfmlp_queu | |||
970 | struct kfmlp_semaphore *sem = kfmlp_from_lock(aff->obs.lock); | 970 | struct kfmlp_semaphore *sem = kfmlp_from_lock(aff->obs.lock); |
971 | int replica = kfmlp_get_idx(sem, fq); | 971 | int replica = kfmlp_get_idx(sem, fq); |
972 | int gpu = replica_to_gpu(aff, replica); | 972 | int gpu = replica_to_gpu(aff, replica); |
973 | 973 | ||
974 | // TRACE_CUR("Simple GPU KFMLP notify_freed invoked\n"); | 974 | // TRACE_CUR("Simple GPU KFMLP notify_freed invoked\n"); |
975 | // count the number or resource holders | 975 | // count the number or resource holders |
976 | --(*(aff->q_info[replica].nr_cur_users)); | 976 | --(*(aff->q_info[replica].nr_cur_users)); |
977 | 977 | ||
978 | reg_nv_device(gpu, 0, t); // unregister | 978 | reg_nv_device(gpu, 0, t); // unregister |
979 | } | 979 | } |
980 | 980 | ||
@@ -986,7 +986,7 @@ struct kfmlp_affinity_ops simple_gpu_kfmlp_affinity = | |||
986 | .notify_dequeue = simple_gpu_kfmlp_notify_dequeue, | 986 | .notify_dequeue = simple_gpu_kfmlp_notify_dequeue, |
987 | .notify_acquired = simple_gpu_kfmlp_notify_acquired, | 987 | .notify_acquired = simple_gpu_kfmlp_notify_acquired, |
988 | .notify_freed = simple_gpu_kfmlp_notify_freed, | 988 | .notify_freed = simple_gpu_kfmlp_notify_freed, |
989 | .replica_to_resource = gpu_replica_to_resource, | 989 | .replica_to_resource = gpu_replica_to_resource, |
990 | }; | 990 | }; |
991 | 991 | ||
992 | struct affinity_observer* kfmlp_simple_gpu_aff_obs_new(struct affinity_observer_ops* ops, | 992 | struct affinity_observer* kfmlp_simple_gpu_aff_obs_new(struct affinity_observer_ops* ops, |
diff --git a/litmus/litmus.c b/litmus/litmus.c index b876e67b7a9b..5b301c418b96 100644 --- a/litmus/litmus.c +++ b/litmus/litmus.c | |||
@@ -309,7 +309,7 @@ asmlinkage long sys_null_call(cycles_t __user *ts) | |||
309 | now = get_cycles(); | 309 | now = get_cycles(); |
310 | ret = put_user(now, ts); | 310 | ret = put_user(now, ts); |
311 | } | 311 | } |
312 | 312 | ||
313 | return ret; | 313 | return ret; |
314 | } | 314 | } |
315 | 315 | ||
@@ -323,9 +323,9 @@ void init_gpu_affinity_state(struct task_struct* p) | |||
323 | // critically-damped | 323 | // critically-damped |
324 | // p->rt_param.gpu_fb_param_a = _frac(102, 1000); | 324 | // p->rt_param.gpu_fb_param_a = _frac(102, 1000); |
325 | // p->rt_param.gpu_fb_param_b = _frac(303, 1000); | 325 | // p->rt_param.gpu_fb_param_b = _frac(303, 1000); |
326 | 326 | ||
327 | p->rt_param.gpu_migration = MIG_NONE; | 327 | p->rt_param.gpu_migration = MIG_NONE; |
328 | p->rt_param.last_gpu = -1; | 328 | p->rt_param.last_gpu = -1; |
329 | } | 329 | } |
330 | #endif | 330 | #endif |
331 | 331 | ||
@@ -334,11 +334,11 @@ static void reinit_litmus_state(struct task_struct* p, int restore) | |||
334 | { | 334 | { |
335 | struct rt_task user_config = {}; | 335 | struct rt_task user_config = {}; |
336 | void* ctrl_page = NULL; | 336 | void* ctrl_page = NULL; |
337 | 337 | ||
338 | #ifdef CONFIG_LITMUS_NESTED_LOCKING | 338 | #ifdef CONFIG_LITMUS_NESTED_LOCKING |
339 | binheap_order_t prio_order = NULL; | 339 | binheap_order_t prio_order = NULL; |
340 | #endif | 340 | #endif |
341 | 341 | ||
342 | if (restore) { | 342 | if (restore) { |
343 | /* Safe user-space provided configuration data. | 343 | /* Safe user-space provided configuration data. |
344 | * and allocated page. */ | 344 | * and allocated page. */ |
@@ -346,15 +346,15 @@ static void reinit_litmus_state(struct task_struct* p, int restore) | |||
346 | ctrl_page = p->rt_param.ctrl_page; | 346 | ctrl_page = p->rt_param.ctrl_page; |
347 | } | 347 | } |
348 | 348 | ||
349 | #ifdef CONFIG_LITMUS_NESTED_LOCKING | 349 | #ifdef CONFIG_LITMUS_NESTED_LOCKING |
350 | prio_order = p->rt_param.hp_blocked_tasks.compare; | 350 | prio_order = p->rt_param.hp_blocked_tasks.compare; |
351 | #endif | 351 | #endif |
352 | 352 | ||
353 | /* We probably should not be inheriting any task's priority | 353 | /* We probably should not be inheriting any task's priority |
354 | * at this point in time. | 354 | * at this point in time. |
355 | */ | 355 | */ |
356 | WARN_ON(p->rt_param.inh_task); | 356 | WARN_ON(p->rt_param.inh_task); |
357 | 357 | ||
358 | #ifdef CONFIG_LITMUS_NESTED_LOCKING | 358 | #ifdef CONFIG_LITMUS_NESTED_LOCKING |
359 | WARN_ON(p->rt_param.blocked_lock); | 359 | WARN_ON(p->rt_param.blocked_lock); |
360 | WARN_ON(!binheap_empty(&p->rt_param.hp_blocked_tasks)); | 360 | WARN_ON(!binheap_empty(&p->rt_param.hp_blocked_tasks)); |
@@ -363,7 +363,7 @@ static void reinit_litmus_state(struct task_struct* p, int restore) | |||
363 | #ifdef CONFIG_LITMUS_SOFTIRQD | 363 | #ifdef CONFIG_LITMUS_SOFTIRQD |
364 | /* We probably should not have any tasklets executing for | 364 | /* We probably should not have any tasklets executing for |
365 | * us at this time. | 365 | * us at this time. |
366 | */ | 366 | */ |
367 | WARN_ON(p->rt_param.cur_klitirqd); | 367 | WARN_ON(p->rt_param.cur_klitirqd); |
368 | WARN_ON(atomic_read(&p->rt_param.klitirqd_sem_stat) == HELD); | 368 | WARN_ON(atomic_read(&p->rt_param.klitirqd_sem_stat) == HELD); |
369 | 369 | ||
@@ -377,24 +377,24 @@ static void reinit_litmus_state(struct task_struct* p, int restore) | |||
377 | #ifdef CONFIG_LITMUS_NVIDIA | 377 | #ifdef CONFIG_LITMUS_NVIDIA |
378 | WARN_ON(p->rt_param.held_gpus != 0); | 378 | WARN_ON(p->rt_param.held_gpus != 0); |
379 | #endif | 379 | #endif |
380 | 380 | ||
381 | /* Cleanup everything else. */ | 381 | /* Cleanup everything else. */ |
382 | memset(&p->rt_param, 0, sizeof(p->rt_param)); | 382 | memset(&p->rt_param, 0, sizeof(p->rt_param)); |
383 | 383 | ||
384 | /* Restore preserved fields. */ | 384 | /* Restore preserved fields. */ |
385 | if (restore) { | 385 | if (restore) { |
386 | p->rt_param.task_params = user_config; | 386 | p->rt_param.task_params = user_config; |
387 | p->rt_param.ctrl_page = ctrl_page; | 387 | p->rt_param.ctrl_page = ctrl_page; |
388 | } | 388 | } |
389 | 389 | ||
390 | #if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_AFFINITY_LOCKING) | 390 | #if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_AFFINITY_LOCKING) |
391 | init_gpu_affinity_state(p); | 391 | init_gpu_affinity_state(p); |
392 | #endif | 392 | #endif |
393 | 393 | ||
394 | #ifdef CONFIG_LITMUS_NESTED_LOCKING | 394 | #ifdef CONFIG_LITMUS_NESTED_LOCKING |
395 | INIT_BINHEAP_HANDLE(&p->rt_param.hp_blocked_tasks, prio_order); | 395 | INIT_BINHEAP_HANDLE(&p->rt_param.hp_blocked_tasks, prio_order); |
396 | raw_spin_lock_init(&p->rt_param.hp_blocked_tasks_lock); | 396 | raw_spin_lock_init(&p->rt_param.hp_blocked_tasks_lock); |
397 | #endif | 397 | #endif |
398 | } | 398 | } |
399 | 399 | ||
400 | long litmus_admit_task(struct task_struct* tsk) | 400 | long litmus_admit_task(struct task_struct* tsk) |
@@ -440,8 +440,8 @@ long litmus_admit_task(struct task_struct* tsk) | |||
440 | } else { | 440 | } else { |
441 | bheap_node_init(&tsk_rt(tsk)->heap_node, tsk); | 441 | bheap_node_init(&tsk_rt(tsk)->heap_node, tsk); |
442 | } | 442 | } |
443 | 443 | ||
444 | 444 | ||
445 | #ifdef CONFIG_LITMUS_NVIDIA | 445 | #ifdef CONFIG_LITMUS_NVIDIA |
446 | atomic_set(&tsk_rt(tsk)->nv_int_count, 0); | 446 | atomic_set(&tsk_rt(tsk)->nv_int_count, 0); |
447 | #endif | 447 | #endif |
@@ -460,7 +460,7 @@ long litmus_admit_task(struct task_struct* tsk) | |||
460 | mutex_init(&tsk_rt(tsk)->klitirqd_sem); | 460 | mutex_init(&tsk_rt(tsk)->klitirqd_sem); |
461 | atomic_set(&tsk_rt(tsk)->klitirqd_sem_stat, NOT_HELD); | 461 | atomic_set(&tsk_rt(tsk)->klitirqd_sem_stat, NOT_HELD); |
462 | #endif | 462 | #endif |
463 | 463 | ||
464 | retval = litmus->admit_task(tsk); | 464 | retval = litmus->admit_task(tsk); |
465 | 465 | ||
466 | if (!retval) { | 466 | if (!retval) { |
diff --git a/litmus/litmus_pai_softirq.c b/litmus/litmus_pai_softirq.c index b31eeb8a2538..300571a81bbd 100644 --- a/litmus/litmus_pai_softirq.c +++ b/litmus/litmus_pai_softirq.c | |||
@@ -28,7 +28,7 @@ int __litmus_tasklet_schedule(struct tasklet_struct *t, unsigned int k_id) | |||
28 | } | 28 | } |
29 | 29 | ||
30 | ret = litmus->enqueue_pai_tasklet(t); | 30 | ret = litmus->enqueue_pai_tasklet(t); |
31 | 31 | ||
32 | return(ret); | 32 | return(ret); |
33 | } | 33 | } |
34 | 34 | ||
diff --git a/litmus/litmus_proc.c b/litmus/litmus_proc.c index 381513366c7a..9ab7e015a3c1 100644 --- a/litmus/litmus_proc.c +++ b/litmus/litmus_proc.c | |||
@@ -172,8 +172,8 @@ int __init init_litmus_proc(void) | |||
172 | klitirqd_file = | 172 | klitirqd_file = |
173 | create_proc_read_entry("klitirqd_stats", 0444, litmus_dir, | 173 | create_proc_read_entry("klitirqd_stats", 0444, litmus_dir, |
174 | proc_read_klitirqd_stats, NULL); | 174 | proc_read_klitirqd_stats, NULL); |
175 | #endif | 175 | #endif |
176 | 176 | ||
177 | stat_file = create_proc_read_entry("stats", 0444, litmus_dir, | 177 | stat_file = create_proc_read_entry("stats", 0444, litmus_dir, |
178 | proc_read_stats, NULL); | 178 | proc_read_stats, NULL); |
179 | 179 | ||
diff --git a/litmus/litmus_softirq.c b/litmus/litmus_softirq.c index 6b033508877d..9f7d9da5facb 100644 --- a/litmus/litmus_softirq.c +++ b/litmus/litmus_softirq.c | |||
@@ -44,7 +44,7 @@ struct klitirqd_info | |||
44 | 44 | ||
45 | 45 | ||
46 | raw_spinlock_t lock; | 46 | raw_spinlock_t lock; |
47 | 47 | ||
48 | u32 pending; | 48 | u32 pending; |
49 | atomic_t num_hi_pending; | 49 | atomic_t num_hi_pending; |
50 | atomic_t num_low_pending; | 50 | atomic_t num_low_pending; |
@@ -70,7 +70,7 @@ int proc_read_klitirqd_stats(char *page, char **start, | |||
70 | int len = snprintf(page, PAGE_SIZE, | 70 | int len = snprintf(page, PAGE_SIZE, |
71 | "num ready klitirqds: %d\n\n", | 71 | "num ready klitirqds: %d\n\n", |
72 | atomic_read(&num_ready_klitirqds)); | 72 | atomic_read(&num_ready_klitirqds)); |
73 | 73 | ||
74 | if(klitirqd_is_ready()) | 74 | if(klitirqd_is_ready()) |
75 | { | 75 | { |
76 | int i; | 76 | int i; |
@@ -100,7 +100,7 @@ int proc_read_klitirqd_stats(char *page, char **start, | |||
100 | return(len); | 100 | return(len); |
101 | } | 101 | } |
102 | 102 | ||
103 | 103 | ||
104 | 104 | ||
105 | 105 | ||
106 | 106 | ||
@@ -185,9 +185,9 @@ inline unsigned int klitirqd_id(struct task_struct* tsk) | |||
185 | return i; | 185 | return i; |
186 | } | 186 | } |
187 | } | 187 | } |
188 | 188 | ||
189 | BUG(); | 189 | BUG(); |
190 | 190 | ||
191 | return 0; | 191 | return 0; |
192 | } | 192 | } |
193 | 193 | ||
@@ -217,11 +217,11 @@ inline static u32 litirq_pending(struct klitirqd_info* which) | |||
217 | { | 217 | { |
218 | unsigned long flags; | 218 | unsigned long flags; |
219 | u32 pending; | 219 | u32 pending; |
220 | 220 | ||
221 | raw_spin_lock_irqsave(&which->lock, flags); | 221 | raw_spin_lock_irqsave(&which->lock, flags); |
222 | pending = litirq_pending_irqoff(which); | 222 | pending = litirq_pending_irqoff(which); |
223 | raw_spin_unlock_irqrestore(&which->lock, flags); | 223 | raw_spin_unlock_irqrestore(&which->lock, flags); |
224 | 224 | ||
225 | return pending; | 225 | return pending; |
226 | }; | 226 | }; |
227 | 227 | ||
@@ -296,13 +296,13 @@ static int tasklet_ownership_change( | |||
296 | int ret = 0; | 296 | int ret = 0; |
297 | 297 | ||
298 | raw_spin_lock_irqsave(&which->lock, flags); | 298 | raw_spin_lock_irqsave(&which->lock, flags); |
299 | 299 | ||
300 | switch(taskletQ) | 300 | switch(taskletQ) |
301 | { | 301 | { |
302 | case LIT_TASKLET_HI: | 302 | case LIT_TASKLET_HI: |
303 | if(litirq_pending_hi_irqoff(which)) | 303 | if(litirq_pending_hi_irqoff(which)) |
304 | { | 304 | { |
305 | ret = (which->pending_tasklets_hi.head->owner != | 305 | ret = (which->pending_tasklets_hi.head->owner != |
306 | which->current_owner); | 306 | which->current_owner); |
307 | } | 307 | } |
308 | break; | 308 | break; |
@@ -316,11 +316,11 @@ static int tasklet_ownership_change( | |||
316 | default: | 316 | default: |
317 | break; | 317 | break; |
318 | } | 318 | } |
319 | 319 | ||
320 | raw_spin_unlock_irqrestore(&which->lock, flags); | 320 | raw_spin_unlock_irqrestore(&which->lock, flags); |
321 | 321 | ||
322 | TRACE_TASK(which->klitirqd, "ownership change needed: %d\n", ret); | 322 | TRACE_TASK(which->klitirqd, "ownership change needed: %d\n", ret); |
323 | 323 | ||
324 | return ret; | 324 | return ret; |
325 | } | 325 | } |
326 | 326 | ||
@@ -329,12 +329,12 @@ static void __reeval_prio(struct klitirqd_info* which) | |||
329 | { | 329 | { |
330 | struct task_struct* next_owner = NULL; | 330 | struct task_struct* next_owner = NULL; |
331 | struct task_struct* klitirqd = which->klitirqd; | 331 | struct task_struct* klitirqd = which->klitirqd; |
332 | 332 | ||
333 | /* Check in prio-order */ | 333 | /* Check in prio-order */ |
334 | u32 pending = litirq_pending_irqoff(which); | 334 | u32 pending = litirq_pending_irqoff(which); |
335 | 335 | ||
336 | //__dump_state(which, "__reeval_prio: before"); | 336 | //__dump_state(which, "__reeval_prio: before"); |
337 | 337 | ||
338 | if(pending) | 338 | if(pending) |
339 | { | 339 | { |
340 | if(pending & LIT_TASKLET_HI) | 340 | if(pending & LIT_TASKLET_HI) |
@@ -375,9 +375,9 @@ static void __reeval_prio(struct klitirqd_info* which) | |||
375 | TRACE("%s: Ownership change: %s/%d to %s/%d\n", __FUNCTION__, | 375 | TRACE("%s: Ownership change: %s/%d to %s/%d\n", __FUNCTION__, |
376 | ((tsk_rt(klitirqd)->inh_task) ? tsk_rt(klitirqd)->inh_task : klitirqd)->comm, | 376 | ((tsk_rt(klitirqd)->inh_task) ? tsk_rt(klitirqd)->inh_task : klitirqd)->comm, |
377 | ((tsk_rt(klitirqd)->inh_task) ? tsk_rt(klitirqd)->inh_task : klitirqd)->pid, | 377 | ((tsk_rt(klitirqd)->inh_task) ? tsk_rt(klitirqd)->inh_task : klitirqd)->pid, |
378 | next_owner->comm, next_owner->pid); | 378 | next_owner->comm, next_owner->pid); |
379 | } | 379 | } |
380 | 380 | ||
381 | litmus->increase_prio_inheritance_klitirqd(klitirqd, old_owner, next_owner); | 381 | litmus->increase_prio_inheritance_klitirqd(klitirqd, old_owner, next_owner); |
382 | } | 382 | } |
383 | else | 383 | else |
@@ -391,21 +391,21 @@ static void __reeval_prio(struct klitirqd_info* which) | |||
391 | { | 391 | { |
392 | // is this a bug? | 392 | // is this a bug? |
393 | TRACE("%s: Ownership change: %s/%d to NULL (reverting)\n", | 393 | TRACE("%s: Ownership change: %s/%d to NULL (reverting)\n", |
394 | __FUNCTION__, klitirqd->comm, klitirqd->pid); | 394 | __FUNCTION__, klitirqd->comm, klitirqd->pid); |
395 | } | 395 | } |
396 | 396 | ||
397 | BUG_ON(pending != 0); | 397 | BUG_ON(pending != 0); |
398 | litmus->decrease_prio_inheritance_klitirqd(klitirqd, old_owner, NULL); | 398 | litmus->decrease_prio_inheritance_klitirqd(klitirqd, old_owner, NULL); |
399 | } | 399 | } |
400 | } | 400 | } |
401 | 401 | ||
402 | //__dump_state(which, "__reeval_prio: after"); | 402 | //__dump_state(which, "__reeval_prio: after"); |
403 | } | 403 | } |
404 | 404 | ||
405 | static void reeval_prio(struct klitirqd_info* which) | 405 | static void reeval_prio(struct klitirqd_info* which) |
406 | { | 406 | { |
407 | unsigned long flags; | 407 | unsigned long flags; |
408 | 408 | ||
409 | raw_spin_lock_irqsave(&which->lock, flags); | 409 | raw_spin_lock_irqsave(&which->lock, flags); |
410 | __reeval_prio(which); | 410 | __reeval_prio(which); |
411 | raw_spin_unlock_irqrestore(&which->lock, flags); | 411 | raw_spin_unlock_irqrestore(&which->lock, flags); |
@@ -438,41 +438,41 @@ static void do_lit_tasklet(struct klitirqd_info* which, | |||
438 | atomic_t* count; | 438 | atomic_t* count; |
439 | 439 | ||
440 | raw_spin_lock_irqsave(&which->lock, flags); | 440 | raw_spin_lock_irqsave(&which->lock, flags); |
441 | 441 | ||
442 | //__dump_state(which, "do_lit_tasklet: before steal"); | 442 | //__dump_state(which, "do_lit_tasklet: before steal"); |
443 | 443 | ||
444 | /* copy out the tasklets for our private use. */ | 444 | /* copy out the tasklets for our private use. */ |
445 | list = pending_tasklets->head; | 445 | list = pending_tasklets->head; |
446 | pending_tasklets->head = NULL; | 446 | pending_tasklets->head = NULL; |
447 | pending_tasklets->tail = &pending_tasklets->head; | 447 | pending_tasklets->tail = &pending_tasklets->head; |
448 | 448 | ||
449 | /* remove pending flag */ | 449 | /* remove pending flag */ |
450 | which->pending &= (pending_tasklets == &which->pending_tasklets) ? | 450 | which->pending &= (pending_tasklets == &which->pending_tasklets) ? |
451 | ~LIT_TASKLET_LOW : | 451 | ~LIT_TASKLET_LOW : |
452 | ~LIT_TASKLET_HI; | 452 | ~LIT_TASKLET_HI; |
453 | 453 | ||
454 | count = (pending_tasklets == &which->pending_tasklets) ? | 454 | count = (pending_tasklets == &which->pending_tasklets) ? |
455 | &which->num_low_pending: | 455 | &which->num_low_pending: |
456 | &which->num_hi_pending; | 456 | &which->num_hi_pending; |
457 | 457 | ||
458 | //__dump_state(which, "do_lit_tasklet: after steal"); | 458 | //__dump_state(which, "do_lit_tasklet: after steal"); |
459 | 459 | ||
460 | raw_spin_unlock_irqrestore(&which->lock, flags); | 460 | raw_spin_unlock_irqrestore(&which->lock, flags); |
461 | 461 | ||
462 | 462 | ||
463 | while(list) | 463 | while(list) |
464 | { | 464 | { |
465 | struct tasklet_struct *t = list; | 465 | struct tasklet_struct *t = list; |
466 | 466 | ||
467 | /* advance, lest we forget */ | 467 | /* advance, lest we forget */ |
468 | list = list->next; | 468 | list = list->next; |
469 | 469 | ||
470 | /* execute tasklet if it has my priority and is free */ | 470 | /* execute tasklet if it has my priority and is free */ |
471 | if ((t->owner == which->current_owner) && tasklet_trylock(t)) { | 471 | if ((t->owner == which->current_owner) && tasklet_trylock(t)) { |
472 | if (!atomic_read(&t->count)) { | 472 | if (!atomic_read(&t->count)) { |
473 | 473 | ||
474 | sched_trace_tasklet_begin(t->owner); | 474 | sched_trace_tasklet_begin(t->owner); |
475 | 475 | ||
476 | if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) | 476 | if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) |
477 | { | 477 | { |
478 | BUG(); | 478 | BUG(); |
@@ -480,18 +480,18 @@ static void do_lit_tasklet(struct klitirqd_info* which, | |||
480 | TRACE_CUR("%s: Invoking tasklet.\n", __FUNCTION__); | 480 | TRACE_CUR("%s: Invoking tasklet.\n", __FUNCTION__); |
481 | t->func(t->data); | 481 | t->func(t->data); |
482 | tasklet_unlock(t); | 482 | tasklet_unlock(t); |
483 | 483 | ||
484 | atomic_dec(count); | 484 | atomic_dec(count); |
485 | 485 | ||
486 | sched_trace_tasklet_end(t->owner, 0ul); | 486 | sched_trace_tasklet_end(t->owner, 0ul); |
487 | 487 | ||
488 | continue; /* process more tasklets */ | 488 | continue; /* process more tasklets */ |
489 | } | 489 | } |
490 | tasklet_unlock(t); | 490 | tasklet_unlock(t); |
491 | } | 491 | } |
492 | 492 | ||
493 | TRACE_CUR("%s: Could not invoke tasklet. Requeuing.\n", __FUNCTION__); | 493 | TRACE_CUR("%s: Could not invoke tasklet. Requeuing.\n", __FUNCTION__); |
494 | 494 | ||
495 | /* couldn't process tasklet. put it back at the end of the queue. */ | 495 | /* couldn't process tasklet. put it back at the end of the queue. */ |
496 | if(pending_tasklets == &which->pending_tasklets) | 496 | if(pending_tasklets == &which->pending_tasklets) |
497 | ___litmus_tasklet_schedule(t, which, 0); | 497 | ___litmus_tasklet_schedule(t, which, 0); |
@@ -507,13 +507,13 @@ static int do_litirq(struct klitirqd_info* which) | |||
507 | { | 507 | { |
508 | u32 pending; | 508 | u32 pending; |
509 | int resched = 0; | 509 | int resched = 0; |
510 | 510 | ||
511 | if(in_interrupt()) | 511 | if(in_interrupt()) |
512 | { | 512 | { |
513 | TRACE("%s: exiting early: in interrupt context!\n", __FUNCTION__); | 513 | TRACE("%s: exiting early: in interrupt context!\n", __FUNCTION__); |
514 | return(0); | 514 | return(0); |
515 | } | 515 | } |
516 | 516 | ||
517 | if(which->klitirqd != current) | 517 | if(which->klitirqd != current) |
518 | { | 518 | { |
519 | TRACE_CUR("%s: exiting early: thread/info mismatch! Running %s/%d but given %s/%d.\n", | 519 | TRACE_CUR("%s: exiting early: thread/info mismatch! Running %s/%d but given %s/%d.\n", |
@@ -521,7 +521,7 @@ static int do_litirq(struct klitirqd_info* which) | |||
521 | which->klitirqd->comm, which->klitirqd->pid); | 521 | which->klitirqd->comm, which->klitirqd->pid); |
522 | return(0); | 522 | return(0); |
523 | } | 523 | } |
524 | 524 | ||
525 | if(!is_realtime(current)) | 525 | if(!is_realtime(current)) |
526 | { | 526 | { |
527 | TRACE_CUR("%s: exiting early: klitirqd is not real-time. Sched Policy = %d\n", | 527 | TRACE_CUR("%s: exiting early: klitirqd is not real-time. Sched Policy = %d\n", |
@@ -529,9 +529,9 @@ static int do_litirq(struct klitirqd_info* which) | |||
529 | return(0); | 529 | return(0); |
530 | } | 530 | } |
531 | 531 | ||
532 | 532 | ||
533 | /* We only handle tasklets & work objects, no need for RCU triggers? */ | 533 | /* We only handle tasklets & work objects, no need for RCU triggers? */ |
534 | 534 | ||
535 | pending = litirq_pending(which); | 535 | pending = litirq_pending(which); |
536 | if(pending) | 536 | if(pending) |
537 | { | 537 | { |
@@ -541,20 +541,20 @@ static int do_litirq(struct klitirqd_info* which) | |||
541 | TRACE_CUR("%s: Invoking HI tasklets.\n", __FUNCTION__); | 541 | TRACE_CUR("%s: Invoking HI tasklets.\n", __FUNCTION__); |
542 | do_lit_tasklet(which, &which->pending_tasklets_hi); | 542 | do_lit_tasklet(which, &which->pending_tasklets_hi); |
543 | resched = tasklet_ownership_change(which, LIT_TASKLET_HI); | 543 | resched = tasklet_ownership_change(which, LIT_TASKLET_HI); |
544 | 544 | ||
545 | if(resched) | 545 | if(resched) |
546 | { | 546 | { |
547 | TRACE_CUR("%s: HI tasklets of another owner remain. " | 547 | TRACE_CUR("%s: HI tasklets of another owner remain. " |
548 | "Skipping any LOW tasklets.\n", __FUNCTION__); | 548 | "Skipping any LOW tasklets.\n", __FUNCTION__); |
549 | } | 549 | } |
550 | } | 550 | } |
551 | 551 | ||
552 | if(!resched && (pending & LIT_TASKLET_LOW)) | 552 | if(!resched && (pending & LIT_TASKLET_LOW)) |
553 | { | 553 | { |
554 | TRACE_CUR("%s: Invoking LOW tasklets.\n", __FUNCTION__); | 554 | TRACE_CUR("%s: Invoking LOW tasklets.\n", __FUNCTION__); |
555 | do_lit_tasklet(which, &which->pending_tasklets); | 555 | do_lit_tasklet(which, &which->pending_tasklets); |
556 | resched = tasklet_ownership_change(which, LIT_TASKLET_LOW); | 556 | resched = tasklet_ownership_change(which, LIT_TASKLET_LOW); |
557 | 557 | ||
558 | if(resched) | 558 | if(resched) |
559 | { | 559 | { |
560 | TRACE_CUR("%s: LOW tasklets of another owner remain. " | 560 | TRACE_CUR("%s: LOW tasklets of another owner remain. " |
@@ -562,7 +562,7 @@ static int do_litirq(struct klitirqd_info* which) | |||
562 | } | 562 | } |
563 | } | 563 | } |
564 | } | 564 | } |
565 | 565 | ||
566 | return(resched); | 566 | return(resched); |
567 | } | 567 | } |
568 | 568 | ||
@@ -572,11 +572,11 @@ static void do_work(struct klitirqd_info* which) | |||
572 | unsigned long flags; | 572 | unsigned long flags; |
573 | work_func_t f; | 573 | work_func_t f; |
574 | struct work_struct* work; | 574 | struct work_struct* work; |
575 | 575 | ||
576 | // only execute one work-queue item to yield to tasklets. | 576 | // only execute one work-queue item to yield to tasklets. |
577 | // ...is this a good idea, or should we just batch them? | 577 | // ...is this a good idea, or should we just batch them? |
578 | raw_spin_lock_irqsave(&which->lock, flags); | 578 | raw_spin_lock_irqsave(&which->lock, flags); |
579 | 579 | ||
580 | if(!litirq_pending_work_irqoff(which)) | 580 | if(!litirq_pending_work_irqoff(which)) |
581 | { | 581 | { |
582 | raw_spin_unlock_irqrestore(&which->lock, flags); | 582 | raw_spin_unlock_irqrestore(&which->lock, flags); |
@@ -585,16 +585,16 @@ static void do_work(struct klitirqd_info* which) | |||
585 | 585 | ||
586 | work = list_first_entry(&which->worklist, struct work_struct, entry); | 586 | work = list_first_entry(&which->worklist, struct work_struct, entry); |
587 | list_del_init(&work->entry); | 587 | list_del_init(&work->entry); |
588 | 588 | ||
589 | if(list_empty(&which->worklist)) | 589 | if(list_empty(&which->worklist)) |
590 | { | 590 | { |
591 | which->pending &= ~LIT_WORK; | 591 | which->pending &= ~LIT_WORK; |
592 | } | 592 | } |
593 | 593 | ||
594 | raw_spin_unlock_irqrestore(&which->lock, flags); | 594 | raw_spin_unlock_irqrestore(&which->lock, flags); |
595 | 595 | ||
596 | 596 | ||
597 | 597 | ||
598 | /* safe to read current_owner outside of lock since only this thread | 598 | /* safe to read current_owner outside of lock since only this thread |
599 | may write to the pointer. */ | 599 | may write to the pointer. */ |
600 | if(work->owner == which->current_owner) | 600 | if(work->owner == which->current_owner) |
@@ -605,7 +605,7 @@ static void do_work(struct klitirqd_info* which) | |||
605 | f = work->func; | 605 | f = work->func; |
606 | f(work); /* can't touch 'work' after this point, | 606 | f(work); /* can't touch 'work' after this point, |
607 | the user may have freed it. */ | 607 | the user may have freed it. */ |
608 | 608 | ||
609 | atomic_dec(&which->num_work_pending); | 609 | atomic_dec(&which->num_work_pending); |
610 | } | 610 | } |
611 | else | 611 | else |
@@ -614,7 +614,7 @@ static void do_work(struct klitirqd_info* which) | |||
614 | __FUNCTION__); | 614 | __FUNCTION__); |
615 | ___litmus_schedule_work(work, which, 0); | 615 | ___litmus_schedule_work(work, which, 0); |
616 | } | 616 | } |
617 | 617 | ||
618 | no_work: | 618 | no_work: |
619 | return; | 619 | return; |
620 | } | 620 | } |
@@ -628,7 +628,7 @@ static int set_litmus_daemon_sched(void) | |||
628 | 628 | ||
629 | TODO: Transition to a new job whenever a | 629 | TODO: Transition to a new job whenever a |
630 | new tasklet is handled */ | 630 | new tasklet is handled */ |
631 | 631 | ||
632 | int ret = 0; | 632 | int ret = 0; |
633 | 633 | ||
634 | struct rt_task tp = { | 634 | struct rt_task tp = { |
@@ -639,20 +639,20 @@ static int set_litmus_daemon_sched(void) | |||
639 | .budget_policy = NO_ENFORCEMENT, | 639 | .budget_policy = NO_ENFORCEMENT, |
640 | .cls = RT_CLASS_BEST_EFFORT | 640 | .cls = RT_CLASS_BEST_EFFORT |
641 | }; | 641 | }; |
642 | 642 | ||
643 | struct sched_param param = { .sched_priority = 0}; | 643 | struct sched_param param = { .sched_priority = 0}; |
644 | 644 | ||
645 | 645 | ||
646 | /* set task params, mark as proxy thread, and init other data */ | 646 | /* set task params, mark as proxy thread, and init other data */ |
647 | tsk_rt(current)->task_params = tp; | 647 | tsk_rt(current)->task_params = tp; |
648 | tsk_rt(current)->is_proxy_thread = 1; | 648 | tsk_rt(current)->is_proxy_thread = 1; |
649 | tsk_rt(current)->cur_klitirqd = NULL; | 649 | tsk_rt(current)->cur_klitirqd = NULL; |
650 | mutex_init(&tsk_rt(current)->klitirqd_sem); | 650 | mutex_init(&tsk_rt(current)->klitirqd_sem); |
651 | atomic_set(&tsk_rt(current)->klitirqd_sem_stat, NOT_HELD); | 651 | atomic_set(&tsk_rt(current)->klitirqd_sem_stat, NOT_HELD); |
652 | 652 | ||
653 | /* inform the OS we're SCHED_LITMUS -- | 653 | /* inform the OS we're SCHED_LITMUS -- |
654 | sched_setscheduler_nocheck() calls litmus_admit_task(). */ | 654 | sched_setscheduler_nocheck() calls litmus_admit_task(). */ |
655 | sched_setscheduler_nocheck(current, SCHED_LITMUS, ¶m); | 655 | sched_setscheduler_nocheck(current, SCHED_LITMUS, ¶m); |
656 | 656 | ||
657 | return ret; | 657 | return ret; |
658 | } | 658 | } |
@@ -682,7 +682,7 @@ static void exit_execution_phase(struct klitirqd_info* which, | |||
682 | up_and_set_stat(current, NOT_HELD, sem); | 682 | up_and_set_stat(current, NOT_HELD, sem); |
683 | TRACE_CUR("%s: Execution phase exited! " | 683 | TRACE_CUR("%s: Execution phase exited! " |
684 | "Released semaphore of %s/%d\n", __FUNCTION__, | 684 | "Released semaphore of %s/%d\n", __FUNCTION__, |
685 | t->comm, t->pid); | 685 | t->comm, t->pid); |
686 | } | 686 | } |
687 | else | 687 | else |
688 | { | 688 | { |
@@ -704,11 +704,11 @@ static int run_klitirqd(void* unused) | |||
704 | TRACE_CUR("%s: Failed to transition to rt-task.\n", __FUNCTION__); | 704 | TRACE_CUR("%s: Failed to transition to rt-task.\n", __FUNCTION__); |
705 | goto rt_failed; | 705 | goto rt_failed; |
706 | } | 706 | } |
707 | 707 | ||
708 | atomic_inc(&num_ready_klitirqds); | 708 | atomic_inc(&num_ready_klitirqds); |
709 | 709 | ||
710 | set_current_state(TASK_INTERRUPTIBLE); | 710 | set_current_state(TASK_INTERRUPTIBLE); |
711 | 711 | ||
712 | while (!kthread_should_stop()) | 712 | while (!kthread_should_stop()) |
713 | { | 713 | { |
714 | preempt_disable(); | 714 | preempt_disable(); |
@@ -728,7 +728,7 @@ static int run_klitirqd(void* unused) | |||
728 | 728 | ||
729 | preempt_disable(); | 729 | preempt_disable(); |
730 | } | 730 | } |
731 | 731 | ||
732 | __set_current_state(TASK_RUNNING); | 732 | __set_current_state(TASK_RUNNING); |
733 | 733 | ||
734 | while (litirq_pending_and_sem_and_owner(which, &sem, &owner)) | 734 | while (litirq_pending_and_sem_and_owner(which, &sem, &owner)) |
@@ -736,7 +736,7 @@ static int run_klitirqd(void* unused) | |||
736 | int needs_resched = 0; | 736 | int needs_resched = 0; |
737 | 737 | ||
738 | preempt_enable_no_resched(); | 738 | preempt_enable_no_resched(); |
739 | 739 | ||
740 | BUG_ON(sem == NULL); | 740 | BUG_ON(sem == NULL); |
741 | 741 | ||
742 | // wait to enter execution phase; wait for 'current_owner' to block. | 742 | // wait to enter execution phase; wait for 'current_owner' to block. |
@@ -749,28 +749,28 @@ static int run_klitirqd(void* unused) | |||
749 | } | 749 | } |
750 | 750 | ||
751 | preempt_disable(); | 751 | preempt_disable(); |
752 | 752 | ||
753 | /* Double check that there's still pending work and the owner hasn't | 753 | /* Double check that there's still pending work and the owner hasn't |
754 | * changed. Pending items may have been flushed while we were sleeping. | 754 | * changed. Pending items may have been flushed while we were sleeping. |
755 | */ | 755 | */ |
756 | if(litirq_pending_with_owner(which, owner)) | 756 | if(litirq_pending_with_owner(which, owner)) |
757 | { | 757 | { |
758 | TRACE_CUR("%s: Executing tasklets and/or work objects.\n", | 758 | TRACE_CUR("%s: Executing tasklets and/or work objects.\n", |
759 | __FUNCTION__); | 759 | __FUNCTION__); |
760 | 760 | ||
761 | needs_resched = do_litirq(which); | 761 | needs_resched = do_litirq(which); |
762 | 762 | ||
763 | preempt_enable_no_resched(); | 763 | preempt_enable_no_resched(); |
764 | 764 | ||
765 | // work objects are preemptible. | 765 | // work objects are preemptible. |
766 | if(!needs_resched) | 766 | if(!needs_resched) |
767 | { | 767 | { |
768 | do_work(which); | 768 | do_work(which); |
769 | } | 769 | } |
770 | 770 | ||
771 | // exit execution phase. | 771 | // exit execution phase. |
772 | exit_execution_phase(which, sem, owner); | 772 | exit_execution_phase(which, sem, owner); |
773 | 773 | ||
774 | TRACE_CUR("%s: Setting up next priority.\n", __FUNCTION__); | 774 | TRACE_CUR("%s: Setting up next priority.\n", __FUNCTION__); |
775 | reeval_prio(which); /* check if we need to change priority here */ | 775 | reeval_prio(which); /* check if we need to change priority here */ |
776 | } | 776 | } |
@@ -778,7 +778,7 @@ static int run_klitirqd(void* unused) | |||
778 | { | 778 | { |
779 | TRACE_CUR("%s: Pending work was flushed! Prev owner was %s/%d\n", | 779 | TRACE_CUR("%s: Pending work was flushed! Prev owner was %s/%d\n", |
780 | __FUNCTION__, | 780 | __FUNCTION__, |
781 | owner->comm, owner->pid); | 781 | owner->comm, owner->pid); |
782 | preempt_enable_no_resched(); | 782 | preempt_enable_no_resched(); |
783 | 783 | ||
784 | // exit execution phase. | 784 | // exit execution phase. |
@@ -792,12 +792,12 @@ static int run_klitirqd(void* unused) | |||
792 | set_current_state(TASK_INTERRUPTIBLE); | 792 | set_current_state(TASK_INTERRUPTIBLE); |
793 | } | 793 | } |
794 | __set_current_state(TASK_RUNNING); | 794 | __set_current_state(TASK_RUNNING); |
795 | 795 | ||
796 | atomic_dec(&num_ready_klitirqds); | 796 | atomic_dec(&num_ready_klitirqds); |
797 | 797 | ||
798 | rt_failed: | 798 | rt_failed: |
799 | litmus_exit_task(current); | 799 | litmus_exit_task(current); |
800 | 800 | ||
801 | return rt_status; | 801 | return rt_status; |
802 | } | 802 | } |
803 | 803 | ||
@@ -812,18 +812,18 @@ struct klitirqd_launch_data | |||
812 | static void launch_klitirqd(struct work_struct *work) | 812 | static void launch_klitirqd(struct work_struct *work) |
813 | { | 813 | { |
814 | int i; | 814 | int i; |
815 | 815 | ||
816 | struct klitirqd_launch_data* launch_data = | 816 | struct klitirqd_launch_data* launch_data = |
817 | container_of(work, struct klitirqd_launch_data, work); | 817 | container_of(work, struct klitirqd_launch_data, work); |
818 | 818 | ||
819 | TRACE("%s: Creating %d klitirqds\n", __FUNCTION__, NR_LITMUS_SOFTIRQD); | 819 | TRACE("%s: Creating %d klitirqds\n", __FUNCTION__, NR_LITMUS_SOFTIRQD); |
820 | 820 | ||
821 | /* create the daemon threads */ | 821 | /* create the daemon threads */ |
822 | for(i = 0; i < NR_LITMUS_SOFTIRQD; ++i) | 822 | for(i = 0; i < NR_LITMUS_SOFTIRQD; ++i) |
823 | { | 823 | { |
824 | if(launch_data->cpu_affinity) | 824 | if(launch_data->cpu_affinity) |
825 | { | 825 | { |
826 | klitirqds[i].klitirqd = | 826 | klitirqds[i].klitirqd = |
827 | kthread_create( | 827 | kthread_create( |
828 | run_klitirqd, | 828 | run_klitirqd, |
829 | /* treat the affinity as a pointer, we'll cast it back later */ | 829 | /* treat the affinity as a pointer, we'll cast it back later */ |
@@ -831,13 +831,13 @@ static void launch_klitirqd(struct work_struct *work) | |||
831 | "klitirqd_th%d/%d", | 831 | "klitirqd_th%d/%d", |
832 | i, | 832 | i, |
833 | launch_data->cpu_affinity[i]); | 833 | launch_data->cpu_affinity[i]); |
834 | 834 | ||
835 | /* litmus will put is in the right cluster. */ | 835 | /* litmus will put is in the right cluster. */ |
836 | kthread_bind(klitirqds[i].klitirqd, launch_data->cpu_affinity[i]); | 836 | kthread_bind(klitirqds[i].klitirqd, launch_data->cpu_affinity[i]); |
837 | } | 837 | } |
838 | else | 838 | else |
839 | { | 839 | { |
840 | klitirqds[i].klitirqd = | 840 | klitirqds[i].klitirqd = |
841 | kthread_create( | 841 | kthread_create( |
842 | run_klitirqd, | 842 | run_klitirqd, |
843 | /* treat the affinity as a pointer, we'll cast it back later */ | 843 | /* treat the affinity as a pointer, we'll cast it back later */ |
@@ -845,16 +845,16 @@ static void launch_klitirqd(struct work_struct *work) | |||
845 | "klitirqd_th%d", | 845 | "klitirqd_th%d", |
846 | i); | 846 | i); |
847 | } | 847 | } |
848 | } | 848 | } |
849 | 849 | ||
850 | TRACE("%s: Launching %d klitirqds\n", __FUNCTION__, NR_LITMUS_SOFTIRQD); | 850 | TRACE("%s: Launching %d klitirqds\n", __FUNCTION__, NR_LITMUS_SOFTIRQD); |
851 | 851 | ||
852 | /* unleash the daemons */ | 852 | /* unleash the daemons */ |
853 | for(i = 0; i < NR_LITMUS_SOFTIRQD; ++i) | 853 | for(i = 0; i < NR_LITMUS_SOFTIRQD; ++i) |
854 | { | 854 | { |
855 | wake_up_process(klitirqds[i].klitirqd); | 855 | wake_up_process(klitirqds[i].klitirqd); |
856 | } | 856 | } |
857 | 857 | ||
858 | if(launch_data->cpu_affinity) | 858 | if(launch_data->cpu_affinity) |
859 | kfree(launch_data->cpu_affinity); | 859 | kfree(launch_data->cpu_affinity); |
860 | kfree(launch_data); | 860 | kfree(launch_data); |
@@ -865,38 +865,38 @@ void spawn_klitirqd(int* affinity) | |||
865 | { | 865 | { |
866 | int i; | 866 | int i; |
867 | struct klitirqd_launch_data* delayed_launch; | 867 | struct klitirqd_launch_data* delayed_launch; |
868 | 868 | ||
869 | if(atomic_read(&num_ready_klitirqds) != 0) | 869 | if(atomic_read(&num_ready_klitirqds) != 0) |
870 | { | 870 | { |
871 | TRACE("%s: At least one klitirqd is already running! Need to call kill_klitirqd()?\n"); | 871 | TRACE("%s: At least one klitirqd is already running! Need to call kill_klitirqd()?\n"); |
872 | return; | 872 | return; |
873 | } | 873 | } |
874 | 874 | ||
875 | /* init the tasklet & work queues */ | 875 | /* init the tasklet & work queues */ |
876 | for(i = 0; i < NR_LITMUS_SOFTIRQD; ++i) | 876 | for(i = 0; i < NR_LITMUS_SOFTIRQD; ++i) |
877 | { | 877 | { |
878 | klitirqds[i].terminating = 0; | 878 | klitirqds[i].terminating = 0; |
879 | klitirqds[i].pending = 0; | 879 | klitirqds[i].pending = 0; |
880 | 880 | ||
881 | klitirqds[i].num_hi_pending.counter = 0; | 881 | klitirqds[i].num_hi_pending.counter = 0; |
882 | klitirqds[i].num_low_pending.counter = 0; | 882 | klitirqds[i].num_low_pending.counter = 0; |
883 | klitirqds[i].num_work_pending.counter = 0; | 883 | klitirqds[i].num_work_pending.counter = 0; |
884 | 884 | ||
885 | klitirqds[i].pending_tasklets_hi.head = NULL; | 885 | klitirqds[i].pending_tasklets_hi.head = NULL; |
886 | klitirqds[i].pending_tasklets_hi.tail = &klitirqds[i].pending_tasklets_hi.head; | 886 | klitirqds[i].pending_tasklets_hi.tail = &klitirqds[i].pending_tasklets_hi.head; |
887 | 887 | ||
888 | klitirqds[i].pending_tasklets.head = NULL; | 888 | klitirqds[i].pending_tasklets.head = NULL; |
889 | klitirqds[i].pending_tasklets.tail = &klitirqds[i].pending_tasklets.head; | 889 | klitirqds[i].pending_tasklets.tail = &klitirqds[i].pending_tasklets.head; |
890 | 890 | ||
891 | INIT_LIST_HEAD(&klitirqds[i].worklist); | 891 | INIT_LIST_HEAD(&klitirqds[i].worklist); |
892 | 892 | ||
893 | raw_spin_lock_init(&klitirqds[i].lock); | 893 | raw_spin_lock_init(&klitirqds[i].lock); |
894 | } | 894 | } |
895 | 895 | ||
896 | /* wait to flush the initializations to memory since other threads | 896 | /* wait to flush the initializations to memory since other threads |
897 | will access it. */ | 897 | will access it. */ |
898 | mb(); | 898 | mb(); |
899 | 899 | ||
900 | /* tell a work queue to launch the threads. we can't make scheduling | 900 | /* tell a work queue to launch the threads. we can't make scheduling |
901 | calls since we're in an atomic state. */ | 901 | calls since we're in an atomic state. */ |
902 | TRACE("%s: Setting callback up to launch klitirqds\n", __FUNCTION__); | 902 | TRACE("%s: Setting callback up to launch klitirqds\n", __FUNCTION__); |
@@ -905,7 +905,7 @@ void spawn_klitirqd(int* affinity) | |||
905 | { | 905 | { |
906 | delayed_launch->cpu_affinity = | 906 | delayed_launch->cpu_affinity = |
907 | kmalloc(sizeof(int)*NR_LITMUS_SOFTIRQD, GFP_ATOMIC); | 907 | kmalloc(sizeof(int)*NR_LITMUS_SOFTIRQD, GFP_ATOMIC); |
908 | 908 | ||
909 | memcpy(delayed_launch->cpu_affinity, affinity, | 909 | memcpy(delayed_launch->cpu_affinity, affinity, |
910 | sizeof(int)*NR_LITMUS_SOFTIRQD); | 910 | sizeof(int)*NR_LITMUS_SOFTIRQD); |
911 | } | 911 | } |
@@ -923,9 +923,9 @@ void kill_klitirqd(void) | |||
923 | if(!klitirqd_is_dead()) | 923 | if(!klitirqd_is_dead()) |
924 | { | 924 | { |
925 | int i; | 925 | int i; |
926 | 926 | ||
927 | TRACE("%s: Killing %d klitirqds\n", __FUNCTION__, NR_LITMUS_SOFTIRQD); | 927 | TRACE("%s: Killing %d klitirqds\n", __FUNCTION__, NR_LITMUS_SOFTIRQD); |
928 | 928 | ||
929 | for(i = 0; i < NR_LITMUS_SOFTIRQD; ++i) | 929 | for(i = 0; i < NR_LITMUS_SOFTIRQD; ++i) |
930 | { | 930 | { |
931 | if(klitirqds[i].terminating != 1) | 931 | if(klitirqds[i].terminating != 1) |
@@ -961,35 +961,35 @@ struct task_struct* get_klitirqd(unsigned int k_id) | |||
961 | 961 | ||
962 | void flush_pending(struct task_struct* klitirqd_thread, | 962 | void flush_pending(struct task_struct* klitirqd_thread, |
963 | struct task_struct* owner) | 963 | struct task_struct* owner) |
964 | { | 964 | { |
965 | unsigned int k_id = klitirqd_id(klitirqd_thread); | 965 | unsigned int k_id = klitirqd_id(klitirqd_thread); |
966 | struct klitirqd_info *which = &klitirqds[k_id]; | 966 | struct klitirqd_info *which = &klitirqds[k_id]; |
967 | 967 | ||
968 | unsigned long flags; | 968 | unsigned long flags; |
969 | struct tasklet_struct *list; | 969 | struct tasklet_struct *list; |
970 | 970 | ||
971 | u32 work_flushed = 0; | 971 | u32 work_flushed = 0; |
972 | 972 | ||
973 | raw_spin_lock_irqsave(&which->lock, flags); | 973 | raw_spin_lock_irqsave(&which->lock, flags); |
974 | 974 | ||
975 | //__dump_state(which, "flush_pending: before"); | 975 | //__dump_state(which, "flush_pending: before"); |
976 | 976 | ||
977 | // flush hi tasklets. | 977 | // flush hi tasklets. |
978 | if(litirq_pending_hi_irqoff(which)) | 978 | if(litirq_pending_hi_irqoff(which)) |
979 | { | 979 | { |
980 | which->pending &= ~LIT_TASKLET_HI; | 980 | which->pending &= ~LIT_TASKLET_HI; |
981 | 981 | ||
982 | list = which->pending_tasklets_hi.head; | 982 | list = which->pending_tasklets_hi.head; |
983 | which->pending_tasklets_hi.head = NULL; | 983 | which->pending_tasklets_hi.head = NULL; |
984 | which->pending_tasklets_hi.tail = &which->pending_tasklets_hi.head; | 984 | which->pending_tasklets_hi.tail = &which->pending_tasklets_hi.head; |
985 | 985 | ||
986 | TRACE("%s: Handing HI tasklets back to Linux.\n", __FUNCTION__); | 986 | TRACE("%s: Handing HI tasklets back to Linux.\n", __FUNCTION__); |
987 | 987 | ||
988 | while(list) | 988 | while(list) |
989 | { | 989 | { |
990 | struct tasklet_struct *t = list; | 990 | struct tasklet_struct *t = list; |
991 | list = list->next; | 991 | list = list->next; |
992 | 992 | ||
993 | if(likely((t->owner == owner) || (owner == NULL))) | 993 | if(likely((t->owner == owner) || (owner == NULL))) |
994 | { | 994 | { |
995 | if(unlikely(!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))) | 995 | if(unlikely(!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))) |
@@ -1021,23 +1021,23 @@ void flush_pending(struct task_struct* klitirqd_thread, | |||
1021 | } | 1021 | } |
1022 | } | 1022 | } |
1023 | } | 1023 | } |
1024 | 1024 | ||
1025 | // flush low tasklets. | 1025 | // flush low tasklets. |
1026 | if(litirq_pending_low_irqoff(which)) | 1026 | if(litirq_pending_low_irqoff(which)) |
1027 | { | 1027 | { |
1028 | which->pending &= ~LIT_TASKLET_LOW; | 1028 | which->pending &= ~LIT_TASKLET_LOW; |
1029 | 1029 | ||
1030 | list = which->pending_tasklets.head; | 1030 | list = which->pending_tasklets.head; |
1031 | which->pending_tasklets.head = NULL; | 1031 | which->pending_tasklets.head = NULL; |
1032 | which->pending_tasklets.tail = &which->pending_tasklets.head; | 1032 | which->pending_tasklets.tail = &which->pending_tasklets.head; |
1033 | 1033 | ||
1034 | TRACE("%s: Handing LOW tasklets back to Linux.\n", __FUNCTION__); | 1034 | TRACE("%s: Handing LOW tasklets back to Linux.\n", __FUNCTION__); |
1035 | 1035 | ||
1036 | while(list) | 1036 | while(list) |
1037 | { | 1037 | { |
1038 | struct tasklet_struct *t = list; | 1038 | struct tasklet_struct *t = list; |
1039 | list = list->next; | 1039 | list = list->next; |
1040 | 1040 | ||
1041 | if(likely((t->owner == owner) || (owner == NULL))) | 1041 | if(likely((t->owner == owner) || (owner == NULL))) |
1042 | { | 1042 | { |
1043 | if(unlikely(!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))) | 1043 | if(unlikely(!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))) |
@@ -1046,7 +1046,7 @@ void flush_pending(struct task_struct* klitirqd_thread, | |||
1046 | } | 1046 | } |
1047 | 1047 | ||
1048 | work_flushed |= LIT_TASKLET_LOW; | 1048 | work_flushed |= LIT_TASKLET_LOW; |
1049 | 1049 | ||
1050 | t->owner = NULL; | 1050 | t->owner = NULL; |
1051 | sched_trace_tasklet_end(owner, 1ul); | 1051 | sched_trace_tasklet_end(owner, 1ul); |
1052 | 1052 | ||
@@ -1069,20 +1069,20 @@ void flush_pending(struct task_struct* klitirqd_thread, | |||
1069 | } | 1069 | } |
1070 | } | 1070 | } |
1071 | } | 1071 | } |
1072 | 1072 | ||
1073 | // flush work objects | 1073 | // flush work objects |
1074 | if(litirq_pending_work_irqoff(which)) | 1074 | if(litirq_pending_work_irqoff(which)) |
1075 | { | 1075 | { |
1076 | which->pending &= ~LIT_WORK; | 1076 | which->pending &= ~LIT_WORK; |
1077 | 1077 | ||
1078 | TRACE("%s: Handing work objects back to Linux.\n", __FUNCTION__); | 1078 | TRACE("%s: Handing work objects back to Linux.\n", __FUNCTION__); |
1079 | 1079 | ||
1080 | while(!list_empty(&which->worklist)) | 1080 | while(!list_empty(&which->worklist)) |
1081 | { | 1081 | { |
1082 | struct work_struct* work = | 1082 | struct work_struct* work = |
1083 | list_first_entry(&which->worklist, struct work_struct, entry); | 1083 | list_first_entry(&which->worklist, struct work_struct, entry); |
1084 | list_del_init(&work->entry); | 1084 | list_del_init(&work->entry); |
1085 | 1085 | ||
1086 | if(likely((work->owner == owner) || (owner == NULL))) | 1086 | if(likely((work->owner == owner) || (owner == NULL))) |
1087 | { | 1087 | { |
1088 | work_flushed |= LIT_WORK; | 1088 | work_flushed |= LIT_WORK; |
@@ -1100,9 +1100,9 @@ void flush_pending(struct task_struct* klitirqd_thread, | |||
1100 | } | 1100 | } |
1101 | } | 1101 | } |
1102 | } | 1102 | } |
1103 | 1103 | ||
1104 | //__dump_state(which, "flush_pending: after (before reeval prio)"); | 1104 | //__dump_state(which, "flush_pending: after (before reeval prio)"); |
1105 | 1105 | ||
1106 | 1106 | ||
1107 | mb(); /* commit changes to pending flags */ | 1107 | mb(); /* commit changes to pending flags */ |
1108 | 1108 | ||
@@ -1122,7 +1122,7 @@ void flush_pending(struct task_struct* klitirqd_thread, | |||
1122 | TRACE_CUR("%s: no work flushed, so __reeval_prio() skipped\n", __FUNCTION__); | 1122 | TRACE_CUR("%s: no work flushed, so __reeval_prio() skipped\n", __FUNCTION__); |
1123 | } | 1123 | } |
1124 | 1124 | ||
1125 | raw_spin_unlock_irqrestore(&which->lock, flags); | 1125 | raw_spin_unlock_irqrestore(&which->lock, flags); |
1126 | } | 1126 | } |
1127 | 1127 | ||
1128 | 1128 | ||
@@ -1136,29 +1136,29 @@ static void ___litmus_tasklet_schedule(struct tasklet_struct *t, | |||
1136 | u32 old_pending; | 1136 | u32 old_pending; |
1137 | 1137 | ||
1138 | t->next = NULL; | 1138 | t->next = NULL; |
1139 | 1139 | ||
1140 | raw_spin_lock_irqsave(&which->lock, flags); | 1140 | raw_spin_lock_irqsave(&which->lock, flags); |
1141 | 1141 | ||
1142 | //__dump_state(which, "___litmus_tasklet_schedule: before queuing"); | 1142 | //__dump_state(which, "___litmus_tasklet_schedule: before queuing"); |
1143 | 1143 | ||
1144 | *(which->pending_tasklets.tail) = t; | 1144 | *(which->pending_tasklets.tail) = t; |
1145 | which->pending_tasklets.tail = &t->next; | 1145 | which->pending_tasklets.tail = &t->next; |
1146 | 1146 | ||
1147 | old_pending = which->pending; | 1147 | old_pending = which->pending; |
1148 | which->pending |= LIT_TASKLET_LOW; | 1148 | which->pending |= LIT_TASKLET_LOW; |
1149 | 1149 | ||
1150 | atomic_inc(&which->num_low_pending); | 1150 | atomic_inc(&which->num_low_pending); |
1151 | 1151 | ||
1152 | mb(); | 1152 | mb(); |
1153 | 1153 | ||
1154 | if(!old_pending && wakeup) | 1154 | if(!old_pending && wakeup) |
1155 | { | 1155 | { |
1156 | wakeup_litirqd_locked(which); /* wake up the klitirqd */ | 1156 | wakeup_litirqd_locked(which); /* wake up the klitirqd */ |
1157 | } | 1157 | } |
1158 | 1158 | ||
1159 | //__dump_state(which, "___litmus_tasklet_schedule: after queuing"); | 1159 | //__dump_state(which, "___litmus_tasklet_schedule: after queuing"); |
1160 | 1160 | ||
1161 | raw_spin_unlock_irqrestore(&which->lock, flags); | 1161 | raw_spin_unlock_irqrestore(&which->lock, flags); |
1162 | } | 1162 | } |
1163 | 1163 | ||
1164 | int __litmus_tasklet_schedule(struct tasklet_struct *t, unsigned int k_id) | 1164 | int __litmus_tasklet_schedule(struct tasklet_struct *t, unsigned int k_id) |
@@ -1173,7 +1173,7 @@ int __litmus_tasklet_schedule(struct tasklet_struct *t, unsigned int k_id) | |||
1173 | if(unlikely(k_id >= NR_LITMUS_SOFTIRQD)) | 1173 | if(unlikely(k_id >= NR_LITMUS_SOFTIRQD)) |
1174 | { | 1174 | { |
1175 | TRACE("%s: No klitirqd_th%d!\n", __FUNCTION__, k_id); | 1175 | TRACE("%s: No klitirqd_th%d!\n", __FUNCTION__, k_id); |
1176 | BUG(); | 1176 | BUG(); |
1177 | } | 1177 | } |
1178 | 1178 | ||
1179 | if(likely(!klitirqds[k_id].terminating)) | 1179 | if(likely(!klitirqds[k_id].terminating)) |
@@ -1209,25 +1209,25 @@ static void ___litmus_tasklet_hi_schedule(struct tasklet_struct *t, | |||
1209 | u32 old_pending; | 1209 | u32 old_pending; |
1210 | 1210 | ||
1211 | t->next = NULL; | 1211 | t->next = NULL; |
1212 | 1212 | ||
1213 | raw_spin_lock_irqsave(&which->lock, flags); | 1213 | raw_spin_lock_irqsave(&which->lock, flags); |
1214 | 1214 | ||
1215 | *(which->pending_tasklets_hi.tail) = t; | 1215 | *(which->pending_tasklets_hi.tail) = t; |
1216 | which->pending_tasklets_hi.tail = &t->next; | 1216 | which->pending_tasklets_hi.tail = &t->next; |
1217 | 1217 | ||
1218 | old_pending = which->pending; | 1218 | old_pending = which->pending; |
1219 | which->pending |= LIT_TASKLET_HI; | 1219 | which->pending |= LIT_TASKLET_HI; |
1220 | 1220 | ||
1221 | atomic_inc(&which->num_hi_pending); | 1221 | atomic_inc(&which->num_hi_pending); |
1222 | 1222 | ||
1223 | mb(); | 1223 | mb(); |
1224 | 1224 | ||
1225 | if(!old_pending && wakeup) | 1225 | if(!old_pending && wakeup) |
1226 | { | 1226 | { |
1227 | wakeup_litirqd_locked(which); /* wake up the klitirqd */ | 1227 | wakeup_litirqd_locked(which); /* wake up the klitirqd */ |
1228 | } | 1228 | } |
1229 | 1229 | ||
1230 | raw_spin_unlock_irqrestore(&which->lock, flags); | 1230 | raw_spin_unlock_irqrestore(&which->lock, flags); |
1231 | } | 1231 | } |
1232 | 1232 | ||
1233 | int __litmus_tasklet_hi_schedule(struct tasklet_struct *t, unsigned int k_id) | 1233 | int __litmus_tasklet_hi_schedule(struct tasklet_struct *t, unsigned int k_id) |
@@ -1238,19 +1238,19 @@ int __litmus_tasklet_hi_schedule(struct tasklet_struct *t, unsigned int k_id) | |||
1238 | TRACE("%s: No owner associated with this tasklet!\n", __FUNCTION__); | 1238 | TRACE("%s: No owner associated with this tasklet!\n", __FUNCTION__); |
1239 | BUG(); | 1239 | BUG(); |
1240 | } | 1240 | } |
1241 | 1241 | ||
1242 | if(unlikely(k_id >= NR_LITMUS_SOFTIRQD)) | 1242 | if(unlikely(k_id >= NR_LITMUS_SOFTIRQD)) |
1243 | { | 1243 | { |
1244 | TRACE("%s: No klitirqd_th%d!\n", __FUNCTION__, k_id); | 1244 | TRACE("%s: No klitirqd_th%d!\n", __FUNCTION__, k_id); |
1245 | BUG(); | 1245 | BUG(); |
1246 | } | 1246 | } |
1247 | 1247 | ||
1248 | if(unlikely(!klitirqd_is_ready())) | 1248 | if(unlikely(!klitirqd_is_ready())) |
1249 | { | 1249 | { |
1250 | TRACE("%s: klitirqd is not ready!\n", __FUNCTION__, k_id); | 1250 | TRACE("%s: klitirqd is not ready!\n", __FUNCTION__, k_id); |
1251 | BUG(); | 1251 | BUG(); |
1252 | } | 1252 | } |
1253 | 1253 | ||
1254 | if(likely(!klitirqds[k_id].terminating)) | 1254 | if(likely(!klitirqds[k_id].terminating)) |
1255 | { | 1255 | { |
1256 | if(likely(atomic_read(&klitirqds[k_id].num_work_pending) == 0)) | 1256 | if(likely(atomic_read(&klitirqds[k_id].num_work_pending) == 0)) |
@@ -1276,41 +1276,41 @@ int __litmus_tasklet_hi_schedule_first(struct tasklet_struct *t, unsigned int k_ | |||
1276 | u32 old_pending; | 1276 | u32 old_pending; |
1277 | 1277 | ||
1278 | BUG_ON(!irqs_disabled()); | 1278 | BUG_ON(!irqs_disabled()); |
1279 | 1279 | ||
1280 | if(unlikely((t->owner == NULL) || !is_realtime(t->owner))) | 1280 | if(unlikely((t->owner == NULL) || !is_realtime(t->owner))) |
1281 | { | 1281 | { |
1282 | TRACE("%s: No owner associated with this tasklet!\n", __FUNCTION__); | 1282 | TRACE("%s: No owner associated with this tasklet!\n", __FUNCTION__); |
1283 | BUG(); | 1283 | BUG(); |
1284 | } | 1284 | } |
1285 | 1285 | ||
1286 | if(unlikely(k_id >= NR_LITMUS_SOFTIRQD)) | 1286 | if(unlikely(k_id >= NR_LITMUS_SOFTIRQD)) |
1287 | { | 1287 | { |
1288 | TRACE("%s: No klitirqd_th%u!\n", __FUNCTION__, k_id); | 1288 | TRACE("%s: No klitirqd_th%u!\n", __FUNCTION__, k_id); |
1289 | BUG(); | 1289 | BUG(); |
1290 | } | 1290 | } |
1291 | 1291 | ||
1292 | if(unlikely(!klitirqd_is_ready())) | 1292 | if(unlikely(!klitirqd_is_ready())) |
1293 | { | 1293 | { |
1294 | TRACE("%s: klitirqd is not ready!\n", __FUNCTION__, k_id); | 1294 | TRACE("%s: klitirqd is not ready!\n", __FUNCTION__, k_id); |
1295 | BUG(); | 1295 | BUG(); |
1296 | } | 1296 | } |
1297 | 1297 | ||
1298 | if(likely(!klitirqds[k_id].terminating)) | 1298 | if(likely(!klitirqds[k_id].terminating)) |
1299 | { | 1299 | { |
1300 | raw_spin_lock(&klitirqds[k_id].lock); | 1300 | raw_spin_lock(&klitirqds[k_id].lock); |
1301 | 1301 | ||
1302 | if(likely(atomic_read(&klitirqds[k_id].num_work_pending) == 0)) | 1302 | if(likely(atomic_read(&klitirqds[k_id].num_work_pending) == 0)) |
1303 | { | 1303 | { |
1304 | ret = 1; // success! | 1304 | ret = 1; // success! |
1305 | 1305 | ||
1306 | t->next = klitirqds[k_id].pending_tasklets_hi.head; | 1306 | t->next = klitirqds[k_id].pending_tasklets_hi.head; |
1307 | klitirqds[k_id].pending_tasklets_hi.head = t; | 1307 | klitirqds[k_id].pending_tasklets_hi.head = t; |
1308 | 1308 | ||
1309 | old_pending = klitirqds[k_id].pending; | 1309 | old_pending = klitirqds[k_id].pending; |
1310 | klitirqds[k_id].pending |= LIT_TASKLET_HI; | 1310 | klitirqds[k_id].pending |= LIT_TASKLET_HI; |
1311 | 1311 | ||
1312 | atomic_inc(&klitirqds[k_id].num_hi_pending); | 1312 | atomic_inc(&klitirqds[k_id].num_hi_pending); |
1313 | 1313 | ||
1314 | mb(); | 1314 | mb(); |
1315 | 1315 | ||
1316 | if(!old_pending) | 1316 | if(!old_pending) |
@@ -1339,22 +1339,22 @@ static void ___litmus_schedule_work(struct work_struct *w, | |||
1339 | u32 old_pending; | 1339 | u32 old_pending; |
1340 | 1340 | ||
1341 | raw_spin_lock_irqsave(&which->lock, flags); | 1341 | raw_spin_lock_irqsave(&which->lock, flags); |
1342 | 1342 | ||
1343 | work_pending(w); | 1343 | work_pending(w); |
1344 | list_add_tail(&w->entry, &which->worklist); | 1344 | list_add_tail(&w->entry, &which->worklist); |
1345 | 1345 | ||
1346 | old_pending = which->pending; | 1346 | old_pending = which->pending; |
1347 | which->pending |= LIT_WORK; | 1347 | which->pending |= LIT_WORK; |
1348 | 1348 | ||
1349 | atomic_inc(&which->num_work_pending); | 1349 | atomic_inc(&which->num_work_pending); |
1350 | 1350 | ||
1351 | mb(); | 1351 | mb(); |
1352 | 1352 | ||
1353 | if(!old_pending && wakeup) | 1353 | if(!old_pending && wakeup) |
1354 | { | 1354 | { |
1355 | wakeup_litirqd_locked(which); /* wakeup the klitirqd */ | 1355 | wakeup_litirqd_locked(which); /* wakeup the klitirqd */ |
1356 | } | 1356 | } |
1357 | 1357 | ||
1358 | raw_spin_unlock_irqrestore(&which->lock, flags); | 1358 | raw_spin_unlock_irqrestore(&which->lock, flags); |
1359 | } | 1359 | } |
1360 | 1360 | ||
@@ -1366,18 +1366,18 @@ int __litmus_schedule_work(struct work_struct *w, unsigned int k_id) | |||
1366 | TRACE("%s: No owner associated with this work object!\n", __FUNCTION__); | 1366 | TRACE("%s: No owner associated with this work object!\n", __FUNCTION__); |
1367 | BUG(); | 1367 | BUG(); |
1368 | } | 1368 | } |
1369 | 1369 | ||
1370 | if(unlikely(k_id >= NR_LITMUS_SOFTIRQD)) | 1370 | if(unlikely(k_id >= NR_LITMUS_SOFTIRQD)) |
1371 | { | 1371 | { |
1372 | TRACE("%s: No klitirqd_th%u!\n", k_id); | 1372 | TRACE("%s: No klitirqd_th%u!\n", k_id); |
1373 | BUG(); | 1373 | BUG(); |
1374 | } | 1374 | } |
1375 | 1375 | ||
1376 | if(unlikely(!klitirqd_is_ready())) | 1376 | if(unlikely(!klitirqd_is_ready())) |
1377 | { | 1377 | { |
1378 | TRACE("%s: klitirqd is not ready!\n", __FUNCTION__, k_id); | 1378 | TRACE("%s: klitirqd is not ready!\n", __FUNCTION__, k_id); |
1379 | BUG(); | 1379 | BUG(); |
1380 | } | 1380 | } |
1381 | 1381 | ||
1382 | if(likely(!klitirqds[k_id].terminating)) | 1382 | if(likely(!klitirqds[k_id].terminating)) |
1383 | ___litmus_schedule_work(w, &klitirqds[k_id], 1); | 1383 | ___litmus_schedule_work(w, &klitirqds[k_id], 1); |
@@ -1485,13 +1485,13 @@ void release_klitirqd_lock(struct task_struct* t) | |||
1485 | { | 1485 | { |
1486 | struct mutex* sem; | 1486 | struct mutex* sem; |
1487 | struct task_struct* owner = t; | 1487 | struct task_struct* owner = t; |
1488 | 1488 | ||
1489 | if(t->state == TASK_RUNNING) | 1489 | if(t->state == TASK_RUNNING) |
1490 | { | 1490 | { |
1491 | TRACE_TASK(t, "NOT giving up klitirqd_sem because we're not blocked!\n"); | 1491 | TRACE_TASK(t, "NOT giving up klitirqd_sem because we're not blocked!\n"); |
1492 | return; | 1492 | return; |
1493 | } | 1493 | } |
1494 | 1494 | ||
1495 | if(likely(!tsk_rt(t)->is_proxy_thread)) | 1495 | if(likely(!tsk_rt(t)->is_proxy_thread)) |
1496 | { | 1496 | { |
1497 | sem = &tsk_rt(t)->klitirqd_sem; | 1497 | sem = &tsk_rt(t)->klitirqd_sem; |
@@ -1510,7 +1510,7 @@ void release_klitirqd_lock(struct task_struct* t) | |||
1510 | else | 1510 | else |
1511 | { | 1511 | { |
1512 | BUG(); | 1512 | BUG(); |
1513 | 1513 | ||
1514 | // We had the rug pulled out from under us. Abort attempt | 1514 | // We had the rug pulled out from under us. Abort attempt |
1515 | // to reacquire the lock since our client no longer needs us. | 1515 | // to reacquire the lock since our client no longer needs us. |
1516 | TRACE_CUR("HUH?! How did this happen?\n"); | 1516 | TRACE_CUR("HUH?! How did this happen?\n"); |
@@ -1518,7 +1518,7 @@ void release_klitirqd_lock(struct task_struct* t) | |||
1518 | return; | 1518 | return; |
1519 | } | 1519 | } |
1520 | } | 1520 | } |
1521 | 1521 | ||
1522 | //TRACE_CUR("Releasing semaphore of %s/%d...\n", owner->comm, owner->pid); | 1522 | //TRACE_CUR("Releasing semaphore of %s/%d...\n", owner->comm, owner->pid); |
1523 | up_and_set_stat(t, NEED_TO_REACQUIRE, sem); | 1523 | up_and_set_stat(t, NEED_TO_REACQUIRE, sem); |
1524 | //TRACE_CUR("Semaphore of %s/%d released!\n", owner->comm, owner->pid); | 1524 | //TRACE_CUR("Semaphore of %s/%d released!\n", owner->comm, owner->pid); |
@@ -1539,17 +1539,17 @@ int reacquire_klitirqd_lock(struct task_struct* t) | |||
1539 | { | 1539 | { |
1540 | struct mutex* sem; | 1540 | struct mutex* sem; |
1541 | struct task_struct* owner = t; | 1541 | struct task_struct* owner = t; |
1542 | 1542 | ||
1543 | if(likely(!tsk_rt(t)->is_proxy_thread)) | 1543 | if(likely(!tsk_rt(t)->is_proxy_thread)) |
1544 | { | 1544 | { |
1545 | sem = &tsk_rt(t)->klitirqd_sem; | 1545 | sem = &tsk_rt(t)->klitirqd_sem; |
1546 | } | 1546 | } |
1547 | else | 1547 | else |
1548 | { | 1548 | { |
1549 | unsigned int k_id = klitirqd_id(t); | 1549 | unsigned int k_id = klitirqd_id(t); |
1550 | //struct task_struct* owner = klitirqds[k_id].current_owner; | 1550 | //struct task_struct* owner = klitirqds[k_id].current_owner; |
1551 | owner = klitirqds[k_id].current_owner; | 1551 | owner = klitirqds[k_id].current_owner; |
1552 | 1552 | ||
1553 | BUG_ON(t != klitirqds[k_id].klitirqd); | 1553 | BUG_ON(t != klitirqds[k_id].klitirqd); |
1554 | 1554 | ||
1555 | if(likely(owner)) | 1555 | if(likely(owner)) |
@@ -1565,7 +1565,7 @@ int reacquire_klitirqd_lock(struct task_struct* t) | |||
1565 | return(0); | 1565 | return(0); |
1566 | } | 1566 | } |
1567 | } | 1567 | } |
1568 | 1568 | ||
1569 | //TRACE_CUR("Trying to reacquire semaphore of %s/%d\n", owner->comm, owner->pid); | 1569 | //TRACE_CUR("Trying to reacquire semaphore of %s/%d\n", owner->comm, owner->pid); |
1570 | __down_and_reset_and_set_stat(t, REACQUIRING, HELD, sem); | 1570 | __down_and_reset_and_set_stat(t, REACQUIRING, HELD, sem); |
1571 | //TRACE_CUR("Reacquired semaphore %s/%d\n", owner->comm, owner->pid); | 1571 | //TRACE_CUR("Reacquired semaphore %s/%d\n", owner->comm, owner->pid); |
diff --git a/litmus/locking.c b/litmus/locking.c index fd3c7260319f..e754b2fa2634 100644 --- a/litmus/locking.c +++ b/litmus/locking.c | |||
@@ -57,7 +57,7 @@ static int create_generic_lock(void** obj_ref, obj_type_t type, void* __user ar | |||
57 | "most uses. (exception: IKGLP donors)\n"); | 57 | "most uses. (exception: IKGLP donors)\n"); |
58 | } | 58 | } |
59 | #endif | 59 | #endif |
60 | lock->type = type; | 60 | lock->type = type; |
61 | lock->ident = atomic_inc_return(&lock_id_gen); | 61 | lock->ident = atomic_inc_return(&lock_id_gen); |
62 | *obj_ref = lock; | 62 | *obj_ref = lock; |
63 | } | 63 | } |
@@ -300,14 +300,14 @@ static long do_litmus_dgl_lock(dgl_wait_state_t *dgl_wait) | |||
300 | 300 | ||
301 | TRACE_CUR("As many as %d locks in DGL are pending. Suspending.\n", | 301 | TRACE_CUR("As many as %d locks in DGL are pending. Suspending.\n", |
302 | dgl_wait->nr_remaining); | 302 | dgl_wait->nr_remaining); |
303 | 303 | ||
304 | #if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA) | 304 | #if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA) |
305 | // KLUDGE: don't count this suspension as time in the critical gpu | 305 | // KLUDGE: don't count this suspension as time in the critical gpu |
306 | // critical section | 306 | // critical section |
307 | if(tsk_rt(dgl_wait->task)->held_gpus) { | 307 | if(tsk_rt(dgl_wait->task)->held_gpus) { |
308 | tsk_rt(dgl_wait->task)->suspend_gpu_tracker_on_block = 1; | 308 | tsk_rt(dgl_wait->task)->suspend_gpu_tracker_on_block = 1; |
309 | } | 309 | } |
310 | #endif | 310 | #endif |
311 | 311 | ||
312 | // note reverse order. see comments in select_next_lock for reason. | 312 | // note reverse order. see comments in select_next_lock for reason. |
313 | for(i = dgl_wait->size - 1; i >= 0; --i) { | 313 | for(i = dgl_wait->size - 1; i >= 0; --i) { |
diff --git a/litmus/nvidia_info.c b/litmus/nvidia_info.c index fd6398121fbf..889313c854e6 100644 --- a/litmus/nvidia_info.c +++ b/litmus/nvidia_info.c | |||
@@ -41,10 +41,10 @@ typedef struct | |||
41 | { | 41 | { |
42 | void *priv; /* private data */ | 42 | void *priv; /* private data */ |
43 | void *os_state; /* os-specific device state */ | 43 | void *os_state; /* os-specific device state */ |
44 | 44 | ||
45 | int rmInitialized; | 45 | int rmInitialized; |
46 | int flags; | 46 | int flags; |
47 | 47 | ||
48 | /* PCI config info */ | 48 | /* PCI config info */ |
49 | NvU32 domain; | 49 | NvU32 domain; |
50 | NvU16 bus; | 50 | NvU16 bus; |
@@ -54,31 +54,31 @@ typedef struct | |||
54 | NvU16 subsystem_id; | 54 | NvU16 subsystem_id; |
55 | NvU32 gpu_id; | 55 | NvU32 gpu_id; |
56 | void *handle; | 56 | void *handle; |
57 | 57 | ||
58 | NvU32 pci_cfg_space[16]; | 58 | NvU32 pci_cfg_space[16]; |
59 | 59 | ||
60 | /* physical characteristics */ | 60 | /* physical characteristics */ |
61 | litmus_nv_aperture_t bars[3]; | 61 | litmus_nv_aperture_t bars[3]; |
62 | litmus_nv_aperture_t *regs; | 62 | litmus_nv_aperture_t *regs; |
63 | litmus_nv_aperture_t *fb, ud; | 63 | litmus_nv_aperture_t *fb, ud; |
64 | litmus_nv_aperture_t agp; | 64 | litmus_nv_aperture_t agp; |
65 | 65 | ||
66 | NvU32 interrupt_line; | 66 | NvU32 interrupt_line; |
67 | 67 | ||
68 | NvU32 agp_config; | 68 | NvU32 agp_config; |
69 | NvU32 agp_status; | 69 | NvU32 agp_status; |
70 | 70 | ||
71 | NvU32 primary_vga; | 71 | NvU32 primary_vga; |
72 | 72 | ||
73 | NvU32 sim_env; | 73 | NvU32 sim_env; |
74 | 74 | ||
75 | NvU32 rc_timer_enabled; | 75 | NvU32 rc_timer_enabled; |
76 | 76 | ||
77 | /* list of events allocated for this device */ | 77 | /* list of events allocated for this device */ |
78 | void *event_list; | 78 | void *event_list; |
79 | 79 | ||
80 | void *kern_mappings; | 80 | void *kern_mappings; |
81 | 81 | ||
82 | } litmus_nv_state_t; | 82 | } litmus_nv_state_t; |
83 | 83 | ||
84 | typedef struct work_struct litmus_nv_task_t; | 84 | typedef struct work_struct litmus_nv_task_t; |
@@ -91,11 +91,11 @@ typedef struct litmus_nv_work_s { | |||
91 | typedef struct litmus_nv_linux_state_s { | 91 | typedef struct litmus_nv_linux_state_s { |
92 | litmus_nv_state_t nv_state; | 92 | litmus_nv_state_t nv_state; |
93 | atomic_t usage_count; | 93 | atomic_t usage_count; |
94 | 94 | ||
95 | struct pci_dev *dev; | 95 | struct pci_dev *dev; |
96 | void *agp_bridge; | 96 | void *agp_bridge; |
97 | void *alloc_queue; | 97 | void *alloc_queue; |
98 | 98 | ||
99 | void *timer_sp; | 99 | void *timer_sp; |
100 | void *isr_sp; | 100 | void *isr_sp; |
101 | void *pci_cfgchk_sp; | 101 | void *pci_cfgchk_sp; |
@@ -108,29 +108,29 @@ typedef struct litmus_nv_linux_state_s { | |||
108 | /* keep track of any pending bottom halfes */ | 108 | /* keep track of any pending bottom halfes */ |
109 | struct tasklet_struct tasklet; | 109 | struct tasklet_struct tasklet; |
110 | litmus_nv_work_t work; | 110 | litmus_nv_work_t work; |
111 | 111 | ||
112 | /* get a timer callback every second */ | 112 | /* get a timer callback every second */ |
113 | struct timer_list rc_timer; | 113 | struct timer_list rc_timer; |
114 | 114 | ||
115 | /* lock for linux-specific data, not used by core rm */ | 115 | /* lock for linux-specific data, not used by core rm */ |
116 | struct semaphore ldata_lock; | 116 | struct semaphore ldata_lock; |
117 | 117 | ||
118 | /* lock for linux-specific alloc queue */ | 118 | /* lock for linux-specific alloc queue */ |
119 | struct semaphore at_lock; | 119 | struct semaphore at_lock; |
120 | 120 | ||
121 | #if 0 | 121 | #if 0 |
122 | #if defined(NV_USER_MAP) | 122 | #if defined(NV_USER_MAP) |
123 | /* list of user mappings */ | 123 | /* list of user mappings */ |
124 | struct nv_usermap_s *usermap_list; | 124 | struct nv_usermap_s *usermap_list; |
125 | 125 | ||
126 | /* lock for VMware-specific mapping list */ | 126 | /* lock for VMware-specific mapping list */ |
127 | struct semaphore mt_lock; | 127 | struct semaphore mt_lock; |
128 | #endif /* defined(NV_USER_MAP) */ | 128 | #endif /* defined(NV_USER_MAP) */ |
129 | #if defined(NV_PM_SUPPORT_OLD_STYLE_APM) | 129 | #if defined(NV_PM_SUPPORT_OLD_STYLE_APM) |
130 | void *apm_nv_dev; | 130 | void *apm_nv_dev; |
131 | #endif | 131 | #endif |
132 | #endif | 132 | #endif |
133 | 133 | ||
134 | NvU32 device_num; | 134 | NvU32 device_num; |
135 | struct litmus_nv_linux_state_s *next; | 135 | struct litmus_nv_linux_state_s *next; |
136 | } litmus_nv_linux_state_t; | 136 | } litmus_nv_linux_state_t; |
@@ -140,9 +140,9 @@ void dump_nvidia_info(const struct tasklet_struct *t) | |||
140 | litmus_nv_state_t* nvstate = NULL; | 140 | litmus_nv_state_t* nvstate = NULL; |
141 | litmus_nv_linux_state_t* linuxstate = NULL; | 141 | litmus_nv_linux_state_t* linuxstate = NULL; |
142 | struct pci_dev* pci = NULL; | 142 | struct pci_dev* pci = NULL; |
143 | 143 | ||
144 | nvstate = (litmus_nv_state_t*)(t->data); | 144 | nvstate = (litmus_nv_state_t*)(t->data); |
145 | 145 | ||
146 | if(nvstate) | 146 | if(nvstate) |
147 | { | 147 | { |
148 | TRACE("NV State:\n" | 148 | TRACE("NV State:\n" |
@@ -170,21 +170,21 @@ void dump_nvidia_info(const struct tasklet_struct *t) | |||
170 | nvstate->subsystem_id, | 170 | nvstate->subsystem_id, |
171 | nvstate->gpu_id, | 171 | nvstate->gpu_id, |
172 | nvstate->interrupt_line); | 172 | nvstate->interrupt_line); |
173 | 173 | ||
174 | linuxstate = container_of(nvstate, litmus_nv_linux_state_t, nv_state); | 174 | linuxstate = container_of(nvstate, litmus_nv_linux_state_t, nv_state); |
175 | } | 175 | } |
176 | else | 176 | else |
177 | { | 177 | { |
178 | TRACE("INVALID NVSTATE????\n"); | 178 | TRACE("INVALID NVSTATE????\n"); |
179 | } | 179 | } |
180 | 180 | ||
181 | if(linuxstate) | 181 | if(linuxstate) |
182 | { | 182 | { |
183 | int ls_offset = (void*)(&(linuxstate->device_num)) - (void*)(linuxstate); | 183 | int ls_offset = (void*)(&(linuxstate->device_num)) - (void*)(linuxstate); |
184 | int ns_offset_raw = (void*)(&(linuxstate->device_num)) - (void*)(&(linuxstate->nv_state)); | 184 | int ns_offset_raw = (void*)(&(linuxstate->device_num)) - (void*)(&(linuxstate->nv_state)); |
185 | int ns_offset_desired = (void*)(&(linuxstate->device_num)) - (void*)(nvstate); | 185 | int ns_offset_desired = (void*)(&(linuxstate->device_num)) - (void*)(nvstate); |
186 | 186 | ||
187 | 187 | ||
188 | TRACE("LINUX NV State:\n" | 188 | TRACE("LINUX NV State:\n" |
189 | "\tlinux nv state ptr: %p\n" | 189 | "\tlinux nv state ptr: %p\n" |
190 | "\taddress of tasklet: %p\n" | 190 | "\taddress of tasklet: %p\n" |
@@ -200,9 +200,9 @@ void dump_nvidia_info(const struct tasklet_struct *t) | |||
200 | linuxstate->device_num, | 200 | linuxstate->device_num, |
201 | (t == &(linuxstate->tasklet)), | 201 | (t == &(linuxstate->tasklet)), |
202 | linuxstate->dev); | 202 | linuxstate->dev); |
203 | 203 | ||
204 | pci = linuxstate->dev; | 204 | pci = linuxstate->dev; |
205 | 205 | ||
206 | TRACE("Offsets:\n" | 206 | TRACE("Offsets:\n" |
207 | "\tOffset from LinuxState: %d, %x\n" | 207 | "\tOffset from LinuxState: %d, %x\n" |
208 | "\tOffset from NVState: %d, %x\n" | 208 | "\tOffset from NVState: %d, %x\n" |
@@ -249,7 +249,7 @@ int init_nvidia_info(void) | |||
249 | { | 249 | { |
250 | mutex_lock(&module_mutex); | 250 | mutex_lock(&module_mutex); |
251 | nvidia_mod = find_module("nvidia"); | 251 | nvidia_mod = find_module("nvidia"); |
252 | mutex_unlock(&module_mutex); | 252 | mutex_unlock(&module_mutex); |
253 | if(nvidia_mod != NULL) | 253 | if(nvidia_mod != NULL) |
254 | { | 254 | { |
255 | TRACE("%s : Found NVIDIA module. Core Code: %p to %p\n", __FUNCTION__, | 255 | TRACE("%s : Found NVIDIA module. Core Code: %p to %p\n", __FUNCTION__, |
@@ -280,7 +280,7 @@ int is_nvidia_func(void* func_addr) | |||
280 | __FUNCTION__, func_addr, ret); | 280 | __FUNCTION__, func_addr, ret); |
281 | }*/ | 281 | }*/ |
282 | } | 282 | } |
283 | 283 | ||
284 | return(ret); | 284 | return(ret); |
285 | } | 285 | } |
286 | 286 | ||
@@ -303,7 +303,7 @@ u32 get_tasklet_nv_device_num(const struct tasklet_struct *t) | |||
303 | 303 | ||
304 | void* state = (void*)(t->data); | 304 | void* state = (void*)(t->data); |
305 | void* device_num_ptr = state + DEVICE_NUM_OFFSET; | 305 | void* device_num_ptr = state + DEVICE_NUM_OFFSET; |
306 | 306 | ||
307 | //dump_nvidia_info(t); | 307 | //dump_nvidia_info(t); |
308 | return(*((u32*)device_num_ptr)); | 308 | return(*((u32*)device_num_ptr)); |
309 | #endif | 309 | #endif |
@@ -334,14 +334,14 @@ static nv_device_registry_t NV_DEVICE_REG[NV_DEVICE_NUM]; | |||
334 | int init_nv_device_reg(void) | 334 | int init_nv_device_reg(void) |
335 | { | 335 | { |
336 | int i; | 336 | int i; |
337 | 337 | ||
338 | memset(NV_DEVICE_REG, 0, sizeof(NV_DEVICE_REG)); | 338 | memset(NV_DEVICE_REG, 0, sizeof(NV_DEVICE_REG)); |
339 | 339 | ||
340 | for(i = 0; i < NV_DEVICE_NUM; ++i) | 340 | for(i = 0; i < NV_DEVICE_NUM; ++i) |
341 | { | 341 | { |
342 | raw_spin_lock_init(&NV_DEVICE_REG[i].lock); | 342 | raw_spin_lock_init(&NV_DEVICE_REG[i].lock); |
343 | } | 343 | } |
344 | 344 | ||
345 | return(1); | 345 | return(1); |
346 | } | 346 | } |
347 | 347 | ||
@@ -360,7 +360,7 @@ int get_nv_device_id(struct task_struct* owner) | |||
360 | if(NV_DEVICE_REG[i].device_owner == owner) | 360 | if(NV_DEVICE_REG[i].device_owner == owner) |
361 | return(i); | 361 | return(i); |
362 | } | 362 | } |
363 | return(-1); | 363 | return(-1); |
364 | } | 364 | } |
365 | */ | 365 | */ |
366 | 366 | ||
@@ -380,42 +380,42 @@ void pai_check_priority_increase(struct task_struct *t, int reg_device_id) | |||
380 | { | 380 | { |
381 | unsigned long flags; | 381 | unsigned long flags; |
382 | nv_device_registry_t *reg = &NV_DEVICE_REG[reg_device_id]; | 382 | nv_device_registry_t *reg = &NV_DEVICE_REG[reg_device_id]; |
383 | 383 | ||
384 | if(reg->max_prio_owner != t) { | 384 | if(reg->max_prio_owner != t) { |
385 | 385 | ||
386 | raw_spin_lock_irqsave(®->lock, flags); | 386 | raw_spin_lock_irqsave(®->lock, flags); |
387 | 387 | ||
388 | if(reg->max_prio_owner != t) { | 388 | if(reg->max_prio_owner != t) { |
389 | if(litmus->compare(t, reg->max_prio_owner)) { | 389 | if(litmus->compare(t, reg->max_prio_owner)) { |
390 | litmus->change_prio_pai_tasklet(reg->max_prio_owner, t); | 390 | litmus->change_prio_pai_tasklet(reg->max_prio_owner, t); |
391 | reg->max_prio_owner = t; | 391 | reg->max_prio_owner = t; |
392 | } | 392 | } |
393 | } | 393 | } |
394 | 394 | ||
395 | raw_spin_unlock_irqrestore(®->lock, flags); | 395 | raw_spin_unlock_irqrestore(®->lock, flags); |
396 | } | 396 | } |
397 | } | 397 | } |
398 | 398 | ||
399 | 399 | ||
400 | void pai_check_priority_decrease(struct task_struct *t, int reg_device_id) | 400 | void pai_check_priority_decrease(struct task_struct *t, int reg_device_id) |
401 | { | 401 | { |
402 | unsigned long flags; | 402 | unsigned long flags; |
403 | nv_device_registry_t *reg = &NV_DEVICE_REG[reg_device_id]; | 403 | nv_device_registry_t *reg = &NV_DEVICE_REG[reg_device_id]; |
404 | 404 | ||
405 | if(reg->max_prio_owner == t) { | 405 | if(reg->max_prio_owner == t) { |
406 | 406 | ||
407 | raw_spin_lock_irqsave(®->lock, flags); | 407 | raw_spin_lock_irqsave(®->lock, flags); |
408 | 408 | ||
409 | if(reg->max_prio_owner == t) { | 409 | if(reg->max_prio_owner == t) { |
410 | reg->max_prio_owner = find_hp_owner(reg, NULL); | 410 | reg->max_prio_owner = find_hp_owner(reg, NULL); |
411 | if(reg->max_prio_owner != t) { | 411 | if(reg->max_prio_owner != t) { |
412 | litmus->change_prio_pai_tasklet(t, reg->max_prio_owner); | 412 | litmus->change_prio_pai_tasklet(t, reg->max_prio_owner); |
413 | } | 413 | } |
414 | } | 414 | } |
415 | 415 | ||
416 | raw_spin_unlock_irqrestore(®->lock, flags); | 416 | raw_spin_unlock_irqrestore(®->lock, flags); |
417 | } | 417 | } |
418 | } | 418 | } |
419 | #endif | 419 | #endif |
420 | 420 | ||
421 | static int __reg_nv_device(int reg_device_id, struct task_struct *t) | 421 | static int __reg_nv_device(int reg_device_id, struct task_struct *t) |
@@ -430,30 +430,30 @@ static int __reg_nv_device(int reg_device_id, struct task_struct *t) | |||
430 | // TODO: check if taks is already registered. | 430 | // TODO: check if taks is already registered. |
431 | return ret; // assume already registered. | 431 | return ret; // assume already registered. |
432 | } | 432 | } |
433 | 433 | ||
434 | raw_spin_lock_irqsave(®->lock, flags); | 434 | raw_spin_lock_irqsave(®->lock, flags); |
435 | 435 | ||
436 | if(reg->nr_owners < NV_MAX_SIMULT_USERS) { | 436 | if(reg->nr_owners < NV_MAX_SIMULT_USERS) { |
437 | TRACE_TASK(t, "registers GPU %d\n", reg_device_id); | 437 | TRACE_TASK(t, "registers GPU %d\n", reg_device_id); |
438 | for(i = 0; i < NV_MAX_SIMULT_USERS; ++i) { | 438 | for(i = 0; i < NV_MAX_SIMULT_USERS; ++i) { |
439 | if(reg->owners[i] == NULL) { | 439 | if(reg->owners[i] == NULL) { |
440 | reg->owners[i] = t; | 440 | reg->owners[i] = t; |
441 | 441 | ||
442 | //if(edf_higher_prio(t, reg->max_prio_owner)) { | 442 | //if(edf_higher_prio(t, reg->max_prio_owner)) { |
443 | if(litmus->compare(t, reg->max_prio_owner)) { | 443 | if(litmus->compare(t, reg->max_prio_owner)) { |
444 | old_max = reg->max_prio_owner; | 444 | old_max = reg->max_prio_owner; |
445 | reg->max_prio_owner = t; | 445 | reg->max_prio_owner = t; |
446 | 446 | ||
447 | #ifdef CONFIG_LITMUS_PAI_SOFTIRQD | 447 | #ifdef CONFIG_LITMUS_PAI_SOFTIRQD |
448 | litmus->change_prio_pai_tasklet(old_max, t); | 448 | litmus->change_prio_pai_tasklet(old_max, t); |
449 | #endif | 449 | #endif |
450 | } | 450 | } |
451 | 451 | ||
452 | #ifdef CONFIG_LITMUS_SOFTIRQD | 452 | #ifdef CONFIG_LITMUS_SOFTIRQD |
453 | down_and_set_stat(t, HELD, &tsk_rt(t)->klitirqd_sem); | 453 | down_and_set_stat(t, HELD, &tsk_rt(t)->klitirqd_sem); |
454 | #endif | 454 | #endif |
455 | ++(reg->nr_owners); | 455 | ++(reg->nr_owners); |
456 | 456 | ||
457 | break; | 457 | break; |
458 | } | 458 | } |
459 | } | 459 | } |
@@ -461,13 +461,13 @@ static int __reg_nv_device(int reg_device_id, struct task_struct *t) | |||
461 | else | 461 | else |
462 | { | 462 | { |
463 | TRACE_CUR("%s: device %d is already in use!\n", __FUNCTION__, reg_device_id); | 463 | TRACE_CUR("%s: device %d is already in use!\n", __FUNCTION__, reg_device_id); |
464 | ret = -EBUSY; | 464 | ret = -EBUSY; |
465 | } | 465 | } |
466 | 466 | ||
467 | raw_spin_unlock_irqrestore(®->lock, flags); | 467 | raw_spin_unlock_irqrestore(®->lock, flags); |
468 | 468 | ||
469 | __set_bit(reg_device_id, &tsk_rt(t)->held_gpus); | 469 | __set_bit(reg_device_id, &tsk_rt(t)->held_gpus); |
470 | 470 | ||
471 | return(ret); | 471 | return(ret); |
472 | } | 472 | } |
473 | 473 | ||
@@ -476,43 +476,43 @@ static int __clear_reg_nv_device(int de_reg_device_id, struct task_struct *t) | |||
476 | int ret = 0; | 476 | int ret = 0; |
477 | int i; | 477 | int i; |
478 | unsigned long flags; | 478 | unsigned long flags; |
479 | nv_device_registry_t *reg = &NV_DEVICE_REG[de_reg_device_id]; | 479 | nv_device_registry_t *reg = &NV_DEVICE_REG[de_reg_device_id]; |
480 | 480 | ||
481 | #ifdef CONFIG_LITMUS_SOFTIRQD | 481 | #ifdef CONFIG_LITMUS_SOFTIRQD |
482 | struct task_struct* klitirqd_th = get_klitirqd(de_reg_device_id); | 482 | struct task_struct* klitirqd_th = get_klitirqd(de_reg_device_id); |
483 | #endif | 483 | #endif |
484 | 484 | ||
485 | WARN_ON(!test_bit(de_reg_device_id, &tsk_rt(t)->held_gpus)); | 485 | WARN_ON(!test_bit(de_reg_device_id, &tsk_rt(t)->held_gpus)); |
486 | 486 | ||
487 | raw_spin_lock_irqsave(®->lock, flags); | 487 | raw_spin_lock_irqsave(®->lock, flags); |
488 | 488 | ||
489 | TRACE_TASK(t, "unregisters GPU %d\n", de_reg_device_id); | 489 | TRACE_TASK(t, "unregisters GPU %d\n", de_reg_device_id); |
490 | 490 | ||
491 | for(i = 0; i < NV_MAX_SIMULT_USERS; ++i) { | 491 | for(i = 0; i < NV_MAX_SIMULT_USERS; ++i) { |
492 | if(reg->owners[i] == t) { | 492 | if(reg->owners[i] == t) { |
493 | #ifdef CONFIG_LITMUS_SOFTIRQD | 493 | #ifdef CONFIG_LITMUS_SOFTIRQD |
494 | flush_pending(klitirqd_th, t); | 494 | flush_pending(klitirqd_th, t); |
495 | #endif | 495 | #endif |
496 | if(reg->max_prio_owner == t) { | 496 | if(reg->max_prio_owner == t) { |
497 | reg->max_prio_owner = find_hp_owner(reg, t); | 497 | reg->max_prio_owner = find_hp_owner(reg, t); |
498 | #ifdef CONFIG_LITMUS_PAI_SOFTIRQD | 498 | #ifdef CONFIG_LITMUS_PAI_SOFTIRQD |
499 | litmus->change_prio_pai_tasklet(t, reg->max_prio_owner); | 499 | litmus->change_prio_pai_tasklet(t, reg->max_prio_owner); |
500 | #endif | 500 | #endif |
501 | } | 501 | } |
502 | 502 | ||
503 | #ifdef CONFIG_LITMUS_SOFTIRQD | 503 | #ifdef CONFIG_LITMUS_SOFTIRQD |
504 | up_and_set_stat(t, NOT_HELD, &tsk_rt(t)->klitirqd_sem); | 504 | up_and_set_stat(t, NOT_HELD, &tsk_rt(t)->klitirqd_sem); |
505 | #endif | 505 | #endif |
506 | 506 | ||
507 | reg->owners[i] = NULL; | 507 | reg->owners[i] = NULL; |
508 | --(reg->nr_owners); | 508 | --(reg->nr_owners); |
509 | 509 | ||
510 | break; | 510 | break; |
511 | } | 511 | } |
512 | } | 512 | } |
513 | 513 | ||
514 | raw_spin_unlock_irqrestore(®->lock, flags); | 514 | raw_spin_unlock_irqrestore(®->lock, flags); |
515 | 515 | ||
516 | __clear_bit(de_reg_device_id, &tsk_rt(t)->held_gpus); | 516 | __clear_bit(de_reg_device_id, &tsk_rt(t)->held_gpus); |
517 | 517 | ||
518 | return(ret); | 518 | return(ret); |
diff --git a/litmus/rsm_lock.c b/litmus/rsm_lock.c index 0a851cd430a7..6a4bb500c4ae 100644 --- a/litmus/rsm_lock.c +++ b/litmus/rsm_lock.c | |||
@@ -206,7 +206,7 @@ int rsm_mutex_lock(struct litmus_lock* l) | |||
206 | 206 | ||
207 | if (mutex->owner) { | 207 | if (mutex->owner) { |
208 | TRACE_TASK(t, "Blocking on lock %d.\n", l->ident); | 208 | TRACE_TASK(t, "Blocking on lock %d.\n", l->ident); |
209 | 209 | ||
210 | #if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA) | 210 | #if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA) |
211 | // KLUDGE: don't count this suspension as time in the critical gpu | 211 | // KLUDGE: don't count this suspension as time in the critical gpu |
212 | // critical section | 212 | // critical section |
@@ -214,7 +214,7 @@ int rsm_mutex_lock(struct litmus_lock* l) | |||
214 | tsk_rt(t)->suspend_gpu_tracker_on_block = 1; | 214 | tsk_rt(t)->suspend_gpu_tracker_on_block = 1; |
215 | } | 215 | } |
216 | #endif | 216 | #endif |
217 | 217 | ||
218 | /* resource is not free => must suspend and wait */ | 218 | /* resource is not free => must suspend and wait */ |
219 | 219 | ||
220 | owner = mutex->owner; | 220 | owner = mutex->owner; |
diff --git a/litmus/sched_cedf.c b/litmus/sched_cedf.c index a55fc894340d..132c44a43564 100644 --- a/litmus/sched_cedf.c +++ b/litmus/sched_cedf.c | |||
@@ -131,7 +131,7 @@ typedef struct clusterdomain { | |||
131 | 131 | ||
132 | #ifdef CONFIG_LITMUS_PAI_SOFTIRQD | 132 | #ifdef CONFIG_LITMUS_PAI_SOFTIRQD |
133 | struct tasklet_head pending_tasklets; | 133 | struct tasklet_head pending_tasklets; |
134 | #endif | 134 | #endif |
135 | } cedf_domain_t; | 135 | } cedf_domain_t; |
136 | 136 | ||
137 | /* a cedf_domain per cluster; allocation is done at init/activation time */ | 137 | /* a cedf_domain per cluster; allocation is done at init/activation time */ |
@@ -442,7 +442,7 @@ static void __do_lit_tasklet(struct tasklet_struct* tasklet, unsigned long flush | |||
442 | if(tasklet->owner) { | 442 | if(tasklet->owner) { |
443 | sched_trace_tasklet_begin(tasklet->owner); | 443 | sched_trace_tasklet_begin(tasklet->owner); |
444 | } | 444 | } |
445 | 445 | ||
446 | if (!test_and_clear_bit(TASKLET_STATE_SCHED, &tasklet->state)) | 446 | if (!test_and_clear_bit(TASKLET_STATE_SCHED, &tasklet->state)) |
447 | { | 447 | { |
448 | BUG(); | 448 | BUG(); |
@@ -453,7 +453,7 @@ static void __do_lit_tasklet(struct tasklet_struct* tasklet, unsigned long flush | |||
453 | (tasklet->owner) ? 0 : 1); | 453 | (tasklet->owner) ? 0 : 1); |
454 | tasklet->func(tasklet->data); | 454 | tasklet->func(tasklet->data); |
455 | tasklet_unlock(tasklet); | 455 | tasklet_unlock(tasklet); |
456 | 456 | ||
457 | if(tasklet->owner) { | 457 | if(tasklet->owner) { |
458 | sched_trace_tasklet_end(tasklet->owner, flushed); | 458 | sched_trace_tasklet_end(tasklet->owner, flushed); |
459 | } | 459 | } |
@@ -469,20 +469,20 @@ static void flush_tasklets(cedf_domain_t* cluster, struct task_struct* task) | |||
469 | // lazy flushing. | 469 | // lazy flushing. |
470 | // just change ownership to NULL and let an idle processor | 470 | // just change ownership to NULL and let an idle processor |
471 | // take care of it. :P | 471 | // take care of it. :P |
472 | 472 | ||
473 | struct tasklet_struct* step; | 473 | struct tasklet_struct* step; |
474 | unsigned long flags; | 474 | unsigned long flags; |
475 | 475 | ||
476 | raw_spin_lock_irqsave(&cluster->cluster_lock, flags); | 476 | raw_spin_lock_irqsave(&cluster->cluster_lock, flags); |
477 | 477 | ||
478 | for(step = cluster->pending_tasklets.head; step != NULL; step = step->next) { | 478 | for(step = cluster->pending_tasklets.head; step != NULL; step = step->next) { |
479 | if(step->owner == task) { | 479 | if(step->owner == task) { |
480 | TRACE("%s: Found tasklet to flush: %d\n", __FUNCTION__, step->owner->pid); | 480 | TRACE("%s: Found tasklet to flush: %d\n", __FUNCTION__, step->owner->pid); |
481 | step->owner = NULL; | 481 | step->owner = NULL; |
482 | } | 482 | } |
483 | } | 483 | } |
484 | 484 | ||
485 | raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags); | 485 | raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags); |
486 | } | 486 | } |
487 | 487 | ||
488 | 488 | ||
@@ -491,18 +491,18 @@ static void do_lit_tasklets(cedf_domain_t* cluster, struct task_struct* sched_ta | |||
491 | int work_to_do = 1; | 491 | int work_to_do = 1; |
492 | struct tasklet_struct *tasklet = NULL; | 492 | struct tasklet_struct *tasklet = NULL; |
493 | unsigned long flags; | 493 | unsigned long flags; |
494 | 494 | ||
495 | while(work_to_do) { | 495 | while(work_to_do) { |
496 | 496 | ||
497 | TS_NV_SCHED_BOTISR_START; | 497 | TS_NV_SCHED_BOTISR_START; |
498 | 498 | ||
499 | // remove tasklet at head of list if it has higher priority. | 499 | // remove tasklet at head of list if it has higher priority. |
500 | raw_spin_lock_irqsave(&cluster->cluster_lock, flags); | 500 | raw_spin_lock_irqsave(&cluster->cluster_lock, flags); |
501 | 501 | ||
502 | if(cluster->pending_tasklets.head != NULL) { | 502 | if(cluster->pending_tasklets.head != NULL) { |
503 | // remove tasklet at head. | 503 | // remove tasklet at head. |
504 | tasklet = cluster->pending_tasklets.head; | 504 | tasklet = cluster->pending_tasklets.head; |
505 | 505 | ||
506 | if(edf_higher_prio(tasklet->owner, sched_task)) { | 506 | if(edf_higher_prio(tasklet->owner, sched_task)) { |
507 | 507 | ||
508 | if(NULL == tasklet->next) { | 508 | if(NULL == tasklet->next) { |
@@ -526,12 +526,12 @@ static void do_lit_tasklets(cedf_domain_t* cluster, struct task_struct* sched_ta | |||
526 | } | 526 | } |
527 | 527 | ||
528 | raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags); | 528 | raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags); |
529 | 529 | ||
530 | TS_NV_SCHED_BOTISR_END; | 530 | TS_NV_SCHED_BOTISR_END; |
531 | 531 | ||
532 | if(tasklet) { | 532 | if(tasklet) { |
533 | __do_lit_tasklet(tasklet, 0ul); | 533 | __do_lit_tasklet(tasklet, 0ul); |
534 | tasklet = NULL; | 534 | tasklet = NULL; |
535 | } | 535 | } |
536 | else { | 536 | else { |
537 | work_to_do = 0; | 537 | work_to_do = 0; |
@@ -562,7 +562,7 @@ static void run_tasklets(struct task_struct* sched_task) | |||
562 | static void __add_pai_tasklet(struct tasklet_struct* tasklet, cedf_domain_t* cluster) | 562 | static void __add_pai_tasklet(struct tasklet_struct* tasklet, cedf_domain_t* cluster) |
563 | { | 563 | { |
564 | struct tasklet_struct* step; | 564 | struct tasklet_struct* step; |
565 | 565 | ||
566 | tasklet->next = NULL; // make sure there are no old values floating around | 566 | tasklet->next = NULL; // make sure there are no old values floating around |
567 | 567 | ||
568 | step = cluster->pending_tasklets.head; | 568 | step = cluster->pending_tasklets.head; |
@@ -570,13 +570,13 @@ static void __add_pai_tasklet(struct tasklet_struct* tasklet, cedf_domain_t* clu | |||
570 | TRACE("%s: tasklet queue empty. inserting tasklet for %d at head.\n", __FUNCTION__, tasklet->owner->pid); | 570 | TRACE("%s: tasklet queue empty. inserting tasklet for %d at head.\n", __FUNCTION__, tasklet->owner->pid); |
571 | // insert at tail. | 571 | // insert at tail. |
572 | *(cluster->pending_tasklets.tail) = tasklet; | 572 | *(cluster->pending_tasklets.tail) = tasklet; |
573 | cluster->pending_tasklets.tail = &(tasklet->next); | 573 | cluster->pending_tasklets.tail = &(tasklet->next); |
574 | } | 574 | } |
575 | else if((*(cluster->pending_tasklets.tail) != NULL) && | 575 | else if((*(cluster->pending_tasklets.tail) != NULL) && |
576 | edf_higher_prio((*(cluster->pending_tasklets.tail))->owner, tasklet->owner)) { | 576 | edf_higher_prio((*(cluster->pending_tasklets.tail))->owner, tasklet->owner)) { |
577 | // insert at tail. | 577 | // insert at tail. |
578 | TRACE("%s: tasklet belongs at end. inserting tasklet for %d at tail.\n", __FUNCTION__, tasklet->owner->pid); | 578 | TRACE("%s: tasklet belongs at end. inserting tasklet for %d at tail.\n", __FUNCTION__, tasklet->owner->pid); |
579 | 579 | ||
580 | *(cluster->pending_tasklets.tail) = tasklet; | 580 | *(cluster->pending_tasklets.tail) = tasklet; |
581 | cluster->pending_tasklets.tail = &(tasklet->next); | 581 | cluster->pending_tasklets.tail = &(tasklet->next); |
582 | } | 582 | } |
@@ -589,9 +589,9 @@ static void __add_pai_tasklet(struct tasklet_struct* tasklet, cedf_domain_t* clu | |||
589 | while(step->next && edf_higher_prio(step->next->owner, tasklet->owner)) { | 589 | while(step->next && edf_higher_prio(step->next->owner, tasklet->owner)) { |
590 | step = step->next; | 590 | step = step->next; |
591 | } | 591 | } |
592 | 592 | ||
593 | // insert tasklet right before step->next. | 593 | // insert tasklet right before step->next. |
594 | 594 | ||
595 | TRACE("%s: inserting tasklet for %d between %d and %d.\n", __FUNCTION__, | 595 | TRACE("%s: inserting tasklet for %d between %d and %d.\n", __FUNCTION__, |
596 | tasklet->owner->pid, | 596 | tasklet->owner->pid, |
597 | (step->owner) ? | 597 | (step->owner) ? |
@@ -602,7 +602,7 @@ static void __add_pai_tasklet(struct tasklet_struct* tasklet, cedf_domain_t* clu | |||
602 | step->next->owner->pid : | 602 | step->next->owner->pid : |
603 | -1) : | 603 | -1) : |
604 | -1); | 604 | -1); |
605 | 605 | ||
606 | tasklet->next = step->next; | 606 | tasklet->next = step->next; |
607 | step->next = tasklet; | 607 | step->next = tasklet; |
608 | 608 | ||
@@ -623,23 +623,23 @@ static int enqueue_pai_tasklet(struct tasklet_struct* tasklet) | |||
623 | int runLocal = 0; | 623 | int runLocal = 0; |
624 | int runNow = 0; | 624 | int runNow = 0; |
625 | unsigned long flags; | 625 | unsigned long flags; |
626 | 626 | ||
627 | if(unlikely((tasklet->owner == NULL) || !is_realtime(tasklet->owner))) | 627 | if(unlikely((tasklet->owner == NULL) || !is_realtime(tasklet->owner))) |
628 | { | 628 | { |
629 | TRACE("%s: No owner associated with this tasklet!\n", __FUNCTION__); | 629 | TRACE("%s: No owner associated with this tasklet!\n", __FUNCTION__); |
630 | return 0; | 630 | return 0; |
631 | } | 631 | } |
632 | 632 | ||
633 | cluster = task_cpu_cluster(tasklet->owner); | 633 | cluster = task_cpu_cluster(tasklet->owner); |
634 | 634 | ||
635 | raw_spin_lock_irqsave(&cluster->cluster_lock, flags); | 635 | raw_spin_lock_irqsave(&cluster->cluster_lock, flags); |
636 | 636 | ||
637 | thisCPU = smp_processor_id(); | 637 | thisCPU = smp_processor_id(); |
638 | 638 | ||
639 | #ifdef CONFIG_SCHED_CPU_AFFINITY | 639 | #ifdef CONFIG_SCHED_CPU_AFFINITY |
640 | { | 640 | { |
641 | cpu_entry_t* affinity = NULL; | 641 | cpu_entry_t* affinity = NULL; |
642 | 642 | ||
643 | // use this CPU if it is in our cluster and isn't running any RT work. | 643 | // use this CPU if it is in our cluster and isn't running any RT work. |
644 | if(cpu_isset(thisCPU, *cluster->cpu_map) && (__get_cpu_var(cedf_cpu_entries).linked == NULL)) { | 644 | if(cpu_isset(thisCPU, *cluster->cpu_map) && (__get_cpu_var(cedf_cpu_entries).linked == NULL)) { |
645 | affinity = &(__get_cpu_var(cedf_cpu_entries)); | 645 | affinity = &(__get_cpu_var(cedf_cpu_entries)); |
@@ -648,7 +648,7 @@ static int enqueue_pai_tasklet(struct tasklet_struct* tasklet) | |||
648 | // this CPU is busy or shouldn't run tasklet in this cluster. | 648 | // this CPU is busy or shouldn't run tasklet in this cluster. |
649 | // look for available near by CPUs. | 649 | // look for available near by CPUs. |
650 | // NOTE: Affinity towards owner and not this CPU. Is this right? | 650 | // NOTE: Affinity towards owner and not this CPU. Is this right? |
651 | affinity = | 651 | affinity = |
652 | cedf_get_nearest_available_cpu(cluster, | 652 | cedf_get_nearest_available_cpu(cluster, |
653 | &per_cpu(cedf_cpu_entries, task_cpu(tasklet->owner))); | 653 | &per_cpu(cedf_cpu_entries, task_cpu(tasklet->owner))); |
654 | } | 654 | } |
@@ -677,15 +677,15 @@ static int enqueue_pai_tasklet(struct tasklet_struct* tasklet) | |||
677 | runLocal = 0; | 677 | runLocal = 0; |
678 | runNow = 0; | 678 | runNow = 0; |
679 | } | 679 | } |
680 | 680 | ||
681 | if(!runLocal) { | 681 | if(!runLocal) { |
682 | // enqueue the tasklet | 682 | // enqueue the tasklet |
683 | __add_pai_tasklet(tasklet, cluster); | 683 | __add_pai_tasklet(tasklet, cluster); |
684 | } | 684 | } |
685 | 685 | ||
686 | raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags); | 686 | raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags); |
687 | 687 | ||
688 | 688 | ||
689 | if (runLocal /*&& runNow */) { // runNow == 1 is implied | 689 | if (runLocal /*&& runNow */) { // runNow == 1 is implied |
690 | TRACE("%s: Running tasklet on CPU where it was received.\n", __FUNCTION__); | 690 | TRACE("%s: Running tasklet on CPU where it was received.\n", __FUNCTION__); |
691 | __do_lit_tasklet(tasklet, 0ul); | 691 | __do_lit_tasklet(tasklet, 0ul); |
@@ -697,7 +697,7 @@ static int enqueue_pai_tasklet(struct tasklet_struct* tasklet) | |||
697 | else { | 697 | else { |
698 | TRACE("%s: Scheduling of tasklet was deferred.\n", __FUNCTION__); | 698 | TRACE("%s: Scheduling of tasklet was deferred.\n", __FUNCTION__); |
699 | } | 699 | } |
700 | 700 | ||
701 | return(1); // success | 701 | return(1); // success |
702 | } | 702 | } |
703 | 703 | ||
@@ -965,8 +965,8 @@ static void cedf_task_exit(struct task_struct * t) | |||
965 | 965 | ||
966 | #ifdef CONFIG_LITMUS_PAI_SOFTIRQD | 966 | #ifdef CONFIG_LITMUS_PAI_SOFTIRQD |
967 | flush_tasklets(cluster, t); | 967 | flush_tasklets(cluster, t); |
968 | #endif | 968 | #endif |
969 | 969 | ||
970 | /* unlink if necessary */ | 970 | /* unlink if necessary */ |
971 | raw_spin_lock_irqsave(&cluster->cluster_lock, flags); | 971 | raw_spin_lock_irqsave(&cluster->cluster_lock, flags); |
972 | unlink(t); | 972 | unlink(t); |
@@ -997,21 +997,21 @@ static long cedf_admit_task(struct task_struct* tsk) | |||
997 | static void __set_priority_inheritance(struct task_struct* t, struct task_struct* prio_inh) | 997 | static void __set_priority_inheritance(struct task_struct* t, struct task_struct* prio_inh) |
998 | { | 998 | { |
999 | int linked_on; | 999 | int linked_on; |
1000 | int check_preempt = 0; | 1000 | int check_preempt = 0; |
1001 | 1001 | ||
1002 | cedf_domain_t* cluster = task_cpu_cluster(t); | 1002 | cedf_domain_t* cluster = task_cpu_cluster(t); |
1003 | 1003 | ||
1004 | if(prio_inh != NULL) | 1004 | if(prio_inh != NULL) |
1005 | TRACE_TASK(t, "inherits priority from %s/%d\n", prio_inh->comm, prio_inh->pid); | 1005 | TRACE_TASK(t, "inherits priority from %s/%d\n", prio_inh->comm, prio_inh->pid); |
1006 | else | 1006 | else |
1007 | TRACE_TASK(t, "inherits priority from %p\n", prio_inh); | 1007 | TRACE_TASK(t, "inherits priority from %p\n", prio_inh); |
1008 | 1008 | ||
1009 | sched_trace_eff_prio_change(t, prio_inh); | 1009 | sched_trace_eff_prio_change(t, prio_inh); |
1010 | 1010 | ||
1011 | tsk_rt(t)->inh_task = prio_inh; | 1011 | tsk_rt(t)->inh_task = prio_inh; |
1012 | 1012 | ||
1013 | linked_on = tsk_rt(t)->linked_on; | 1013 | linked_on = tsk_rt(t)->linked_on; |
1014 | 1014 | ||
1015 | /* If it is scheduled, then we need to reorder the CPU heap. */ | 1015 | /* If it is scheduled, then we need to reorder the CPU heap. */ |
1016 | if (linked_on != NO_CPU) { | 1016 | if (linked_on != NO_CPU) { |
1017 | TRACE_TASK(t, "%s: linked on %d\n", | 1017 | TRACE_TASK(t, "%s: linked on %d\n", |
@@ -1029,12 +1029,12 @@ static void __set_priority_inheritance(struct task_struct* t, struct task_struct | |||
1029 | raw_spin_lock(&cluster->domain.release_lock); | 1029 | raw_spin_lock(&cluster->domain.release_lock); |
1030 | if (is_queued(t)) { | 1030 | if (is_queued(t)) { |
1031 | TRACE_TASK(t, "%s: is queued\n", __FUNCTION__); | 1031 | TRACE_TASK(t, "%s: is queued\n", __FUNCTION__); |
1032 | 1032 | ||
1033 | /* We need to update the position of holder in some | 1033 | /* We need to update the position of holder in some |
1034 | * heap. Note that this could be a release heap if we | 1034 | * heap. Note that this could be a release heap if we |
1035 | * budget enforcement is used and this job overran. */ | 1035 | * budget enforcement is used and this job overran. */ |
1036 | check_preempt = !bheap_decrease(edf_ready_order, tsk_rt(t)->heap_node); | 1036 | check_preempt = !bheap_decrease(edf_ready_order, tsk_rt(t)->heap_node); |
1037 | 1037 | ||
1038 | } else { | 1038 | } else { |
1039 | /* Nothing to do: if it is not queued and not linked | 1039 | /* Nothing to do: if it is not queued and not linked |
1040 | * then it is either sleeping or currently being moved | 1040 | * then it is either sleeping or currently being moved |
@@ -1044,7 +1044,7 @@ static void __set_priority_inheritance(struct task_struct* t, struct task_struct | |||
1044 | TRACE_TASK(t, "%s: is NOT queued => Done.\n", __FUNCTION__); | 1044 | TRACE_TASK(t, "%s: is NOT queued => Done.\n", __FUNCTION__); |
1045 | } | 1045 | } |
1046 | raw_spin_unlock(&cluster->domain.release_lock); | 1046 | raw_spin_unlock(&cluster->domain.release_lock); |
1047 | 1047 | ||
1048 | /* If holder was enqueued in a release heap, then the following | 1048 | /* If holder was enqueued in a release heap, then the following |
1049 | * preemption check is pointless, but we can't easily detect | 1049 | * preemption check is pointless, but we can't easily detect |
1050 | * that case. If you want to fix this, then consider that | 1050 | * that case. If you want to fix this, then consider that |
@@ -1065,21 +1065,21 @@ static void __set_priority_inheritance(struct task_struct* t, struct task_struct | |||
1065 | static void set_priority_inheritance(struct task_struct* t, struct task_struct* prio_inh) | 1065 | static void set_priority_inheritance(struct task_struct* t, struct task_struct* prio_inh) |
1066 | { | 1066 | { |
1067 | cedf_domain_t* cluster = task_cpu_cluster(t); | 1067 | cedf_domain_t* cluster = task_cpu_cluster(t); |
1068 | 1068 | ||
1069 | raw_spin_lock(&cluster->cluster_lock); | 1069 | raw_spin_lock(&cluster->cluster_lock); |
1070 | 1070 | ||
1071 | __set_priority_inheritance(t, prio_inh); | 1071 | __set_priority_inheritance(t, prio_inh); |
1072 | 1072 | ||
1073 | #ifdef CONFIG_LITMUS_SOFTIRQD | 1073 | #ifdef CONFIG_LITMUS_SOFTIRQD |
1074 | if(tsk_rt(t)->cur_klitirqd != NULL) | 1074 | if(tsk_rt(t)->cur_klitirqd != NULL) |
1075 | { | 1075 | { |
1076 | TRACE_TASK(t, "%s/%d inherits a new priority!\n", | 1076 | TRACE_TASK(t, "%s/%d inherits a new priority!\n", |
1077 | tsk_rt(t)->cur_klitirqd->comm, tsk_rt(t)->cur_klitirqd->pid); | 1077 | tsk_rt(t)->cur_klitirqd->comm, tsk_rt(t)->cur_klitirqd->pid); |
1078 | 1078 | ||
1079 | __set_priority_inheritance(tsk_rt(t)->cur_klitirqd, prio_inh); | 1079 | __set_priority_inheritance(tsk_rt(t)->cur_klitirqd, prio_inh); |
1080 | } | 1080 | } |
1081 | #endif | 1081 | #endif |
1082 | 1082 | ||
1083 | raw_spin_unlock(&cluster->cluster_lock); | 1083 | raw_spin_unlock(&cluster->cluster_lock); |
1084 | } | 1084 | } |
1085 | 1085 | ||
@@ -1088,13 +1088,13 @@ static void set_priority_inheritance(struct task_struct* t, struct task_struct* | |||
1088 | static void __clear_priority_inheritance(struct task_struct* t) | 1088 | static void __clear_priority_inheritance(struct task_struct* t) |
1089 | { | 1089 | { |
1090 | TRACE_TASK(t, "priority restored\n"); | 1090 | TRACE_TASK(t, "priority restored\n"); |
1091 | 1091 | ||
1092 | if(tsk_rt(t)->scheduled_on != NO_CPU) | 1092 | if(tsk_rt(t)->scheduled_on != NO_CPU) |
1093 | { | 1093 | { |
1094 | sched_trace_eff_prio_change(t, NULL); | 1094 | sched_trace_eff_prio_change(t, NULL); |
1095 | 1095 | ||
1096 | tsk_rt(t)->inh_task = NULL; | 1096 | tsk_rt(t)->inh_task = NULL; |
1097 | 1097 | ||
1098 | /* Check if rescheduling is necessary. We can't use heap_decrease() | 1098 | /* Check if rescheduling is necessary. We can't use heap_decrease() |
1099 | * since the priority was effectively lowered. */ | 1099 | * since the priority was effectively lowered. */ |
1100 | unlink(t); | 1100 | unlink(t); |
@@ -1104,19 +1104,19 @@ static void __clear_priority_inheritance(struct task_struct* t) | |||
1104 | { | 1104 | { |
1105 | __set_priority_inheritance(t, NULL); | 1105 | __set_priority_inheritance(t, NULL); |
1106 | } | 1106 | } |
1107 | 1107 | ||
1108 | #ifdef CONFIG_LITMUS_SOFTIRQD | 1108 | #ifdef CONFIG_LITMUS_SOFTIRQD |
1109 | if(tsk_rt(t)->cur_klitirqd != NULL) | 1109 | if(tsk_rt(t)->cur_klitirqd != NULL) |
1110 | { | 1110 | { |
1111 | TRACE_TASK(t, "%s/%d inheritance set back to owner.\n", | 1111 | TRACE_TASK(t, "%s/%d inheritance set back to owner.\n", |
1112 | tsk_rt(t)->cur_klitirqd->comm, tsk_rt(t)->cur_klitirqd->pid); | 1112 | tsk_rt(t)->cur_klitirqd->comm, tsk_rt(t)->cur_klitirqd->pid); |
1113 | 1113 | ||
1114 | if(tsk_rt(tsk_rt(t)->cur_klitirqd)->scheduled_on != NO_CPU) | 1114 | if(tsk_rt(tsk_rt(t)->cur_klitirqd)->scheduled_on != NO_CPU) |
1115 | { | 1115 | { |
1116 | sched_trace_eff_prio_change(tsk_rt(t)->cur_klitirqd, t); | 1116 | sched_trace_eff_prio_change(tsk_rt(t)->cur_klitirqd, t); |
1117 | 1117 | ||
1118 | tsk_rt(tsk_rt(t)->cur_klitirqd)->inh_task = t; | 1118 | tsk_rt(tsk_rt(t)->cur_klitirqd)->inh_task = t; |
1119 | 1119 | ||
1120 | /* Check if rescheduling is necessary. We can't use heap_decrease() | 1120 | /* Check if rescheduling is necessary. We can't use heap_decrease() |
1121 | * since the priority was effectively lowered. */ | 1121 | * since the priority was effectively lowered. */ |
1122 | unlink(tsk_rt(t)->cur_klitirqd); | 1122 | unlink(tsk_rt(t)->cur_klitirqd); |
@@ -1134,7 +1134,7 @@ static void __clear_priority_inheritance(struct task_struct* t) | |||
1134 | static void clear_priority_inheritance(struct task_struct* t) | 1134 | static void clear_priority_inheritance(struct task_struct* t) |
1135 | { | 1135 | { |
1136 | cedf_domain_t* cluster = task_cpu_cluster(t); | 1136 | cedf_domain_t* cluster = task_cpu_cluster(t); |
1137 | 1137 | ||
1138 | raw_spin_lock(&cluster->cluster_lock); | 1138 | raw_spin_lock(&cluster->cluster_lock); |
1139 | __clear_priority_inheritance(t); | 1139 | __clear_priority_inheritance(t); |
1140 | raw_spin_unlock(&cluster->cluster_lock); | 1140 | raw_spin_unlock(&cluster->cluster_lock); |
@@ -1149,11 +1149,11 @@ static void set_priority_inheritance_klitirqd(struct task_struct* klitirqd, | |||
1149 | struct task_struct* new_owner) | 1149 | struct task_struct* new_owner) |
1150 | { | 1150 | { |
1151 | cedf_domain_t* cluster = task_cpu_cluster(klitirqd); | 1151 | cedf_domain_t* cluster = task_cpu_cluster(klitirqd); |
1152 | 1152 | ||
1153 | BUG_ON(!(tsk_rt(klitirqd)->is_proxy_thread)); | 1153 | BUG_ON(!(tsk_rt(klitirqd)->is_proxy_thread)); |
1154 | 1154 | ||
1155 | raw_spin_lock(&cluster->cluster_lock); | 1155 | raw_spin_lock(&cluster->cluster_lock); |
1156 | 1156 | ||
1157 | if(old_owner != new_owner) | 1157 | if(old_owner != new_owner) |
1158 | { | 1158 | { |
1159 | if(old_owner) | 1159 | if(old_owner) |
@@ -1161,18 +1161,18 @@ static void set_priority_inheritance_klitirqd(struct task_struct* klitirqd, | |||
1161 | // unreachable? | 1161 | // unreachable? |
1162 | tsk_rt(old_owner)->cur_klitirqd = NULL; | 1162 | tsk_rt(old_owner)->cur_klitirqd = NULL; |
1163 | } | 1163 | } |
1164 | 1164 | ||
1165 | TRACE_TASK(klitirqd, "giving ownership to %s/%d.\n", | 1165 | TRACE_TASK(klitirqd, "giving ownership to %s/%d.\n", |
1166 | new_owner->comm, new_owner->pid); | 1166 | new_owner->comm, new_owner->pid); |
1167 | 1167 | ||
1168 | tsk_rt(new_owner)->cur_klitirqd = klitirqd; | 1168 | tsk_rt(new_owner)->cur_klitirqd = klitirqd; |
1169 | } | 1169 | } |
1170 | 1170 | ||
1171 | __set_priority_inheritance(klitirqd, | 1171 | __set_priority_inheritance(klitirqd, |
1172 | (tsk_rt(new_owner)->inh_task == NULL) ? | 1172 | (tsk_rt(new_owner)->inh_task == NULL) ? |
1173 | new_owner : | 1173 | new_owner : |
1174 | tsk_rt(new_owner)->inh_task); | 1174 | tsk_rt(new_owner)->inh_task); |
1175 | 1175 | ||
1176 | raw_spin_unlock(&cluster->cluster_lock); | 1176 | raw_spin_unlock(&cluster->cluster_lock); |
1177 | } | 1177 | } |
1178 | 1178 | ||
@@ -1181,17 +1181,17 @@ static void clear_priority_inheritance_klitirqd(struct task_struct* klitirqd, | |||
1181 | struct task_struct* old_owner) | 1181 | struct task_struct* old_owner) |
1182 | { | 1182 | { |
1183 | cedf_domain_t* cluster = task_cpu_cluster(klitirqd); | 1183 | cedf_domain_t* cluster = task_cpu_cluster(klitirqd); |
1184 | 1184 | ||
1185 | BUG_ON(!(tsk_rt(klitirqd)->is_proxy_thread)); | 1185 | BUG_ON(!(tsk_rt(klitirqd)->is_proxy_thread)); |
1186 | 1186 | ||
1187 | raw_spin_lock(&cluster->cluster_lock); | 1187 | raw_spin_lock(&cluster->cluster_lock); |
1188 | 1188 | ||
1189 | TRACE_TASK(klitirqd, "priority restored\n"); | 1189 | TRACE_TASK(klitirqd, "priority restored\n"); |
1190 | 1190 | ||
1191 | if(tsk_rt(klitirqd)->scheduled_on != NO_CPU) | 1191 | if(tsk_rt(klitirqd)->scheduled_on != NO_CPU) |
1192 | { | 1192 | { |
1193 | tsk_rt(klitirqd)->inh_task = NULL; | 1193 | tsk_rt(klitirqd)->inh_task = NULL; |
1194 | 1194 | ||
1195 | /* Check if rescheduling is necessary. We can't use heap_decrease() | 1195 | /* Check if rescheduling is necessary. We can't use heap_decrease() |
1196 | * since the priority was effectively lowered. */ | 1196 | * since the priority was effectively lowered. */ |
1197 | unlink(klitirqd); | 1197 | unlink(klitirqd); |
@@ -1201,9 +1201,9 @@ static void clear_priority_inheritance_klitirqd(struct task_struct* klitirqd, | |||
1201 | { | 1201 | { |
1202 | __set_priority_inheritance(klitirqd, NULL); | 1202 | __set_priority_inheritance(klitirqd, NULL); |
1203 | } | 1203 | } |
1204 | 1204 | ||
1205 | tsk_rt(old_owner)->cur_klitirqd = NULL; | 1205 | tsk_rt(old_owner)->cur_klitirqd = NULL; |
1206 | 1206 | ||
1207 | raw_spin_unlock(&cluster->cluster_lock); | 1207 | raw_spin_unlock(&cluster->cluster_lock); |
1208 | } | 1208 | } |
1209 | #endif // CONFIG_LITMUS_SOFTIRQD | 1209 | #endif // CONFIG_LITMUS_SOFTIRQD |
@@ -1223,9 +1223,9 @@ struct kfmlp_queue | |||
1223 | struct kfmlp_semaphore | 1223 | struct kfmlp_semaphore |
1224 | { | 1224 | { |
1225 | struct litmus_lock litmus_lock; | 1225 | struct litmus_lock litmus_lock; |
1226 | 1226 | ||
1227 | spinlock_t lock; | 1227 | spinlock_t lock; |
1228 | 1228 | ||
1229 | int num_resources; /* aka k */ | 1229 | int num_resources; /* aka k */ |
1230 | struct kfmlp_queue *queues; /* array */ | 1230 | struct kfmlp_queue *queues; /* array */ |
1231 | struct kfmlp_queue *shortest_queue; /* pointer to shortest queue */ | 1231 | struct kfmlp_queue *shortest_queue; /* pointer to shortest queue */ |
@@ -1258,11 +1258,11 @@ static struct task_struct* kfmlp_find_hp_waiter(struct kfmlp_queue *kqueue, | |||
1258 | { | 1258 | { |
1259 | struct list_head *pos; | 1259 | struct list_head *pos; |
1260 | struct task_struct *queued, *found = NULL; | 1260 | struct task_struct *queued, *found = NULL; |
1261 | 1261 | ||
1262 | list_for_each(pos, &kqueue->wait.task_list) { | 1262 | list_for_each(pos, &kqueue->wait.task_list) { |
1263 | queued = (struct task_struct*) list_entry(pos, wait_queue_t, | 1263 | queued = (struct task_struct*) list_entry(pos, wait_queue_t, |
1264 | task_list)->private; | 1264 | task_list)->private; |
1265 | 1265 | ||
1266 | /* Compare task prios, find high prio task. */ | 1266 | /* Compare task prios, find high prio task. */ |
1267 | if (queued != skip && edf_higher_prio(queued, found)) | 1267 | if (queued != skip && edf_higher_prio(queued, found)) |
1268 | found = queued; | 1268 | found = queued; |
@@ -1278,7 +1278,7 @@ static inline struct kfmlp_queue* kfmlp_find_shortest( | |||
1278 | // queue list to load-balance across all resources. | 1278 | // queue list to load-balance across all resources. |
1279 | struct kfmlp_queue* step = search_start; | 1279 | struct kfmlp_queue* step = search_start; |
1280 | struct kfmlp_queue* shortest = sem->shortest_queue; | 1280 | struct kfmlp_queue* shortest = sem->shortest_queue; |
1281 | 1281 | ||
1282 | do | 1282 | do |
1283 | { | 1283 | { |
1284 | step = (step+1 != &sem->queues[sem->num_resources]) ? | 1284 | step = (step+1 != &sem->queues[sem->num_resources]) ? |
@@ -1290,22 +1290,22 @@ static inline struct kfmlp_queue* kfmlp_find_shortest( | |||
1290 | break; /* can't get any shorter */ | 1290 | break; /* can't get any shorter */ |
1291 | } | 1291 | } |
1292 | }while(step != search_start); | 1292 | }while(step != search_start); |
1293 | 1293 | ||
1294 | return(shortest); | 1294 | return(shortest); |
1295 | } | 1295 | } |
1296 | 1296 | ||
1297 | static struct task_struct* kfmlp_remove_hp_waiter(struct kfmlp_semaphore* sem) | 1297 | static struct task_struct* kfmlp_remove_hp_waiter(struct kfmlp_semaphore* sem) |
1298 | { | 1298 | { |
1299 | /* must hold sem->lock */ | 1299 | /* must hold sem->lock */ |
1300 | 1300 | ||
1301 | struct kfmlp_queue *my_queue = NULL; | 1301 | struct kfmlp_queue *my_queue = NULL; |
1302 | struct task_struct *max_hp = NULL; | 1302 | struct task_struct *max_hp = NULL; |
1303 | 1303 | ||
1304 | 1304 | ||
1305 | struct list_head *pos; | 1305 | struct list_head *pos; |
1306 | struct task_struct *queued; | 1306 | struct task_struct *queued; |
1307 | int i; | 1307 | int i; |
1308 | 1308 | ||
1309 | for(i = 0; i < sem->num_resources; ++i) | 1309 | for(i = 0; i < sem->num_resources; ++i) |
1310 | { | 1310 | { |
1311 | if( (sem->queues[i].count > 1) && | 1311 | if( (sem->queues[i].count > 1) && |
@@ -1315,11 +1315,11 @@ static struct task_struct* kfmlp_remove_hp_waiter(struct kfmlp_semaphore* sem) | |||
1315 | my_queue = &sem->queues[i]; | 1315 | my_queue = &sem->queues[i]; |
1316 | } | 1316 | } |
1317 | } | 1317 | } |
1318 | 1318 | ||
1319 | if(my_queue) | 1319 | if(my_queue) |
1320 | { | 1320 | { |
1321 | cedf_domain_t* cluster; | 1321 | cedf_domain_t* cluster; |
1322 | 1322 | ||
1323 | max_hp = my_queue->hp_waiter; | 1323 | max_hp = my_queue->hp_waiter; |
1324 | BUG_ON(!max_hp); | 1324 | BUG_ON(!max_hp); |
1325 | 1325 | ||
@@ -1327,9 +1327,9 @@ static struct task_struct* kfmlp_remove_hp_waiter(struct kfmlp_semaphore* sem) | |||
1327 | kfmlp_get_idx(sem, my_queue), | 1327 | kfmlp_get_idx(sem, my_queue), |
1328 | max_hp->comm, max_hp->pid, | 1328 | max_hp->comm, max_hp->pid, |
1329 | kfmlp_get_idx(sem, my_queue)); | 1329 | kfmlp_get_idx(sem, my_queue)); |
1330 | 1330 | ||
1331 | my_queue->hp_waiter = kfmlp_find_hp_waiter(my_queue, max_hp); | 1331 | my_queue->hp_waiter = kfmlp_find_hp_waiter(my_queue, max_hp); |
1332 | 1332 | ||
1333 | /* | 1333 | /* |
1334 | if(my_queue->hp_waiter) | 1334 | if(my_queue->hp_waiter) |
1335 | TRACE_CUR("queue %d: new hp_waiter is %s/%d\n", | 1335 | TRACE_CUR("queue %d: new hp_waiter is %s/%d\n", |
@@ -1340,11 +1340,11 @@ static struct task_struct* kfmlp_remove_hp_waiter(struct kfmlp_semaphore* sem) | |||
1340 | TRACE_CUR("queue %d: new hp_waiter is %p\n", | 1340 | TRACE_CUR("queue %d: new hp_waiter is %p\n", |
1341 | kfmlp_get_idx(sem, my_queue), NULL); | 1341 | kfmlp_get_idx(sem, my_queue), NULL); |
1342 | */ | 1342 | */ |
1343 | 1343 | ||
1344 | cluster = task_cpu_cluster(max_hp); | 1344 | cluster = task_cpu_cluster(max_hp); |
1345 | 1345 | ||
1346 | raw_spin_lock(&cluster->cluster_lock); | 1346 | raw_spin_lock(&cluster->cluster_lock); |
1347 | 1347 | ||
1348 | /* | 1348 | /* |
1349 | if(my_queue->owner) | 1349 | if(my_queue->owner) |
1350 | TRACE_CUR("queue %d: owner is %s/%d\n", | 1350 | TRACE_CUR("queue %d: owner is %s/%d\n", |
@@ -1356,7 +1356,7 @@ static struct task_struct* kfmlp_remove_hp_waiter(struct kfmlp_semaphore* sem) | |||
1356 | kfmlp_get_idx(sem, my_queue), | 1356 | kfmlp_get_idx(sem, my_queue), |
1357 | NULL); | 1357 | NULL); |
1358 | */ | 1358 | */ |
1359 | 1359 | ||
1360 | if(tsk_rt(my_queue->owner)->inh_task == max_hp) | 1360 | if(tsk_rt(my_queue->owner)->inh_task == max_hp) |
1361 | { | 1361 | { |
1362 | __clear_priority_inheritance(my_queue->owner); | 1362 | __clear_priority_inheritance(my_queue->owner); |
@@ -1366,7 +1366,7 @@ static struct task_struct* kfmlp_remove_hp_waiter(struct kfmlp_semaphore* sem) | |||
1366 | } | 1366 | } |
1367 | } | 1367 | } |
1368 | raw_spin_unlock(&cluster->cluster_lock); | 1368 | raw_spin_unlock(&cluster->cluster_lock); |
1369 | 1369 | ||
1370 | list_for_each(pos, &my_queue->wait.task_list) | 1370 | list_for_each(pos, &my_queue->wait.task_list) |
1371 | { | 1371 | { |
1372 | queued = (struct task_struct*) list_entry(pos, wait_queue_t, | 1372 | queued = (struct task_struct*) list_entry(pos, wait_queue_t, |
@@ -1385,7 +1385,7 @@ static struct task_struct* kfmlp_remove_hp_waiter(struct kfmlp_semaphore* sem) | |||
1385 | } | 1385 | } |
1386 | --(my_queue->count); | 1386 | --(my_queue->count); |
1387 | } | 1387 | } |
1388 | 1388 | ||
1389 | return(max_hp); | 1389 | return(max_hp); |
1390 | } | 1390 | } |
1391 | 1391 | ||
@@ -1396,26 +1396,26 @@ int cedf_kfmlp_lock(struct litmus_lock* l) | |||
1396 | struct kfmlp_queue* my_queue; | 1396 | struct kfmlp_queue* my_queue; |
1397 | wait_queue_t wait; | 1397 | wait_queue_t wait; |
1398 | unsigned long flags; | 1398 | unsigned long flags; |
1399 | 1399 | ||
1400 | if (!is_realtime(t)) | 1400 | if (!is_realtime(t)) |
1401 | return -EPERM; | 1401 | return -EPERM; |
1402 | 1402 | ||
1403 | spin_lock_irqsave(&sem->lock, flags); | 1403 | spin_lock_irqsave(&sem->lock, flags); |
1404 | 1404 | ||
1405 | my_queue = sem->shortest_queue; | 1405 | my_queue = sem->shortest_queue; |
1406 | 1406 | ||
1407 | if (my_queue->owner) { | 1407 | if (my_queue->owner) { |
1408 | /* resource is not free => must suspend and wait */ | 1408 | /* resource is not free => must suspend and wait */ |
1409 | TRACE_CUR("queue %d: Resource is not free => must suspend and wait.\n", | 1409 | TRACE_CUR("queue %d: Resource is not free => must suspend and wait.\n", |
1410 | kfmlp_get_idx(sem, my_queue)); | 1410 | kfmlp_get_idx(sem, my_queue)); |
1411 | 1411 | ||
1412 | init_waitqueue_entry(&wait, t); | 1412 | init_waitqueue_entry(&wait, t); |
1413 | 1413 | ||
1414 | /* FIXME: interruptible would be nice some day */ | 1414 | /* FIXME: interruptible would be nice some day */ |
1415 | set_task_state(t, TASK_UNINTERRUPTIBLE); | 1415 | set_task_state(t, TASK_UNINTERRUPTIBLE); |
1416 | 1416 | ||
1417 | __add_wait_queue_tail_exclusive(&my_queue->wait, &wait); | 1417 | __add_wait_queue_tail_exclusive(&my_queue->wait, &wait); |
1418 | 1418 | ||
1419 | /* check if we need to activate priority inheritance */ | 1419 | /* check if we need to activate priority inheritance */ |
1420 | if (edf_higher_prio(t, my_queue->hp_waiter)) | 1420 | if (edf_higher_prio(t, my_queue->hp_waiter)) |
1421 | { | 1421 | { |
@@ -1425,20 +1425,20 @@ int cedf_kfmlp_lock(struct litmus_lock* l) | |||
1425 | set_priority_inheritance(my_queue->owner, my_queue->hp_waiter); | 1425 | set_priority_inheritance(my_queue->owner, my_queue->hp_waiter); |
1426 | } | 1426 | } |
1427 | } | 1427 | } |
1428 | 1428 | ||
1429 | ++(my_queue->count); | 1429 | ++(my_queue->count); |
1430 | sem->shortest_queue = kfmlp_find_shortest(sem, my_queue); | 1430 | sem->shortest_queue = kfmlp_find_shortest(sem, my_queue); |
1431 | 1431 | ||
1432 | /* release lock before sleeping */ | 1432 | /* release lock before sleeping */ |
1433 | spin_unlock_irqrestore(&sem->lock, flags); | 1433 | spin_unlock_irqrestore(&sem->lock, flags); |
1434 | 1434 | ||
1435 | /* We depend on the FIFO order. Thus, we don't need to recheck | 1435 | /* We depend on the FIFO order. Thus, we don't need to recheck |
1436 | * when we wake up; we are guaranteed to have the lock since | 1436 | * when we wake up; we are guaranteed to have the lock since |
1437 | * there is only one wake up per release (or steal). | 1437 | * there is only one wake up per release (or steal). |
1438 | */ | 1438 | */ |
1439 | schedule(); | 1439 | schedule(); |
1440 | 1440 | ||
1441 | 1441 | ||
1442 | if(my_queue->owner == t) | 1442 | if(my_queue->owner == t) |
1443 | { | 1443 | { |
1444 | TRACE_CUR("queue %d: acquired through waiting\n", | 1444 | TRACE_CUR("queue %d: acquired through waiting\n", |
@@ -1458,15 +1458,15 @@ int cedf_kfmlp_lock(struct litmus_lock* l) | |||
1458 | { | 1458 | { |
1459 | TRACE_CUR("queue %d: acquired immediately\n", | 1459 | TRACE_CUR("queue %d: acquired immediately\n", |
1460 | kfmlp_get_idx(sem, my_queue)); | 1460 | kfmlp_get_idx(sem, my_queue)); |
1461 | 1461 | ||
1462 | my_queue->owner = t; | 1462 | my_queue->owner = t; |
1463 | 1463 | ||
1464 | ++(my_queue->count); | 1464 | ++(my_queue->count); |
1465 | sem->shortest_queue = kfmlp_find_shortest(sem, my_queue); | 1465 | sem->shortest_queue = kfmlp_find_shortest(sem, my_queue); |
1466 | 1466 | ||
1467 | spin_unlock_irqrestore(&sem->lock, flags); | 1467 | spin_unlock_irqrestore(&sem->lock, flags); |
1468 | } | 1468 | } |
1469 | 1469 | ||
1470 | return kfmlp_get_idx(sem, my_queue); | 1470 | return kfmlp_get_idx(sem, my_queue); |
1471 | } | 1471 | } |
1472 | 1472 | ||
@@ -1477,16 +1477,16 @@ int cedf_kfmlp_unlock(struct litmus_lock* l) | |||
1477 | struct kfmlp_queue *my_queue; | 1477 | struct kfmlp_queue *my_queue; |
1478 | unsigned long flags; | 1478 | unsigned long flags; |
1479 | int err = 0; | 1479 | int err = 0; |
1480 | 1480 | ||
1481 | spin_lock_irqsave(&sem->lock, flags); | 1481 | spin_lock_irqsave(&sem->lock, flags); |
1482 | 1482 | ||
1483 | my_queue = kfmlp_get_queue(sem, t); | 1483 | my_queue = kfmlp_get_queue(sem, t); |
1484 | 1484 | ||
1485 | if (!my_queue) { | 1485 | if (!my_queue) { |
1486 | err = -EINVAL; | 1486 | err = -EINVAL; |
1487 | goto out; | 1487 | goto out; |
1488 | } | 1488 | } |
1489 | 1489 | ||
1490 | /* check if there are jobs waiting for this resource */ | 1490 | /* check if there are jobs waiting for this resource */ |
1491 | next = __waitqueue_remove_first(&my_queue->wait); | 1491 | next = __waitqueue_remove_first(&my_queue->wait); |
1492 | if (next) { | 1492 | if (next) { |
@@ -1497,16 +1497,16 @@ int cedf_kfmlp_unlock(struct litmus_lock* l) | |||
1497 | */ | 1497 | */ |
1498 | /* next becomes the resouce holder */ | 1498 | /* next becomes the resouce holder */ |
1499 | my_queue->owner = next; | 1499 | my_queue->owner = next; |
1500 | 1500 | ||
1501 | --(my_queue->count); | 1501 | --(my_queue->count); |
1502 | if(my_queue->count < sem->shortest_queue->count) | 1502 | if(my_queue->count < sem->shortest_queue->count) |
1503 | { | 1503 | { |
1504 | sem->shortest_queue = my_queue; | 1504 | sem->shortest_queue = my_queue; |
1505 | } | 1505 | } |
1506 | 1506 | ||
1507 | TRACE_CUR("queue %d: lock ownership passed to %s/%d\n", | 1507 | TRACE_CUR("queue %d: lock ownership passed to %s/%d\n", |
1508 | kfmlp_get_idx(sem, my_queue), next->comm, next->pid); | 1508 | kfmlp_get_idx(sem, my_queue), next->comm, next->pid); |
1509 | 1509 | ||
1510 | /* determine new hp_waiter if necessary */ | 1510 | /* determine new hp_waiter if necessary */ |
1511 | if (next == my_queue->hp_waiter) { | 1511 | if (next == my_queue->hp_waiter) { |
1512 | TRACE_TASK(next, "was highest-prio waiter\n"); | 1512 | TRACE_TASK(next, "was highest-prio waiter\n"); |
@@ -1525,38 +1525,38 @@ int cedf_kfmlp_unlock(struct litmus_lock* l) | |||
1525 | * waiter's priority. */ | 1525 | * waiter's priority. */ |
1526 | set_priority_inheritance(next, my_queue->hp_waiter); | 1526 | set_priority_inheritance(next, my_queue->hp_waiter); |
1527 | } | 1527 | } |
1528 | 1528 | ||
1529 | /* wake up next */ | 1529 | /* wake up next */ |
1530 | wake_up_process(next); | 1530 | wake_up_process(next); |
1531 | } | 1531 | } |
1532 | else | 1532 | else |
1533 | { | 1533 | { |
1534 | TRACE_CUR("queue %d: looking to steal someone...\n", kfmlp_get_idx(sem, my_queue)); | 1534 | TRACE_CUR("queue %d: looking to steal someone...\n", kfmlp_get_idx(sem, my_queue)); |
1535 | 1535 | ||
1536 | next = kfmlp_remove_hp_waiter(sem); /* returns NULL if nothing to steal */ | 1536 | next = kfmlp_remove_hp_waiter(sem); /* returns NULL if nothing to steal */ |
1537 | 1537 | ||
1538 | /* | 1538 | /* |
1539 | if(next) | 1539 | if(next) |
1540 | TRACE_CUR("queue %d: ASSIGNING %s/%d as owner - steal\n", | 1540 | TRACE_CUR("queue %d: ASSIGNING %s/%d as owner - steal\n", |
1541 | kfmlp_get_idx(sem, my_queue), | 1541 | kfmlp_get_idx(sem, my_queue), |
1542 | next->comm, next->pid); | 1542 | next->comm, next->pid); |
1543 | */ | 1543 | */ |
1544 | 1544 | ||
1545 | my_queue->owner = next; | 1545 | my_queue->owner = next; |
1546 | 1546 | ||
1547 | if(next) | 1547 | if(next) |
1548 | { | 1548 | { |
1549 | TRACE_CUR("queue %d: lock ownership passed to %s/%d (which was stolen)\n", | 1549 | TRACE_CUR("queue %d: lock ownership passed to %s/%d (which was stolen)\n", |
1550 | kfmlp_get_idx(sem, my_queue), | 1550 | kfmlp_get_idx(sem, my_queue), |
1551 | next->comm, next->pid); | 1551 | next->comm, next->pid); |
1552 | 1552 | ||
1553 | /* wake up next */ | 1553 | /* wake up next */ |
1554 | wake_up_process(next); | 1554 | wake_up_process(next); |
1555 | } | 1555 | } |
1556 | else | 1556 | else |
1557 | { | 1557 | { |
1558 | TRACE_CUR("queue %d: no one to steal.\n", kfmlp_get_idx(sem, my_queue)); | 1558 | TRACE_CUR("queue %d: no one to steal.\n", kfmlp_get_idx(sem, my_queue)); |
1559 | 1559 | ||
1560 | --(my_queue->count); | 1560 | --(my_queue->count); |
1561 | if(my_queue->count < sem->shortest_queue->count) | 1561 | if(my_queue->count < sem->shortest_queue->count) |
1562 | { | 1562 | { |
@@ -1564,14 +1564,14 @@ int cedf_kfmlp_unlock(struct litmus_lock* l) | |||
1564 | } | 1564 | } |
1565 | } | 1565 | } |
1566 | } | 1566 | } |
1567 | 1567 | ||
1568 | /* we lose the benefit of priority inheritance (if any) */ | 1568 | /* we lose the benefit of priority inheritance (if any) */ |
1569 | if (tsk_rt(t)->inh_task) | 1569 | if (tsk_rt(t)->inh_task) |
1570 | clear_priority_inheritance(t); | 1570 | clear_priority_inheritance(t); |
1571 | 1571 | ||
1572 | out: | 1572 | out: |
1573 | spin_unlock_irqrestore(&sem->lock, flags); | 1573 | spin_unlock_irqrestore(&sem->lock, flags); |
1574 | 1574 | ||
1575 | return err; | 1575 | return err; |
1576 | } | 1576 | } |
1577 | 1577 | ||
@@ -1581,19 +1581,19 @@ int cedf_kfmlp_close(struct litmus_lock* l) | |||
1581 | struct kfmlp_semaphore *sem = kfmlp_from_lock(l); | 1581 | struct kfmlp_semaphore *sem = kfmlp_from_lock(l); |
1582 | struct kfmlp_queue *my_queue; | 1582 | struct kfmlp_queue *my_queue; |
1583 | unsigned long flags; | 1583 | unsigned long flags; |
1584 | 1584 | ||
1585 | int owner; | 1585 | int owner; |
1586 | 1586 | ||
1587 | spin_lock_irqsave(&sem->lock, flags); | 1587 | spin_lock_irqsave(&sem->lock, flags); |
1588 | 1588 | ||
1589 | my_queue = kfmlp_get_queue(sem, t); | 1589 | my_queue = kfmlp_get_queue(sem, t); |
1590 | owner = (my_queue) ? (my_queue->owner == t) : 0; | 1590 | owner = (my_queue) ? (my_queue->owner == t) : 0; |
1591 | 1591 | ||
1592 | spin_unlock_irqrestore(&sem->lock, flags); | 1592 | spin_unlock_irqrestore(&sem->lock, flags); |
1593 | 1593 | ||
1594 | if (owner) | 1594 | if (owner) |
1595 | cedf_kfmlp_unlock(l); | 1595 | cedf_kfmlp_unlock(l); |
1596 | 1596 | ||
1597 | return 0; | 1597 | return 0; |
1598 | } | 1598 | } |
1599 | 1599 | ||
@@ -1616,7 +1616,7 @@ static struct litmus_lock* cedf_new_kfmlp(void* __user arg, int* ret_code) | |||
1616 | struct kfmlp_semaphore* sem; | 1616 | struct kfmlp_semaphore* sem; |
1617 | int num_resources = 0; | 1617 | int num_resources = 0; |
1618 | int i; | 1618 | int i; |
1619 | 1619 | ||
1620 | if(!access_ok(VERIFY_READ, arg, sizeof(num_resources))) | 1620 | if(!access_ok(VERIFY_READ, arg, sizeof(num_resources))) |
1621 | { | 1621 | { |
1622 | *ret_code = -EINVAL; | 1622 | *ret_code = -EINVAL; |
@@ -1630,28 +1630,28 @@ static struct litmus_lock* cedf_new_kfmlp(void* __user arg, int* ret_code) | |||
1630 | if(num_resources < 1) | 1630 | if(num_resources < 1) |
1631 | { | 1631 | { |
1632 | *ret_code = -EINVAL; | 1632 | *ret_code = -EINVAL; |
1633 | return(NULL); | 1633 | return(NULL); |
1634 | } | 1634 | } |
1635 | 1635 | ||
1636 | sem = kmalloc(sizeof(*sem), GFP_KERNEL); | 1636 | sem = kmalloc(sizeof(*sem), GFP_KERNEL); |
1637 | if(!sem) | 1637 | if(!sem) |
1638 | { | 1638 | { |
1639 | *ret_code = -ENOMEM; | 1639 | *ret_code = -ENOMEM; |
1640 | return NULL; | 1640 | return NULL; |
1641 | } | 1641 | } |
1642 | 1642 | ||
1643 | sem->queues = kmalloc(sizeof(struct kfmlp_queue)*num_resources, GFP_KERNEL); | 1643 | sem->queues = kmalloc(sizeof(struct kfmlp_queue)*num_resources, GFP_KERNEL); |
1644 | if(!sem->queues) | 1644 | if(!sem->queues) |
1645 | { | 1645 | { |
1646 | kfree(sem); | 1646 | kfree(sem); |
1647 | *ret_code = -ENOMEM; | 1647 | *ret_code = -ENOMEM; |
1648 | return NULL; | 1648 | return NULL; |
1649 | } | 1649 | } |
1650 | 1650 | ||
1651 | sem->litmus_lock.ops = &cedf_kfmlp_lock_ops; | 1651 | sem->litmus_lock.ops = &cedf_kfmlp_lock_ops; |
1652 | spin_lock_init(&sem->lock); | 1652 | spin_lock_init(&sem->lock); |
1653 | sem->num_resources = num_resources; | 1653 | sem->num_resources = num_resources; |
1654 | 1654 | ||
1655 | for(i = 0; i < num_resources; ++i) | 1655 | for(i = 0; i < num_resources; ++i) |
1656 | { | 1656 | { |
1657 | sem->queues[i].owner = NULL; | 1657 | sem->queues[i].owner = NULL; |
@@ -1659,9 +1659,9 @@ static struct litmus_lock* cedf_new_kfmlp(void* __user arg, int* ret_code) | |||
1659 | init_waitqueue_head(&sem->queues[i].wait); | 1659 | init_waitqueue_head(&sem->queues[i].wait); |
1660 | sem->queues[i].count = 0; | 1660 | sem->queues[i].count = 0; |
1661 | } | 1661 | } |
1662 | 1662 | ||
1663 | sem->shortest_queue = &sem->queues[0]; | 1663 | sem->shortest_queue = &sem->queues[0]; |
1664 | 1664 | ||
1665 | *ret_code = 0; | 1665 | *ret_code = 0; |
1666 | return &sem->litmus_lock; | 1666 | return &sem->litmus_lock; |
1667 | } | 1667 | } |
@@ -1673,7 +1673,7 @@ static long cedf_allocate_lock(struct litmus_lock **lock, int type, | |||
1673 | void* __user arg) | 1673 | void* __user arg) |
1674 | { | 1674 | { |
1675 | int err = -ENXIO; | 1675 | int err = -ENXIO; |
1676 | 1676 | ||
1677 | /* C-EDF currently only supports the FMLP for global resources | 1677 | /* C-EDF currently only supports the FMLP for global resources |
1678 | WITHIN a given cluster. DO NOT USE CROSS-CLUSTER! */ | 1678 | WITHIN a given cluster. DO NOT USE CROSS-CLUSTER! */ |
1679 | switch (type) { | 1679 | switch (type) { |
@@ -1681,7 +1681,7 @@ static long cedf_allocate_lock(struct litmus_lock **lock, int type, | |||
1681 | *lock = cedf_new_kfmlp(arg, &err); | 1681 | *lock = cedf_new_kfmlp(arg, &err); |
1682 | break; | 1682 | break; |
1683 | }; | 1683 | }; |
1684 | 1684 | ||
1685 | return err; | 1685 | return err; |
1686 | } | 1686 | } |
1687 | 1687 | ||
@@ -1781,13 +1781,13 @@ static long cedf_activate_plugin(void) | |||
1781 | INIT_BINHEAP_HANDLE(&(cedf[i].cpu_heap), cpu_lower_prio); | 1781 | INIT_BINHEAP_HANDLE(&(cedf[i].cpu_heap), cpu_lower_prio); |
1782 | edf_domain_init(&(cedf[i].domain), NULL, cedf_release_jobs); | 1782 | edf_domain_init(&(cedf[i].domain), NULL, cedf_release_jobs); |
1783 | 1783 | ||
1784 | 1784 | ||
1785 | #ifdef CONFIG_LITMUS_PAI_SOFTIRQD | 1785 | #ifdef CONFIG_LITMUS_PAI_SOFTIRQD |
1786 | cedf[i].pending_tasklets.head = NULL; | 1786 | cedf[i].pending_tasklets.head = NULL; |
1787 | cedf[i].pending_tasklets.tail = &(cedf[i].pending_tasklets.head); | 1787 | cedf[i].pending_tasklets.tail = &(cedf[i].pending_tasklets.head); |
1788 | #endif | 1788 | #endif |
1789 | 1789 | ||
1790 | 1790 | ||
1791 | if(!zalloc_cpumask_var(&cedf[i].cpu_map, GFP_ATOMIC)) | 1791 | if(!zalloc_cpumask_var(&cedf[i].cpu_map, GFP_ATOMIC)) |
1792 | return -ENOMEM; | 1792 | return -ENOMEM; |
1793 | #ifdef CONFIG_RELEASE_MASTER | 1793 | #ifdef CONFIG_RELEASE_MASTER |
@@ -1846,14 +1846,14 @@ static long cedf_activate_plugin(void) | |||
1846 | break; | 1846 | break; |
1847 | } | 1847 | } |
1848 | } | 1848 | } |
1849 | 1849 | ||
1850 | #ifdef CONFIG_LITMUS_SOFTIRQD | 1850 | #ifdef CONFIG_LITMUS_SOFTIRQD |
1851 | { | 1851 | { |
1852 | /* distribute the daemons evenly across the clusters. */ | 1852 | /* distribute the daemons evenly across the clusters. */ |
1853 | int* affinity = kmalloc(NR_LITMUS_SOFTIRQD * sizeof(int), GFP_ATOMIC); | 1853 | int* affinity = kmalloc(NR_LITMUS_SOFTIRQD * sizeof(int), GFP_ATOMIC); |
1854 | int num_daemons_per_cluster = NR_LITMUS_SOFTIRQD / num_clusters; | 1854 | int num_daemons_per_cluster = NR_LITMUS_SOFTIRQD / num_clusters; |
1855 | int left_over = NR_LITMUS_SOFTIRQD % num_clusters; | 1855 | int left_over = NR_LITMUS_SOFTIRQD % num_clusters; |
1856 | 1856 | ||
1857 | int daemon = 0; | 1857 | int daemon = 0; |
1858 | for(i = 0; i < num_clusters; ++i) | 1858 | for(i = 0; i < num_clusters; ++i) |
1859 | { | 1859 | { |
@@ -1863,23 +1863,23 @@ static long cedf_activate_plugin(void) | |||
1863 | ++num_on_this_cluster; | 1863 | ++num_on_this_cluster; |
1864 | --left_over; | 1864 | --left_over; |
1865 | } | 1865 | } |
1866 | 1866 | ||
1867 | for(j = 0; j < num_on_this_cluster; ++j) | 1867 | for(j = 0; j < num_on_this_cluster; ++j) |
1868 | { | 1868 | { |
1869 | // first CPU of this cluster | 1869 | // first CPU of this cluster |
1870 | affinity[daemon++] = i*cluster_size; | 1870 | affinity[daemon++] = i*cluster_size; |
1871 | } | 1871 | } |
1872 | } | 1872 | } |
1873 | 1873 | ||
1874 | spawn_klitirqd(affinity); | 1874 | spawn_klitirqd(affinity); |
1875 | 1875 | ||
1876 | kfree(affinity); | 1876 | kfree(affinity); |
1877 | } | 1877 | } |
1878 | #endif | 1878 | #endif |
1879 | 1879 | ||
1880 | #ifdef CONFIG_LITMUS_NVIDIA | 1880 | #ifdef CONFIG_LITMUS_NVIDIA |
1881 | init_nvidia_info(); | 1881 | init_nvidia_info(); |
1882 | #endif | 1882 | #endif |
1883 | 1883 | ||
1884 | free_cpumask_var(mask); | 1884 | free_cpumask_var(mask); |
1885 | clusters_allocated = 1; | 1885 | clusters_allocated = 1; |
@@ -1902,7 +1902,7 @@ static struct sched_plugin cedf_plugin __cacheline_aligned_in_smp = { | |||
1902 | #ifdef CONFIG_LITMUS_LOCKING | 1902 | #ifdef CONFIG_LITMUS_LOCKING |
1903 | .allocate_lock = cedf_allocate_lock, | 1903 | .allocate_lock = cedf_allocate_lock, |
1904 | .set_prio_inh = set_priority_inheritance, | 1904 | .set_prio_inh = set_priority_inheritance, |
1905 | .clear_prio_inh = clear_priority_inheritance, | 1905 | .clear_prio_inh = clear_priority_inheritance, |
1906 | #endif | 1906 | #endif |
1907 | #ifdef CONFIG_LITMUS_SOFTIRQD | 1907 | #ifdef CONFIG_LITMUS_SOFTIRQD |
1908 | .set_prio_inh_klitirqd = set_priority_inheritance_klitirqd, | 1908 | .set_prio_inh_klitirqd = set_priority_inheritance_klitirqd, |
diff --git a/litmus/sched_gsn_edf.c b/litmus/sched_gsn_edf.c index 09334aea43ac..422ad0395099 100644 --- a/litmus/sched_gsn_edf.c +++ b/litmus/sched_gsn_edf.c | |||
@@ -366,7 +366,7 @@ static void check_for_preemptions(void) | |||
366 | static noinline void gsnedf_job_arrival(struct task_struct* task) | 366 | static noinline void gsnedf_job_arrival(struct task_struct* task) |
367 | { | 367 | { |
368 | BUG_ON(!task); | 368 | BUG_ON(!task); |
369 | 369 | ||
370 | requeue(task); | 370 | requeue(task); |
371 | check_for_preemptions(); | 371 | check_for_preemptions(); |
372 | } | 372 | } |
@@ -387,7 +387,7 @@ static void gsnedf_release_jobs(rt_domain_t* rt, struct bheap* tasks) | |||
387 | static noinline void job_completion(struct task_struct *t, int forced) | 387 | static noinline void job_completion(struct task_struct *t, int forced) |
388 | { | 388 | { |
389 | BUG_ON(!t); | 389 | BUG_ON(!t); |
390 | 390 | ||
391 | sched_trace_task_completion(t, forced); | 391 | sched_trace_task_completion(t, forced); |
392 | 392 | ||
393 | #ifdef CONFIG_LITMUS_NVIDIA | 393 | #ifdef CONFIG_LITMUS_NVIDIA |
@@ -446,7 +446,7 @@ static void __do_lit_tasklet(struct tasklet_struct* tasklet, unsigned long flush | |||
446 | { | 446 | { |
447 | if (!atomic_read(&tasklet->count)) { | 447 | if (!atomic_read(&tasklet->count)) { |
448 | sched_trace_tasklet_begin(tasklet->owner); | 448 | sched_trace_tasklet_begin(tasklet->owner); |
449 | 449 | ||
450 | if (!test_and_clear_bit(TASKLET_STATE_SCHED, &tasklet->state)) | 450 | if (!test_and_clear_bit(TASKLET_STATE_SCHED, &tasklet->state)) |
451 | { | 451 | { |
452 | BUG(); | 452 | BUG(); |
@@ -454,7 +454,7 @@ static void __do_lit_tasklet(struct tasklet_struct* tasklet, unsigned long flush | |||
454 | TRACE("%s: Invoking tasklet with owner pid = %d (flushed = %d).\n", __FUNCTION__, tasklet->owner->pid, flushed); | 454 | TRACE("%s: Invoking tasklet with owner pid = %d (flushed = %d).\n", __FUNCTION__, tasklet->owner->pid, flushed); |
455 | tasklet->func(tasklet->data); | 455 | tasklet->func(tasklet->data); |
456 | tasklet_unlock(tasklet); | 456 | tasklet_unlock(tasklet); |
457 | 457 | ||
458 | sched_trace_tasklet_end(tasklet->owner, flushed); | 458 | sched_trace_tasklet_end(tasklet->owner, flushed); |
459 | } | 459 | } |
460 | else { | 460 | else { |
@@ -467,54 +467,54 @@ static void do_lit_tasklets(struct task_struct* sched_task) | |||
467 | int work_to_do = 1; | 467 | int work_to_do = 1; |
468 | struct tasklet_struct *tasklet = NULL; | 468 | struct tasklet_struct *tasklet = NULL; |
469 | unsigned long flags; | 469 | unsigned long flags; |
470 | 470 | ||
471 | while(work_to_do) { | 471 | while(work_to_do) { |
472 | 472 | ||
473 | TS_NV_SCHED_BOTISR_START; | 473 | TS_NV_SCHED_BOTISR_START; |
474 | 474 | ||
475 | // execute one tasklet that has higher priority | 475 | // execute one tasklet that has higher priority |
476 | raw_spin_lock_irqsave(&gsnedf_lock, flags); | 476 | raw_spin_lock_irqsave(&gsnedf_lock, flags); |
477 | 477 | ||
478 | if(gsnedf_pending_tasklets.head != NULL) { | 478 | if(gsnedf_pending_tasklets.head != NULL) { |
479 | struct tasklet_struct *prev = NULL; | 479 | struct tasklet_struct *prev = NULL; |
480 | tasklet = gsnedf_pending_tasklets.head; | 480 | tasklet = gsnedf_pending_tasklets.head; |
481 | 481 | ||
482 | while(tasklet && edf_higher_prio(sched_task, tasklet->owner)) { | 482 | while(tasklet && edf_higher_prio(sched_task, tasklet->owner)) { |
483 | prev = tasklet; | 483 | prev = tasklet; |
484 | tasklet = tasklet->next; | 484 | tasklet = tasklet->next; |
485 | } | 485 | } |
486 | 486 | ||
487 | // remove the tasklet from the queue | 487 | // remove the tasklet from the queue |
488 | if(prev) { | 488 | if(prev) { |
489 | prev->next = tasklet->next; | 489 | prev->next = tasklet->next; |
490 | if(prev->next == NULL) { | 490 | if(prev->next == NULL) { |
491 | TRACE("%s: Tasklet for %d is the last element in tasklet queue.\n", __FUNCTION__, tasklet->owner->pid); | 491 | TRACE("%s: Tasklet for %d is the last element in tasklet queue.\n", __FUNCTION__, tasklet->owner->pid); |
492 | gsnedf_pending_tasklets.tail = &(prev); | 492 | gsnedf_pending_tasklets.tail = &(prev); |
493 | } | 493 | } |
494 | } | 494 | } |
495 | else { | 495 | else { |
496 | gsnedf_pending_tasklets.head = tasklet->next; | 496 | gsnedf_pending_tasklets.head = tasklet->next; |
497 | if(tasklet->next == NULL) { | 497 | if(tasklet->next == NULL) { |
498 | TRACE("%s: Tasklet for %d is the last element in tasklet queue.\n", __FUNCTION__, tasklet->owner->pid); | 498 | TRACE("%s: Tasklet for %d is the last element in tasklet queue.\n", __FUNCTION__, tasklet->owner->pid); |
499 | gsnedf_pending_tasklets.tail = &(gsnedf_pending_tasklets.head); | 499 | gsnedf_pending_tasklets.tail = &(gsnedf_pending_tasklets.head); |
500 | } | 500 | } |
501 | } | 501 | } |
502 | } | 502 | } |
503 | else { | 503 | else { |
504 | TRACE("%s: Tasklet queue is empty.\n", __FUNCTION__); | 504 | TRACE("%s: Tasklet queue is empty.\n", __FUNCTION__); |
505 | } | 505 | } |
506 | 506 | ||
507 | raw_spin_unlock_irqrestore(&gsnedf_lock, flags); | 507 | raw_spin_unlock_irqrestore(&gsnedf_lock, flags); |
508 | 508 | ||
509 | if(tasklet) { | 509 | if(tasklet) { |
510 | __do_lit_tasklet(tasklet, 0ul); | 510 | __do_lit_tasklet(tasklet, 0ul); |
511 | tasklet = NULL; | 511 | tasklet = NULL; |
512 | } | 512 | } |
513 | else { | 513 | else { |
514 | work_to_do = 0; | 514 | work_to_do = 0; |
515 | } | 515 | } |
516 | 516 | ||
517 | TS_NV_SCHED_BOTISR_END; | 517 | TS_NV_SCHED_BOTISR_END; |
518 | } | 518 | } |
519 | } | 519 | } |
520 | 520 | ||
@@ -524,29 +524,29 @@ static void do_lit_tasklets(struct task_struct* sched_task) | |||
524 | // struct tasklet_struct *tasklet = NULL; | 524 | // struct tasklet_struct *tasklet = NULL; |
525 | // //struct tasklet_struct *step; | 525 | // //struct tasklet_struct *step; |
526 | // unsigned long flags; | 526 | // unsigned long flags; |
527 | // | 527 | // |
528 | // while(work_to_do) { | 528 | // while(work_to_do) { |
529 | // | 529 | // |
530 | // TS_NV_SCHED_BOTISR_START; | 530 | // TS_NV_SCHED_BOTISR_START; |
531 | // | 531 | // |
532 | // // remove tasklet at head of list if it has higher priority. | 532 | // // remove tasklet at head of list if it has higher priority. |
533 | // raw_spin_lock_irqsave(&gsnedf_lock, flags); | 533 | // raw_spin_lock_irqsave(&gsnedf_lock, flags); |
534 | // | 534 | // |
535 | // if(gsnedf_pending_tasklets.head != NULL) { | 535 | // if(gsnedf_pending_tasklets.head != NULL) { |
536 | // // remove tasklet at head. | 536 | // // remove tasklet at head. |
537 | // tasklet = gsnedf_pending_tasklets.head; | 537 | // tasklet = gsnedf_pending_tasklets.head; |
538 | // | 538 | // |
539 | // if(edf_higher_prio(tasklet->owner, sched_task)) { | 539 | // if(edf_higher_prio(tasklet->owner, sched_task)) { |
540 | // | 540 | // |
541 | // if(NULL == tasklet->next) { | 541 | // if(NULL == tasklet->next) { |
542 | // // tasklet is at the head, list only has one element | 542 | // // tasklet is at the head, list only has one element |
543 | // TRACE("%s: Tasklet for %d is the last element in tasklet queue.\n", __FUNCTION__, tasklet->owner->pid); | 543 | // TRACE("%s: Tasklet for %d is the last element in tasklet queue.\n", __FUNCTION__, tasklet->owner->pid); |
544 | // gsnedf_pending_tasklets.tail = &(gsnedf_pending_tasklets.head); | 544 | // gsnedf_pending_tasklets.tail = &(gsnedf_pending_tasklets.head); |
545 | // } | 545 | // } |
546 | // | 546 | // |
547 | // // remove the tasklet from the queue | 547 | // // remove the tasklet from the queue |
548 | // gsnedf_pending_tasklets.head = tasklet->next; | 548 | // gsnedf_pending_tasklets.head = tasklet->next; |
549 | // | 549 | // |
550 | // TRACE("%s: Removed tasklet for %d from tasklet queue.\n", __FUNCTION__, tasklet->owner->pid); | 550 | // TRACE("%s: Removed tasklet for %d from tasklet queue.\n", __FUNCTION__, tasklet->owner->pid); |
551 | // } | 551 | // } |
552 | // else { | 552 | // else { |
@@ -557,60 +557,60 @@ static void do_lit_tasklets(struct task_struct* sched_task) | |||
557 | // else { | 557 | // else { |
558 | // TRACE("%s: Tasklet queue is empty.\n", __FUNCTION__); | 558 | // TRACE("%s: Tasklet queue is empty.\n", __FUNCTION__); |
559 | // } | 559 | // } |
560 | // | 560 | // |
561 | // raw_spin_unlock_irqrestore(&gsnedf_lock, flags); | 561 | // raw_spin_unlock_irqrestore(&gsnedf_lock, flags); |
562 | // | 562 | // |
563 | // TS_NV_SCHED_BOTISR_END; | 563 | // TS_NV_SCHED_BOTISR_END; |
564 | // | 564 | // |
565 | // if(tasklet) { | 565 | // if(tasklet) { |
566 | // __do_lit_tasklet(tasklet, 0ul); | 566 | // __do_lit_tasklet(tasklet, 0ul); |
567 | // tasklet = NULL; | 567 | // tasklet = NULL; |
568 | // } | 568 | // } |
569 | // else { | 569 | // else { |
570 | // work_to_do = 0; | 570 | // work_to_do = 0; |
571 | // } | 571 | // } |
572 | // } | 572 | // } |
573 | // | 573 | // |
574 | // //TRACE("%s: exited.\n", __FUNCTION__); | 574 | // //TRACE("%s: exited.\n", __FUNCTION__); |
575 | //} | 575 | //} |
576 | 576 | ||
577 | static void __add_pai_tasklet(struct tasklet_struct* tasklet) | 577 | static void __add_pai_tasklet(struct tasklet_struct* tasklet) |
578 | { | 578 | { |
579 | struct tasklet_struct* step; | 579 | struct tasklet_struct* step; |
580 | 580 | ||
581 | tasklet->next = NULL; // make sure there are no old values floating around | 581 | tasklet->next = NULL; // make sure there are no old values floating around |
582 | 582 | ||
583 | step = gsnedf_pending_tasklets.head; | 583 | step = gsnedf_pending_tasklets.head; |
584 | if(step == NULL) { | 584 | if(step == NULL) { |
585 | TRACE("%s: tasklet queue empty. inserting tasklet for %d at head.\n", __FUNCTION__, tasklet->owner->pid); | 585 | TRACE("%s: tasklet queue empty. inserting tasklet for %d at head.\n", __FUNCTION__, tasklet->owner->pid); |
586 | // insert at tail. | 586 | // insert at tail. |
587 | *(gsnedf_pending_tasklets.tail) = tasklet; | 587 | *(gsnedf_pending_tasklets.tail) = tasklet; |
588 | gsnedf_pending_tasklets.tail = &(tasklet->next); | 588 | gsnedf_pending_tasklets.tail = &(tasklet->next); |
589 | } | 589 | } |
590 | else if((*(gsnedf_pending_tasklets.tail) != NULL) && | 590 | else if((*(gsnedf_pending_tasklets.tail) != NULL) && |
591 | edf_higher_prio((*(gsnedf_pending_tasklets.tail))->owner, tasklet->owner)) { | 591 | edf_higher_prio((*(gsnedf_pending_tasklets.tail))->owner, tasklet->owner)) { |
592 | // insert at tail. | 592 | // insert at tail. |
593 | TRACE("%s: tasklet belongs at end. inserting tasklet for %d at tail.\n", __FUNCTION__, tasklet->owner->pid); | 593 | TRACE("%s: tasklet belongs at end. inserting tasklet for %d at tail.\n", __FUNCTION__, tasklet->owner->pid); |
594 | 594 | ||
595 | *(gsnedf_pending_tasklets.tail) = tasklet; | 595 | *(gsnedf_pending_tasklets.tail) = tasklet; |
596 | gsnedf_pending_tasklets.tail = &(tasklet->next); | 596 | gsnedf_pending_tasklets.tail = &(tasklet->next); |
597 | } | 597 | } |
598 | else { | 598 | else { |
599 | // insert the tasklet somewhere in the middle. | 599 | // insert the tasklet somewhere in the middle. |
600 | 600 | ||
601 | TRACE("%s: tasklet belongs somewhere in the middle.\n", __FUNCTION__); | 601 | TRACE("%s: tasklet belongs somewhere in the middle.\n", __FUNCTION__); |
602 | 602 | ||
603 | while(step->next && edf_higher_prio(step->next->owner, tasklet->owner)) { | 603 | while(step->next && edf_higher_prio(step->next->owner, tasklet->owner)) { |
604 | step = step->next; | 604 | step = step->next; |
605 | } | 605 | } |
606 | 606 | ||
607 | // insert tasklet right before step->next. | 607 | // insert tasklet right before step->next. |
608 | 608 | ||
609 | TRACE("%s: inserting tasklet for %d between %d and %d.\n", __FUNCTION__, tasklet->owner->pid, step->owner->pid, (step->next) ? step->next->owner->pid : -1); | 609 | TRACE("%s: inserting tasklet for %d between %d and %d.\n", __FUNCTION__, tasklet->owner->pid, step->owner->pid, (step->next) ? step->next->owner->pid : -1); |
610 | 610 | ||
611 | tasklet->next = step->next; | 611 | tasklet->next = step->next; |
612 | step->next = tasklet; | 612 | step->next = tasklet; |
613 | 613 | ||
614 | // patch up the head if needed. | 614 | // patch up the head if needed. |
615 | if(gsnedf_pending_tasklets.head == step) | 615 | if(gsnedf_pending_tasklets.head == step) |
616 | { | 616 | { |
@@ -623,12 +623,12 @@ static void __add_pai_tasklet(struct tasklet_struct* tasklet) | |||
623 | static void gsnedf_run_tasklets(struct task_struct* sched_task) | 623 | static void gsnedf_run_tasklets(struct task_struct* sched_task) |
624 | { | 624 | { |
625 | preempt_disable(); | 625 | preempt_disable(); |
626 | 626 | ||
627 | if(gsnedf_pending_tasklets.head != NULL) { | 627 | if(gsnedf_pending_tasklets.head != NULL) { |
628 | TRACE("%s: There are tasklets to process.\n", __FUNCTION__); | 628 | TRACE("%s: There are tasklets to process.\n", __FUNCTION__); |
629 | do_lit_tasklets(sched_task); | 629 | do_lit_tasklets(sched_task); |
630 | } | 630 | } |
631 | 631 | ||
632 | preempt_enable_no_resched(); | 632 | preempt_enable_no_resched(); |
633 | } | 633 | } |
634 | 634 | ||
@@ -639,22 +639,22 @@ static int gsnedf_enqueue_pai_tasklet(struct tasklet_struct* tasklet) | |||
639 | int runLocal = 0; | 639 | int runLocal = 0; |
640 | int runNow = 0; | 640 | int runNow = 0; |
641 | unsigned long flags; | 641 | unsigned long flags; |
642 | 642 | ||
643 | if(unlikely((tasklet->owner == NULL) || !is_realtime(tasklet->owner))) | 643 | if(unlikely((tasklet->owner == NULL) || !is_realtime(tasklet->owner))) |
644 | { | 644 | { |
645 | TRACE("%s: No owner associated with this tasklet!\n", __FUNCTION__); | 645 | TRACE("%s: No owner associated with this tasklet!\n", __FUNCTION__); |
646 | return 0; | 646 | return 0; |
647 | } | 647 | } |
648 | |||
649 | |||
650 | raw_spin_lock_irqsave(&gsnedf_lock, flags); | ||
648 | 651 | ||
649 | |||
650 | raw_spin_lock_irqsave(&gsnedf_lock, flags); | ||
651 | |||
652 | thisCPU = smp_processor_id(); | 652 | thisCPU = smp_processor_id(); |
653 | 653 | ||
654 | #ifdef CONFIG_SCHED_CPU_AFFINITY | 654 | #ifdef CONFIG_SCHED_CPU_AFFINITY |
655 | { | 655 | { |
656 | cpu_entry_t* affinity = NULL; | 656 | cpu_entry_t* affinity = NULL; |
657 | 657 | ||
658 | // use this CPU if it is in our cluster and isn't running any RT work. | 658 | // use this CPU if it is in our cluster and isn't running any RT work. |
659 | if( | 659 | if( |
660 | #ifdef CONFIG_RELEASE_MASTER | 660 | #ifdef CONFIG_RELEASE_MASTER |
@@ -666,20 +666,20 @@ static int gsnedf_enqueue_pai_tasklet(struct tasklet_struct* tasklet) | |||
666 | else { | 666 | else { |
667 | // this CPU is busy or shouldn't run tasklet in this cluster. | 667 | // this CPU is busy or shouldn't run tasklet in this cluster. |
668 | // look for available near by CPUs. | 668 | // look for available near by CPUs. |
669 | // NOTE: Affinity towards owner and not this CPU. Is this right? | 669 | // NOTE: Affinity towards owner and not this CPU. Is this right? |
670 | affinity = | 670 | affinity = |
671 | gsnedf_get_nearest_available_cpu( | 671 | gsnedf_get_nearest_available_cpu( |
672 | &per_cpu(gsnedf_cpu_entries, task_cpu(tasklet->owner))); | 672 | &per_cpu(gsnedf_cpu_entries, task_cpu(tasklet->owner))); |
673 | } | 673 | } |
674 | 674 | ||
675 | targetCPU = affinity; | 675 | targetCPU = affinity; |
676 | } | 676 | } |
677 | #endif | 677 | #endif |
678 | 678 | ||
679 | if (targetCPU == NULL) { | 679 | if (targetCPU == NULL) { |
680 | targetCPU = lowest_prio_cpu(); | 680 | targetCPU = lowest_prio_cpu(); |
681 | } | 681 | } |
682 | 682 | ||
683 | if (edf_higher_prio(tasklet->owner, targetCPU->linked)) { | 683 | if (edf_higher_prio(tasklet->owner, targetCPU->linked)) { |
684 | if (thisCPU == targetCPU->cpu) { | 684 | if (thisCPU == targetCPU->cpu) { |
685 | TRACE("%s: Run tasklet locally (and now).\n", __FUNCTION__); | 685 | TRACE("%s: Run tasklet locally (and now).\n", __FUNCTION__); |
@@ -696,15 +696,15 @@ static int gsnedf_enqueue_pai_tasklet(struct tasklet_struct* tasklet) | |||
696 | runLocal = 0; | 696 | runLocal = 0; |
697 | runNow = 0; | 697 | runNow = 0; |
698 | } | 698 | } |
699 | 699 | ||
700 | if(!runLocal) { | 700 | if(!runLocal) { |
701 | // enqueue the tasklet | 701 | // enqueue the tasklet |
702 | __add_pai_tasklet(tasklet); | 702 | __add_pai_tasklet(tasklet); |
703 | } | 703 | } |
704 | 704 | ||
705 | raw_spin_unlock_irqrestore(&gsnedf_lock, flags); | 705 | raw_spin_unlock_irqrestore(&gsnedf_lock, flags); |
706 | 706 | ||
707 | 707 | ||
708 | if (runLocal /*&& runNow */) { // runNow == 1 is implied | 708 | if (runLocal /*&& runNow */) { // runNow == 1 is implied |
709 | TRACE("%s: Running tasklet on CPU where it was received.\n", __FUNCTION__); | 709 | TRACE("%s: Running tasklet on CPU where it was received.\n", __FUNCTION__); |
710 | __do_lit_tasklet(tasklet, 0ul); | 710 | __do_lit_tasklet(tasklet, 0ul); |
@@ -716,7 +716,7 @@ static int gsnedf_enqueue_pai_tasklet(struct tasklet_struct* tasklet) | |||
716 | else { | 716 | else { |
717 | TRACE("%s: Scheduling of tasklet was deferred.\n", __FUNCTION__); | 717 | TRACE("%s: Scheduling of tasklet was deferred.\n", __FUNCTION__); |
718 | } | 718 | } |
719 | 719 | ||
720 | return(1); // success | 720 | return(1); // success |
721 | } | 721 | } |
722 | 722 | ||
@@ -725,7 +725,7 @@ static void gsnedf_change_prio_pai_tasklet(struct task_struct *old_prio, | |||
725 | { | 725 | { |
726 | struct tasklet_struct* step; | 726 | struct tasklet_struct* step; |
727 | unsigned long flags; | 727 | unsigned long flags; |
728 | 728 | ||
729 | if(gsnedf_pending_tasklets.head != NULL) { | 729 | if(gsnedf_pending_tasklets.head != NULL) { |
730 | raw_spin_lock_irqsave(&gsnedf_lock, flags); | 730 | raw_spin_lock_irqsave(&gsnedf_lock, flags); |
731 | for(step = gsnedf_pending_tasklets.head; step != NULL; step = step->next) { | 731 | for(step = gsnedf_pending_tasklets.head; step != NULL; step = step->next) { |
@@ -807,7 +807,7 @@ static struct task_struct* gsnedf_schedule(struct task_struct * prev) | |||
807 | blocks, out_of_time, np, sleep, preempt, | 807 | blocks, out_of_time, np, sleep, preempt, |
808 | prev->state, signal_pending(prev)); | 808 | prev->state, signal_pending(prev)); |
809 | */ | 809 | */ |
810 | 810 | ||
811 | if (entry->linked && preempt) | 811 | if (entry->linked && preempt) |
812 | TRACE_TASK(prev, "will be preempted by %s/%d\n", | 812 | TRACE_TASK(prev, "will be preempted by %s/%d\n", |
813 | entry->linked->comm, entry->linked->pid); | 813 | entry->linked->comm, entry->linked->pid); |
@@ -817,8 +817,8 @@ static struct task_struct* gsnedf_schedule(struct task_struct * prev) | |||
817 | if (blocks) { | 817 | if (blocks) { |
818 | unlink(entry->scheduled); | 818 | unlink(entry->scheduled); |
819 | } | 819 | } |
820 | 820 | ||
821 | #if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_AFFINITY_LOCKING) | 821 | #if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_AFFINITY_LOCKING) |
822 | if(exists && is_realtime(entry->scheduled) && tsk_rt(entry->scheduled)->held_gpus) { | 822 | if(exists && is_realtime(entry->scheduled) && tsk_rt(entry->scheduled)->held_gpus) { |
823 | if(!blocks || tsk_rt(entry->scheduled)->suspend_gpu_tracker_on_block) { | 823 | if(!blocks || tsk_rt(entry->scheduled)->suspend_gpu_tracker_on_block) { |
824 | stop_gpu_tracker(entry->scheduled); | 824 | stop_gpu_tracker(entry->scheduled); |
@@ -874,7 +874,7 @@ static struct task_struct* gsnedf_schedule(struct task_struct * prev) | |||
874 | if (exists) | 874 | if (exists) |
875 | next = prev; | 875 | next = prev; |
876 | } | 876 | } |
877 | 877 | ||
878 | sched_state_task_picked(); | 878 | sched_state_task_picked(); |
879 | 879 | ||
880 | raw_spin_unlock(&gsnedf_lock); | 880 | raw_spin_unlock(&gsnedf_lock); |
@@ -898,9 +898,9 @@ static struct task_struct* gsnedf_schedule(struct task_struct * prev) | |||
898 | static void gsnedf_finish_switch(struct task_struct *prev) | 898 | static void gsnedf_finish_switch(struct task_struct *prev) |
899 | { | 899 | { |
900 | cpu_entry_t* entry = &__get_cpu_var(gsnedf_cpu_entries); | 900 | cpu_entry_t* entry = &__get_cpu_var(gsnedf_cpu_entries); |
901 | 901 | ||
902 | entry->scheduled = is_realtime(current) ? current : NULL; | 902 | entry->scheduled = is_realtime(current) ? current : NULL; |
903 | 903 | ||
904 | #ifdef WANT_ALL_SCHED_EVENTS | 904 | #ifdef WANT_ALL_SCHED_EVENTS |
905 | TRACE_TASK(prev, "switched away from\n"); | 905 | TRACE_TASK(prev, "switched away from\n"); |
906 | #endif | 906 | #endif |
@@ -949,13 +949,13 @@ static void gsnedf_task_new(struct task_struct * t, int on_rq, int running) | |||
949 | static void gsnedf_task_wake_up(struct task_struct *task) | 949 | static void gsnedf_task_wake_up(struct task_struct *task) |
950 | { | 950 | { |
951 | unsigned long flags; | 951 | unsigned long flags; |
952 | //lt_t now; | 952 | //lt_t now; |
953 | 953 | ||
954 | TRACE_TASK(task, "wake_up at %llu\n", litmus_clock()); | 954 | TRACE_TASK(task, "wake_up at %llu\n", litmus_clock()); |
955 | 955 | ||
956 | raw_spin_lock_irqsave(&gsnedf_lock, flags); | 956 | raw_spin_lock_irqsave(&gsnedf_lock, flags); |
957 | 957 | ||
958 | 958 | ||
959 | #if 0 // sporadic task model | 959 | #if 0 // sporadic task model |
960 | /* We need to take suspensions because of semaphores into | 960 | /* We need to take suspensions because of semaphores into |
961 | * account! If a job resumes after being suspended due to acquiring | 961 | * account! If a job resumes after being suspended due to acquiring |
@@ -981,7 +981,7 @@ static void gsnedf_task_wake_up(struct task_struct *task) | |||
981 | #else // periodic task model | 981 | #else // periodic task model |
982 | set_rt_flags(task, RT_F_RUNNING); | 982 | set_rt_flags(task, RT_F_RUNNING); |
983 | #endif | 983 | #endif |
984 | 984 | ||
985 | gsnedf_job_arrival(task); | 985 | gsnedf_job_arrival(task); |
986 | raw_spin_unlock_irqrestore(&gsnedf_lock, flags); | 986 | raw_spin_unlock_irqrestore(&gsnedf_lock, flags); |
987 | } | 987 | } |
@@ -995,9 +995,9 @@ static void gsnedf_task_block(struct task_struct *t) | |||
995 | 995 | ||
996 | /* unlink if necessary */ | 996 | /* unlink if necessary */ |
997 | raw_spin_lock_irqsave(&gsnedf_lock, flags); | 997 | raw_spin_lock_irqsave(&gsnedf_lock, flags); |
998 | 998 | ||
999 | unlink(t); | 999 | unlink(t); |
1000 | 1000 | ||
1001 | raw_spin_unlock_irqrestore(&gsnedf_lock, flags); | 1001 | raw_spin_unlock_irqrestore(&gsnedf_lock, flags); |
1002 | 1002 | ||
1003 | BUG_ON(!is_realtime(t)); | 1003 | BUG_ON(!is_realtime(t)); |
@@ -1010,8 +1010,8 @@ static void gsnedf_task_exit(struct task_struct * t) | |||
1010 | 1010 | ||
1011 | #ifdef CONFIG_LITMUS_PAI_SOFTIRQD | 1011 | #ifdef CONFIG_LITMUS_PAI_SOFTIRQD |
1012 | gsnedf_change_prio_pai_tasklet(t, NULL); | 1012 | gsnedf_change_prio_pai_tasklet(t, NULL); |
1013 | #endif | 1013 | #endif |
1014 | 1014 | ||
1015 | /* unlink if necessary */ | 1015 | /* unlink if necessary */ |
1016 | raw_spin_lock_irqsave(&gsnedf_lock, flags); | 1016 | raw_spin_lock_irqsave(&gsnedf_lock, flags); |
1017 | unlink(t); | 1017 | unlink(t); |
@@ -1020,7 +1020,7 @@ static void gsnedf_task_exit(struct task_struct * t) | |||
1020 | tsk_rt(t)->scheduled_on = NO_CPU; | 1020 | tsk_rt(t)->scheduled_on = NO_CPU; |
1021 | } | 1021 | } |
1022 | raw_spin_unlock_irqrestore(&gsnedf_lock, flags); | 1022 | raw_spin_unlock_irqrestore(&gsnedf_lock, flags); |
1023 | 1023 | ||
1024 | BUG_ON(!is_realtime(t)); | 1024 | BUG_ON(!is_realtime(t)); |
1025 | TRACE_TASK(t, "RIP\n"); | 1025 | TRACE_TASK(t, "RIP\n"); |
1026 | } | 1026 | } |
@@ -1131,7 +1131,7 @@ static void __increase_priority_inheritance(struct task_struct* t, | |||
1131 | static void increase_priority_inheritance(struct task_struct* t, struct task_struct* prio_inh) | 1131 | static void increase_priority_inheritance(struct task_struct* t, struct task_struct* prio_inh) |
1132 | { | 1132 | { |
1133 | int i = 0; | 1133 | int i = 0; |
1134 | 1134 | ||
1135 | raw_spin_lock(&gsnedf_lock); | 1135 | raw_spin_lock(&gsnedf_lock); |
1136 | 1136 | ||
1137 | __increase_priority_inheritance(t, prio_inh); | 1137 | __increase_priority_inheritance(t, prio_inh); |
@@ -1141,13 +1141,13 @@ static void increase_priority_inheritance(struct task_struct* t, struct task_str | |||
1141 | { | 1141 | { |
1142 | TRACE_TASK(t, "%s/%d inherits a new priority!\n", | 1142 | TRACE_TASK(t, "%s/%d inherits a new priority!\n", |
1143 | tsk_rt(t)->cur_klitirqd->comm, tsk_rt(t)->cur_klitirqd->pid); | 1143 | tsk_rt(t)->cur_klitirqd->comm, tsk_rt(t)->cur_klitirqd->pid); |
1144 | 1144 | ||
1145 | __increase_priority_inheritance(tsk_rt(t)->cur_klitirqd, prio_inh); | 1145 | __increase_priority_inheritance(tsk_rt(t)->cur_klitirqd, prio_inh); |
1146 | } | 1146 | } |
1147 | #endif | 1147 | #endif |
1148 | 1148 | ||
1149 | raw_spin_unlock(&gsnedf_lock); | 1149 | raw_spin_unlock(&gsnedf_lock); |
1150 | 1150 | ||
1151 | #if defined(CONFIG_LITMUS_PAI_SOFTIRQD) && defined(CONFIG_LITMUS_NVIDIA) | 1151 | #if defined(CONFIG_LITMUS_PAI_SOFTIRQD) && defined(CONFIG_LITMUS_NVIDIA) |
1152 | if(tsk_rt(t)->held_gpus) { | 1152 | if(tsk_rt(t)->held_gpus) { |
1153 | for(i = find_first_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus)); | 1153 | for(i = find_first_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus)); |
@@ -1218,22 +1218,22 @@ static void decrease_priority_inheritance(struct task_struct* t, | |||
1218 | struct task_struct* prio_inh) | 1218 | struct task_struct* prio_inh) |
1219 | { | 1219 | { |
1220 | int i; | 1220 | int i; |
1221 | 1221 | ||
1222 | raw_spin_lock(&gsnedf_lock); | 1222 | raw_spin_lock(&gsnedf_lock); |
1223 | __decrease_priority_inheritance(t, prio_inh); | 1223 | __decrease_priority_inheritance(t, prio_inh); |
1224 | 1224 | ||
1225 | #ifdef CONFIG_LITMUS_SOFTIRQD | 1225 | #ifdef CONFIG_LITMUS_SOFTIRQD |
1226 | if(tsk_rt(t)->cur_klitirqd != NULL) | 1226 | if(tsk_rt(t)->cur_klitirqd != NULL) |
1227 | { | 1227 | { |
1228 | TRACE_TASK(t, "%s/%d decreases in priority!\n", | 1228 | TRACE_TASK(t, "%s/%d decreases in priority!\n", |
1229 | tsk_rt(t)->cur_klitirqd->comm, tsk_rt(t)->cur_klitirqd->pid); | 1229 | tsk_rt(t)->cur_klitirqd->comm, tsk_rt(t)->cur_klitirqd->pid); |
1230 | 1230 | ||
1231 | __decrease_priority_inheritance(tsk_rt(t)->cur_klitirqd, prio_inh); | 1231 | __decrease_priority_inheritance(tsk_rt(t)->cur_klitirqd, prio_inh); |
1232 | } | 1232 | } |
1233 | #endif | 1233 | #endif |
1234 | 1234 | ||
1235 | raw_spin_unlock(&gsnedf_lock); | 1235 | raw_spin_unlock(&gsnedf_lock); |
1236 | 1236 | ||
1237 | #if defined(CONFIG_LITMUS_PAI_SOFTIRQD) && defined(CONFIG_LITMUS_NVIDIA) | 1237 | #if defined(CONFIG_LITMUS_PAI_SOFTIRQD) && defined(CONFIG_LITMUS_NVIDIA) |
1238 | if(tsk_rt(t)->held_gpus) { | 1238 | if(tsk_rt(t)->held_gpus) { |
1239 | for(i = find_first_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus)); | 1239 | for(i = find_first_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus)); |
@@ -1242,7 +1242,7 @@ static void decrease_priority_inheritance(struct task_struct* t, | |||
1242 | pai_check_priority_decrease(t, i); | 1242 | pai_check_priority_decrease(t, i); |
1243 | } | 1243 | } |
1244 | } | 1244 | } |
1245 | #endif | 1245 | #endif |
1246 | } | 1246 | } |
1247 | 1247 | ||
1248 | 1248 | ||
@@ -1253,9 +1253,9 @@ static void increase_priority_inheritance_klitirqd(struct task_struct* klitirqd, | |||
1253 | struct task_struct* new_owner) | 1253 | struct task_struct* new_owner) |
1254 | { | 1254 | { |
1255 | BUG_ON(!(tsk_rt(klitirqd)->is_proxy_thread)); | 1255 | BUG_ON(!(tsk_rt(klitirqd)->is_proxy_thread)); |
1256 | 1256 | ||
1257 | raw_spin_lock(&gsnedf_lock); | 1257 | raw_spin_lock(&gsnedf_lock); |
1258 | 1258 | ||
1259 | if(old_owner != new_owner) | 1259 | if(old_owner != new_owner) |
1260 | { | 1260 | { |
1261 | if(old_owner) | 1261 | if(old_owner) |
@@ -1263,20 +1263,20 @@ static void increase_priority_inheritance_klitirqd(struct task_struct* klitirqd, | |||
1263 | // unreachable? | 1263 | // unreachable? |
1264 | tsk_rt(old_owner)->cur_klitirqd = NULL; | 1264 | tsk_rt(old_owner)->cur_klitirqd = NULL; |
1265 | } | 1265 | } |
1266 | 1266 | ||
1267 | TRACE_TASK(klitirqd, "giving ownership to %s/%d.\n", | 1267 | TRACE_TASK(klitirqd, "giving ownership to %s/%d.\n", |
1268 | new_owner->comm, new_owner->pid); | 1268 | new_owner->comm, new_owner->pid); |
1269 | 1269 | ||
1270 | tsk_rt(new_owner)->cur_klitirqd = klitirqd; | 1270 | tsk_rt(new_owner)->cur_klitirqd = klitirqd; |
1271 | } | 1271 | } |
1272 | 1272 | ||
1273 | __decrease_priority_inheritance(klitirqd, NULL); // kludge to clear out cur prio. | 1273 | __decrease_priority_inheritance(klitirqd, NULL); // kludge to clear out cur prio. |
1274 | 1274 | ||
1275 | __increase_priority_inheritance(klitirqd, | 1275 | __increase_priority_inheritance(klitirqd, |
1276 | (tsk_rt(new_owner)->inh_task == NULL) ? | 1276 | (tsk_rt(new_owner)->inh_task == NULL) ? |
1277 | new_owner : | 1277 | new_owner : |
1278 | tsk_rt(new_owner)->inh_task); | 1278 | tsk_rt(new_owner)->inh_task); |
1279 | 1279 | ||
1280 | raw_spin_unlock(&gsnedf_lock); | 1280 | raw_spin_unlock(&gsnedf_lock); |
1281 | } | 1281 | } |
1282 | 1282 | ||
@@ -1287,15 +1287,15 @@ static void decrease_priority_inheritance_klitirqd(struct task_struct* klitirqd, | |||
1287 | struct task_struct* new_owner) | 1287 | struct task_struct* new_owner) |
1288 | { | 1288 | { |
1289 | BUG_ON(!(tsk_rt(klitirqd)->is_proxy_thread)); | 1289 | BUG_ON(!(tsk_rt(klitirqd)->is_proxy_thread)); |
1290 | 1290 | ||
1291 | raw_spin_lock(&gsnedf_lock); | 1291 | raw_spin_lock(&gsnedf_lock); |
1292 | 1292 | ||
1293 | TRACE_TASK(klitirqd, "priority restored\n"); | 1293 | TRACE_TASK(klitirqd, "priority restored\n"); |
1294 | 1294 | ||
1295 | __decrease_priority_inheritance(klitirqd, new_owner); | 1295 | __decrease_priority_inheritance(klitirqd, new_owner); |
1296 | 1296 | ||
1297 | tsk_rt(old_owner)->cur_klitirqd = NULL; | 1297 | tsk_rt(old_owner)->cur_klitirqd = NULL; |
1298 | 1298 | ||
1299 | raw_spin_unlock(&gsnedf_lock); | 1299 | raw_spin_unlock(&gsnedf_lock); |
1300 | } | 1300 | } |
1301 | #endif | 1301 | #endif |
@@ -1435,10 +1435,10 @@ static struct litmus_lock_ops gsnedf_kfmlp_lock_ops = { | |||
1435 | .unlock = kfmlp_unlock, | 1435 | .unlock = kfmlp_unlock, |
1436 | .close = kfmlp_close, | 1436 | .close = kfmlp_close, |
1437 | .deallocate = kfmlp_free, | 1437 | .deallocate = kfmlp_free, |
1438 | 1438 | ||
1439 | // kfmlp can only be an outer-most lock. | 1439 | // kfmlp can only be an outer-most lock. |
1440 | .propagate_increase_inheritance = NULL, | 1440 | .propagate_increase_inheritance = NULL, |
1441 | .propagate_decrease_inheritance = NULL, | 1441 | .propagate_decrease_inheritance = NULL, |
1442 | }; | 1442 | }; |
1443 | 1443 | ||
1444 | 1444 | ||
@@ -1709,16 +1709,16 @@ static long gsnedf_allocate_affinity_observer( | |||
1709 | void* __user args) | 1709 | void* __user args) |
1710 | { | 1710 | { |
1711 | int err; | 1711 | int err; |
1712 | 1712 | ||
1713 | /* GSN-EDF currently only supports the FMLP for global resources. */ | 1713 | /* GSN-EDF currently only supports the FMLP for global resources. */ |
1714 | switch (type) { | 1714 | switch (type) { |
1715 | 1715 | ||
1716 | case KFMLP_SIMPLE_GPU_AFF_OBS: | 1716 | case KFMLP_SIMPLE_GPU_AFF_OBS: |
1717 | *aff_obs = kfmlp_simple_gpu_aff_obs_new(&gsnedf_kfmlp_affinity_ops, args); | 1717 | *aff_obs = kfmlp_simple_gpu_aff_obs_new(&gsnedf_kfmlp_affinity_ops, args); |
1718 | break; | 1718 | break; |
1719 | case KFMLP_GPU_AFF_OBS: | 1719 | case KFMLP_GPU_AFF_OBS: |
1720 | *aff_obs = kfmlp_gpu_aff_obs_new(&gsnedf_kfmlp_affinity_ops, args); | 1720 | *aff_obs = kfmlp_gpu_aff_obs_new(&gsnedf_kfmlp_affinity_ops, args); |
1721 | break; | 1721 | break; |
1722 | #ifdef CONFIG_LITMUS_NESTED_LOCKING | 1722 | #ifdef CONFIG_LITMUS_NESTED_LOCKING |
1723 | // case IKGLP_GPU_AFF_OBS: | 1723 | // case IKGLP_GPU_AFF_OBS: |
1724 | // *aff_obs = gsnedf_new_ikglp_aff(arg); | 1724 | // *aff_obs = gsnedf_new_ikglp_aff(arg); |
@@ -1728,12 +1728,12 @@ static long gsnedf_allocate_affinity_observer( | |||
1728 | err = -ENXIO; | 1728 | err = -ENXIO; |
1729 | goto UNSUPPORTED_AFF_OBS; | 1729 | goto UNSUPPORTED_AFF_OBS; |
1730 | }; | 1730 | }; |
1731 | 1731 | ||
1732 | if (*aff_obs) | 1732 | if (*aff_obs) |
1733 | err = 0; | 1733 | err = 0; |
1734 | else | 1734 | else |
1735 | err = -ENOMEM; | 1735 | err = -ENOMEM; |
1736 | 1736 | ||
1737 | UNSUPPORTED_AFF_OBS: | 1737 | UNSUPPORTED_AFF_OBS: |
1738 | return err; | 1738 | return err; |
1739 | } | 1739 | } |
@@ -1769,12 +1769,12 @@ static long gsnedf_activate_plugin(void) | |||
1769 | } | 1769 | } |
1770 | #endif | 1770 | #endif |
1771 | } | 1771 | } |
1772 | 1772 | ||
1773 | #ifdef CONFIG_LITMUS_PAI_SOFTIRQD | 1773 | #ifdef CONFIG_LITMUS_PAI_SOFTIRQD |
1774 | gsnedf_pending_tasklets.head = NULL; | 1774 | gsnedf_pending_tasklets.head = NULL; |
1775 | gsnedf_pending_tasklets.tail = &(gsnedf_pending_tasklets.head); | 1775 | gsnedf_pending_tasklets.tail = &(gsnedf_pending_tasklets.head); |
1776 | #endif | 1776 | #endif |
1777 | 1777 | ||
1778 | #ifdef CONFIG_LITMUS_SOFTIRQD | 1778 | #ifdef CONFIG_LITMUS_SOFTIRQD |
1779 | spawn_klitirqd(NULL); | 1779 | spawn_klitirqd(NULL); |
1780 | #endif | 1780 | #endif |
@@ -1782,7 +1782,7 @@ static long gsnedf_activate_plugin(void) | |||
1782 | #ifdef CONFIG_LITMUS_NVIDIA | 1782 | #ifdef CONFIG_LITMUS_NVIDIA |
1783 | init_nvidia_info(); | 1783 | init_nvidia_info(); |
1784 | #endif | 1784 | #endif |
1785 | 1785 | ||
1786 | return 0; | 1786 | return 0; |
1787 | } | 1787 | } |
1788 | 1788 | ||
@@ -1815,7 +1815,7 @@ static struct sched_plugin gsn_edf_plugin __cacheline_aligned_in_smp = { | |||
1815 | #endif | 1815 | #endif |
1816 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING | 1816 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING |
1817 | .allocate_aff_obs = gsnedf_allocate_affinity_observer, | 1817 | .allocate_aff_obs = gsnedf_allocate_affinity_observer, |
1818 | #endif | 1818 | #endif |
1819 | #ifdef CONFIG_LITMUS_SOFTIRQD | 1819 | #ifdef CONFIG_LITMUS_SOFTIRQD |
1820 | .increase_prio_klitirqd = increase_priority_inheritance_klitirqd, | 1820 | .increase_prio_klitirqd = increase_priority_inheritance_klitirqd, |
1821 | .decrease_prio_klitirqd = decrease_priority_inheritance_klitirqd, | 1821 | .decrease_prio_klitirqd = decrease_priority_inheritance_klitirqd, |
diff --git a/litmus/sched_plugin.c b/litmus/sched_plugin.c index a334fdf66c3b..2433297b7482 100644 --- a/litmus/sched_plugin.c +++ b/litmus/sched_plugin.c | |||
@@ -240,8 +240,8 @@ struct sched_plugin linux_sched_plugin = { | |||
240 | #endif | 240 | #endif |
241 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING | 241 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING |
242 | .allocate_aff_obs = litmus_dummy_allocate_aff_obs, | 242 | .allocate_aff_obs = litmus_dummy_allocate_aff_obs, |
243 | #endif | 243 | #endif |
244 | 244 | ||
245 | .admit_task = litmus_dummy_admit_task | 245 | .admit_task = litmus_dummy_admit_task |
246 | }; | 246 | }; |
247 | 247 | ||
@@ -302,7 +302,7 @@ int register_sched_plugin(struct sched_plugin* plugin) | |||
302 | #endif | 302 | #endif |
303 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING | 303 | #ifdef CONFIG_LITMUS_AFFINITY_LOCKING |
304 | CHECK(allocate_aff_obs); | 304 | CHECK(allocate_aff_obs); |
305 | #endif | 305 | #endif |
306 | CHECK(admit_task); | 306 | CHECK(admit_task); |
307 | 307 | ||
308 | if (!plugin->release_at) | 308 | if (!plugin->release_at) |
diff --git a/litmus/sched_task_trace.c b/litmus/sched_task_trace.c index d079df2b292a..2bd3a787611b 100644 --- a/litmus/sched_task_trace.c +++ b/litmus/sched_task_trace.c | |||
@@ -252,7 +252,7 @@ feather_callback void do_sched_trace_tasklet_release(unsigned long id, | |||
252 | { | 252 | { |
253 | struct task_struct *t = (struct task_struct*) _owner; | 253 | struct task_struct *t = (struct task_struct*) _owner; |
254 | struct st_event_record *rec = get_record(ST_TASKLET_RELEASE, t); | 254 | struct st_event_record *rec = get_record(ST_TASKLET_RELEASE, t); |
255 | 255 | ||
256 | if (rec) { | 256 | if (rec) { |
257 | rec->data.tasklet_release.when = now(); | 257 | rec->data.tasklet_release.when = now(); |
258 | put_record(rec); | 258 | put_record(rec); |
@@ -265,7 +265,7 @@ feather_callback void do_sched_trace_tasklet_begin(unsigned long id, | |||
265 | { | 265 | { |
266 | struct task_struct *t = (struct task_struct*) _owner; | 266 | struct task_struct *t = (struct task_struct*) _owner; |
267 | struct st_event_record *rec = get_record(ST_TASKLET_BEGIN, t); | 267 | struct st_event_record *rec = get_record(ST_TASKLET_BEGIN, t); |
268 | 268 | ||
269 | if (rec) { | 269 | if (rec) { |
270 | rec->data.tasklet_begin.when = now(); | 270 | rec->data.tasklet_begin.when = now(); |
271 | 271 | ||
@@ -286,7 +286,7 @@ feather_callback void do_sched_trace_tasklet_end(unsigned long id, | |||
286 | { | 286 | { |
287 | struct task_struct *t = (struct task_struct*) _owner; | 287 | struct task_struct *t = (struct task_struct*) _owner; |
288 | struct st_event_record *rec = get_record(ST_TASKLET_END, t); | 288 | struct st_event_record *rec = get_record(ST_TASKLET_END, t); |
289 | 289 | ||
290 | if (rec) { | 290 | if (rec) { |
291 | rec->data.tasklet_end.when = now(); | 291 | rec->data.tasklet_end.when = now(); |
292 | rec->data.tasklet_end.flushed = _flushed; | 292 | rec->data.tasklet_end.flushed = _flushed; |
@@ -307,7 +307,7 @@ feather_callback void do_sched_trace_work_release(unsigned long id, | |||
307 | { | 307 | { |
308 | struct task_struct *t = (struct task_struct*) _owner; | 308 | struct task_struct *t = (struct task_struct*) _owner; |
309 | struct st_event_record *rec = get_record(ST_WORK_RELEASE, t); | 309 | struct st_event_record *rec = get_record(ST_WORK_RELEASE, t); |
310 | 310 | ||
311 | if (rec) { | 311 | if (rec) { |
312 | rec->data.work_release.when = now(); | 312 | rec->data.work_release.when = now(); |
313 | put_record(rec); | 313 | put_record(rec); |
@@ -321,7 +321,7 @@ feather_callback void do_sched_trace_work_begin(unsigned long id, | |||
321 | { | 321 | { |
322 | struct task_struct *t = (struct task_struct*) _owner; | 322 | struct task_struct *t = (struct task_struct*) _owner; |
323 | struct st_event_record *rec = get_record(ST_WORK_BEGIN, t); | 323 | struct st_event_record *rec = get_record(ST_WORK_BEGIN, t); |
324 | 324 | ||
325 | if (rec) { | 325 | if (rec) { |
326 | struct task_struct *exe = (struct task_struct*) _exe; | 326 | struct task_struct *exe = (struct task_struct*) _exe; |
327 | rec->data.work_begin.exe_pid = exe->pid; | 327 | rec->data.work_begin.exe_pid = exe->pid; |
@@ -339,7 +339,7 @@ feather_callback void do_sched_trace_work_end(unsigned long id, | |||
339 | { | 339 | { |
340 | struct task_struct *t = (struct task_struct*) _owner; | 340 | struct task_struct *t = (struct task_struct*) _owner; |
341 | struct st_event_record *rec = get_record(ST_WORK_END, t); | 341 | struct st_event_record *rec = get_record(ST_WORK_END, t); |
342 | 342 | ||
343 | if (rec) { | 343 | if (rec) { |
344 | struct task_struct *exe = (struct task_struct*) _exe; | 344 | struct task_struct *exe = (struct task_struct*) _exe; |
345 | rec->data.work_end.exe_pid = exe->pid; | 345 | rec->data.work_end.exe_pid = exe->pid; |
@@ -357,14 +357,14 @@ feather_callback void do_sched_trace_eff_prio_change(unsigned long id, | |||
357 | { | 357 | { |
358 | struct task_struct *t = (struct task_struct*) _task; | 358 | struct task_struct *t = (struct task_struct*) _task; |
359 | struct st_event_record *rec = get_record(ST_EFF_PRIO_CHANGE, t); | 359 | struct st_event_record *rec = get_record(ST_EFF_PRIO_CHANGE, t); |
360 | 360 | ||
361 | if (rec) { | 361 | if (rec) { |
362 | struct task_struct *inh = (struct task_struct*) _inh; | 362 | struct task_struct *inh = (struct task_struct*) _inh; |
363 | rec->data.effective_priority_change.when = now(); | 363 | rec->data.effective_priority_change.when = now(); |
364 | rec->data.effective_priority_change.inh_pid = (inh != NULL) ? | 364 | rec->data.effective_priority_change.inh_pid = (inh != NULL) ? |
365 | inh->pid : | 365 | inh->pid : |
366 | 0xffff; | 366 | 0xffff; |
367 | 367 | ||
368 | put_record(rec); | 368 | put_record(rec); |
369 | } | 369 | } |
370 | } | 370 | } |
@@ -401,7 +401,7 @@ feather_callback void do_sched_trace_nv_interrupt_begin(unsigned long id, | |||
401 | int_map->count = 1; | 401 | int_map->count = 1; |
402 | } | 402 | } |
403 | //int_map->data[int_map->count-1] = _device; | 403 | //int_map->data[int_map->count-1] = _device; |
404 | 404 | ||
405 | serial = &per_cpu(intCounter, smp_processor_id()); | 405 | serial = &per_cpu(intCounter, smp_processor_id()); |
406 | *serial += num_online_cpus(); | 406 | *serial += num_online_cpus(); |
407 | serialNum = *serial; | 407 | serialNum = *serial; |