aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--include/litmus/edf_common.h6
-rw-r--r--include/litmus/nvidia_info.h10
-rw-r--r--include/litmus/rt_param.h1
-rw-r--r--include/litmus/sched_plugin.h22
-rw-r--r--kernel/softirq.c6
-rw-r--r--kernel/workqueue.c2
-rw-r--r--litmus/ikglp_lock.c64
-rw-r--r--litmus/kfmlp_lock.c14
-rw-r--r--litmus/litmus.c4
-rw-r--r--litmus/nvidia_info.c248
-rw-r--r--litmus/rsm_lock.c35
-rw-r--r--litmus/sched_gsn_edf.c203
-rw-r--r--litmus/sched_plugin.c23
13 files changed, 412 insertions, 226 deletions
diff --git a/include/litmus/edf_common.h b/include/litmus/edf_common.h
index 818f4094b53c..63dff7efe8fb 100644
--- a/include/litmus/edf_common.h
+++ b/include/litmus/edf_common.h
@@ -27,12 +27,6 @@ int edf_min_heap_order(struct binheap_node *a, struct binheap_node *b);
27int edf_max_heap_base_priority_order(struct binheap_node *a, struct binheap_node *b); 27int edf_max_heap_base_priority_order(struct binheap_node *a, struct binheap_node *b);
28int edf_min_heap_base_priority_order(struct binheap_node *a, struct binheap_node *b); 28int edf_min_heap_base_priority_order(struct binheap_node *a, struct binheap_node *b);
29 29
30typedef enum
31{
32 BASE,
33 EFFECTIVE
34} comparison_mode_t;
35
36int __edf_higher_prio(struct task_struct* first, comparison_mode_t first_mode, 30int __edf_higher_prio(struct task_struct* first, comparison_mode_t first_mode,
37 struct task_struct* second, comparison_mode_t second_mode); 31 struct task_struct* second, comparison_mode_t second_mode);
38 32
diff --git a/include/litmus/nvidia_info.h b/include/litmus/nvidia_info.h
index 9e07a27fdee3..dd41c4c72b85 100644
--- a/include/litmus/nvidia_info.h
+++ b/include/litmus/nvidia_info.h
@@ -28,11 +28,17 @@ int init_nv_device_reg(void);
28 28
29int reg_nv_device(int reg_device_id, int register_device); 29int reg_nv_device(int reg_device_id, int register_device);
30 30
31struct task_struct* get_nv_device_owner(u32 target_device_id); 31struct task_struct* get_nv_max_device_owner(u32 target_device_id);
32//int is_nv_device_owner(u32 target_device_id);
32 33
33void lock_nv_registry(u32 reg_device_id, unsigned long* flags); 34void lock_nv_registry(u32 reg_device_id, unsigned long* flags);
34void unlock_nv_registry(u32 reg_device_id, unsigned long* flags); 35void unlock_nv_registry(u32 reg_device_id, unsigned long* flags);
35 36
36void increment_nv_int_count(u32 device); 37#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
38void pai_check_priority_increase(struct task_struct *t, int reg_device_id);
39void pai_check_priority_decrease(struct task_struct *t, int reg_device_id);
40#endif
41
42//void increment_nv_int_count(u32 device);
37 43
38#endif 44#endif
diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h
index d0040bfd2d0c..b4eb8ee95687 100644
--- a/include/litmus/rt_param.h
+++ b/include/litmus/rt_param.h
@@ -154,6 +154,7 @@ struct rt_param {
154#ifdef CONFIG_LITMUS_NVIDIA 154#ifdef CONFIG_LITMUS_NVIDIA
155 /* number of top-half interrupts handled on behalf of current job */ 155 /* number of top-half interrupts handled on behalf of current job */
156 atomic_t nv_int_count; 156 atomic_t nv_int_count;
157 long unsigned int held_gpus; // bitmap of held GPUs.
157#endif 158#endif
158 159
159#ifdef CONFIG_LITMUS_LOCKING 160#ifdef CONFIG_LITMUS_LOCKING
diff --git a/include/litmus/sched_plugin.h b/include/litmus/sched_plugin.h
index e31008fcdd59..8e65555d9b7f 100644
--- a/include/litmus/sched_plugin.h
+++ b/include/litmus/sched_plugin.h
@@ -74,10 +74,28 @@ typedef void (*decrease_prio_klitirqd_t)(struct task_struct* klitirqd,
74 74
75 75
76typedef int (*enqueue_pai_tasklet_t)(struct tasklet_struct* tasklet); 76typedef int (*enqueue_pai_tasklet_t)(struct tasklet_struct* tasklet);
77typedef void (*change_prio_pai_tasklet_t)(struct task_struct *old_prio,
78 struct task_struct *new_prio);
77typedef void (*run_tasklets_t)(struct task_struct* next); 79typedef void (*run_tasklets_t)(struct task_struct* next);
78 80
79typedef raw_spinlock_t* (*get_dgl_spinlock_t) (struct task_struct *t); 81typedef raw_spinlock_t* (*get_dgl_spinlock_t) (struct task_struct *t);
80 82
83
84typedef int (*higher_prio_t)(struct task_struct* a, struct task_struct* b);
85
86#ifdef CONFIG_LITMUS_NESTED_LOCKING
87
88typedef enum
89{
90 BASE,
91 EFFECTIVE
92} comparison_mode_t;
93
94typedef int (*__higher_prio_t)(struct task_struct* a, comparison_mode_t a_mod,
95 struct task_struct* b, comparison_mode_t b_mod);
96#endif
97
98
81/********************* sys call backends ********************/ 99/********************* sys call backends ********************/
82/* This function causes the caller to sleep until the next release */ 100/* This function causes the caller to sleep until the next release */
83typedef long (*complete_job_t) (void); 101typedef long (*complete_job_t) (void);
@@ -112,6 +130,8 @@ struct sched_plugin {
112 task_block_t task_block; 130 task_block_t task_block;
113 task_exit_t task_exit; 131 task_exit_t task_exit;
114 132
133 higher_prio_t compare;
134
115#ifdef CONFIG_LITMUS_LOCKING 135#ifdef CONFIG_LITMUS_LOCKING
116 /* locking protocols */ 136 /* locking protocols */
117 allocate_lock_t allocate_lock; 137 allocate_lock_t allocate_lock;
@@ -121,6 +141,7 @@ struct sched_plugin {
121#ifdef CONFIG_LITMUS_NESTED_LOCKING 141#ifdef CONFIG_LITMUS_NESTED_LOCKING
122 nested_increase_prio_t nested_increase_prio; 142 nested_increase_prio_t nested_increase_prio;
123 nested_decrease_prio_t nested_decrease_prio; 143 nested_decrease_prio_t nested_decrease_prio;
144 __higher_prio_t __compare;
124#endif 145#endif
125#ifdef CONFIG_LITMUS_DGL_SUPPORT 146#ifdef CONFIG_LITMUS_DGL_SUPPORT
126 get_dgl_spinlock_t get_dgl_spinlock; 147 get_dgl_spinlock_t get_dgl_spinlock;
@@ -132,6 +153,7 @@ struct sched_plugin {
132#endif 153#endif
133#ifdef CONFIG_LITMUS_PAI_SOFTIRQD 154#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
134 enqueue_pai_tasklet_t enqueue_pai_tasklet; 155 enqueue_pai_tasklet_t enqueue_pai_tasklet;
156 change_prio_pai_tasklet_t change_prio_pai_tasklet;
135 run_tasklets_t run_tasklets; 157 run_tasklets_t run_tasklets;
136#endif 158#endif
137} __attribute__ ((__aligned__(SMP_CACHE_BYTES))); 159} __attribute__ ((__aligned__(SMP_CACHE_BYTES)));
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 7c562558a863..1c42e08fdfaa 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -427,7 +427,7 @@ void __tasklet_schedule(struct tasklet_struct *t)
427 427
428 lock_nv_registry(nvidia_device, &flags); 428 lock_nv_registry(nvidia_device, &flags);
429 429
430 device_owner = get_nv_device_owner(nvidia_device); 430 device_owner = get_nv_max_device_owner(nvidia_device);
431 431
432 if(device_owner==NULL) 432 if(device_owner==NULL)
433 { 433 {
@@ -497,7 +497,7 @@ void __tasklet_hi_schedule(struct tasklet_struct *t)
497 497
498 lock_nv_registry(nvidia_device, &flags); 498 lock_nv_registry(nvidia_device, &flags);
499 499
500 device_owner = get_nv_device_owner(nvidia_device); 500 device_owner = get_nv_max_device_owner(nvidia_device);
501 501
502 if(device_owner==NULL) 502 if(device_owner==NULL)
503 { 503 {
@@ -564,7 +564,7 @@ void __tasklet_hi_schedule_first(struct tasklet_struct *t)
564 564
565 lock_nv_registry(nvidia_device, &flags); 565 lock_nv_registry(nvidia_device, &flags);
566 566
567 device_owner = get_nv_device_owner(nvidia_device); 567 device_owner = get_nv_max_device_owner(nvidia_device);
568 568
569 if(device_owner==NULL) 569 if(device_owner==NULL)
570 { 570 {
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 2ceb7b43a045..6b59d59ce3cf 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -2704,7 +2704,7 @@ int schedule_work(struct work_struct *work)
2704 2704
2705 lock_nv_registry(nvidiaDevice, &flags); 2705 lock_nv_registry(nvidiaDevice, &flags);
2706 2706
2707 device_owner = get_nv_device_owner(nvidiaDevice); 2707 device_owner = get_nv_max_device_owner(nvidiaDevice);
2708 2708
2709 //2) If there is an owner, set work->owner to the owner's task struct. 2709 //2) If there is an owner, set work->owner to the owner's task struct.
2710 if(device_owner==NULL) 2710 if(device_owner==NULL)
diff --git a/litmus/ikglp_lock.c b/litmus/ikglp_lock.c
index 0ae9994111fb..a41a9d9a3627 100644
--- a/litmus/ikglp_lock.c
+++ b/litmus/ikglp_lock.c
@@ -5,9 +5,9 @@
5#include <litmus/sched_plugin.h> 5#include <litmus/sched_plugin.h>
6#include <litmus/ikglp_lock.h> 6#include <litmus/ikglp_lock.h>
7 7
8#include <litmus/edf_common.h> 8//#include <litmus/edf_common.h>
9 9
10int ikglp_edf_max_heap_base_priority_order(struct binheap_node *a, 10int ikglp_max_heap_base_priority_order(struct binheap_node *a,
11 struct binheap_node *b) 11 struct binheap_node *b)
12{ 12{
13 ikglp_heap_node_t *d_a = binheap_entry(a, ikglp_heap_node_t, node); 13 ikglp_heap_node_t *d_a = binheap_entry(a, ikglp_heap_node_t, node);
@@ -16,29 +16,32 @@ int ikglp_edf_max_heap_base_priority_order(struct binheap_node *a,
16 BUG_ON(!d_a); 16 BUG_ON(!d_a);
17 BUG_ON(!d_b); 17 BUG_ON(!d_b);
18 18
19 return __edf_higher_prio(d_a->task, BASE, d_b->task, BASE); 19 //return __edf_higher_prio(d_a->task, BASE, d_b->task, BASE);
20 return litmus->__compare(d_a->task, BASE, d_b->task, BASE);
20} 21}
21 22
22int ikglp_edf_min_heap_base_priority_order(struct binheap_node *a, 23int ikglp_min_heap_base_priority_order(struct binheap_node *a,
23 struct binheap_node *b) 24 struct binheap_node *b)
24{ 25{
25 ikglp_heap_node_t *d_a = binheap_entry(a, ikglp_heap_node_t, node); 26 ikglp_heap_node_t *d_a = binheap_entry(a, ikglp_heap_node_t, node);
26 ikglp_heap_node_t *d_b = binheap_entry(b, ikglp_heap_node_t, node); 27 ikglp_heap_node_t *d_b = binheap_entry(b, ikglp_heap_node_t, node);
27 28
28 return __edf_higher_prio(d_b->task, BASE, d_a->task, BASE); 29 //return __edf_higher_prio(d_b->task, BASE, d_a->task, BASE);
30 return litmus->__compare(d_b->task, BASE, d_a->task, BASE);
29} 31}
30 32
31int ikglp_donor_edf_max_heap_base_priority_order(struct binheap_node *a, 33int ikglp_donor_max_heap_base_priority_order(struct binheap_node *a,
32 struct binheap_node *b) 34 struct binheap_node *b)
33{ 35{
34 ikglp_wait_state_t *d_a = binheap_entry(a, ikglp_wait_state_t, node); 36 ikglp_wait_state_t *d_a = binheap_entry(a, ikglp_wait_state_t, node);
35 ikglp_wait_state_t *d_b = binheap_entry(b, ikglp_wait_state_t, node); 37 ikglp_wait_state_t *d_b = binheap_entry(b, ikglp_wait_state_t, node);
36 38
37 return __edf_higher_prio(d_a->task, BASE, d_b->task, BASE); 39 //return __edf_higher_prio(d_a->task, BASE, d_b->task, BASE);
40 return litmus->__compare(d_a->task, BASE, d_b->task, BASE);
38} 41}
39 42
40 43
41int ikglp_edf_min_heap_donee_order(struct binheap_node *a, 44int ikglp_min_heap_donee_order(struct binheap_node *a,
42 struct binheap_node *b) 45 struct binheap_node *b)
43{ 46{
44 struct task_struct *prio_a, *prio_b; 47 struct task_struct *prio_a, *prio_b;
@@ -65,7 +68,8 @@ int ikglp_edf_min_heap_donee_order(struct binheap_node *a,
65 } 68 }
66 69
67 // note reversed order 70 // note reversed order
68 return __edf_higher_prio(prio_b, BASE, prio_a, BASE); 71 //return __edf_higher_prio(prio_b, BASE, prio_a, BASE);
72 return litmus->__compare(prio_b, BASE, prio_a, BASE);
69} 73}
70 74
71 75
@@ -99,7 +103,8 @@ static struct task_struct* ikglp_find_hp_waiter(struct fifo_queue *kqueue,
99 wait_queue_t, task_list)->private; 103 wait_queue_t, task_list)->private;
100 104
101 /* Compare task prios, find high prio task. */ 105 /* Compare task prios, find high prio task. */
102 if (queued != skip && edf_higher_prio(queued, found)) 106 //if (queued != skip && edf_higher_prio(queued, found))
107 if(queued != skip && litmus->compare(queued, found))
103 found = queued; 108 found = queued;
104 } 109 }
105 return found; 110 return found;
@@ -241,7 +246,8 @@ static void ikglp_add_global_list(struct ikglp_semaphore *sem,
241// TRACE_CUR("Top-M After (size = %d):\n", sem->top_m_size); 246// TRACE_CUR("Top-M After (size = %d):\n", sem->top_m_size);
242// print_global_list(sem->top_m.root, 1); 247// print_global_list(sem->top_m.root, 1);
243 } 248 }
244 else if(__edf_higher_prio(t, BASE, ikglp_mth_highest(sem), BASE)) { 249 //else if(__edf_higher_prio(t, BASE, ikglp_mth_highest(sem), BASE)) {
250 else if(litmus->__compare(t, BASE, ikglp_mth_highest(sem), BASE)) {
245 ikglp_heap_node_t *evicted = 251 ikglp_heap_node_t *evicted =
246 binheap_top_entry(&sem->top_m, ikglp_heap_node_t, node); 252 binheap_top_entry(&sem->top_m, ikglp_heap_node_t, node);
247 253
@@ -361,7 +367,8 @@ static void ikglp_refresh_owners_prio_increase(struct task_struct *t,
361 unsigned long flags) 367 unsigned long flags)
362{ 368{
363 // priority of 't' has increased (note: 't' might already be hp_waiter). 369 // priority of 't' has increased (note: 't' might already be hp_waiter).
364 if ((t == fq->hp_waiter) || edf_higher_prio(t, fq->hp_waiter)) { 370 // if ((t == fq->hp_waiter) || edf_higher_prio(t, fq->hp_waiter)) {
371 if ((t == fq->hp_waiter) || litmus->compare(t, fq->hp_waiter)) {
365 struct task_struct *old_max_eff_prio; 372 struct task_struct *old_max_eff_prio;
366 struct task_struct *new_max_eff_prio; 373 struct task_struct *new_max_eff_prio;
367 struct task_struct *new_prio = NULL; 374 struct task_struct *new_prio = NULL;
@@ -397,7 +404,7 @@ static void ikglp_refresh_owners_prio_increase(struct task_struct *t,
397 TRACE_TASK(t, "is new hp_waiter.\n"); 404 TRACE_TASK(t, "is new hp_waiter.\n");
398 405
399 if ((effective_priority(owner) == old_max_eff_prio) || 406 if ((effective_priority(owner) == old_max_eff_prio) ||
400 (__edf_higher_prio(new_max_eff_prio, BASE, 407 (litmus->__compare(new_max_eff_prio, BASE,
401 owner, EFFECTIVE))){ 408 owner, EFFECTIVE))){
402 new_prio = new_max_eff_prio; 409 new_prio = new_max_eff_prio;
403 } 410 }
@@ -471,7 +478,8 @@ static void ikglp_refresh_owners_prio_decrease(struct fifo_queue *fq,
471 TRACE_CUR("Propagating decreased inheritance to holder of fq %d.\n", 478 TRACE_CUR("Propagating decreased inheritance to holder of fq %d.\n",
472 ikglp_get_idx(sem, fq)); 479 ikglp_get_idx(sem, fq));
473 480
474 if(__edf_higher_prio(new_max_eff_prio, BASE, owner, BASE)) { 481 //if(__edf_higher_prio(new_max_eff_prio, BASE, owner, BASE)) {
482 if(litmus->__compare(new_max_eff_prio, BASE, owner, BASE)) {
475 TRACE_CUR("%s/%d has greater base priority than base priority of owner (%s/%d) of fq %d.\n", 483 TRACE_CUR("%s/%d has greater base priority than base priority of owner (%s/%d) of fq %d.\n",
476 (new_max_eff_prio) ? new_max_eff_prio->comm : "nil", 484 (new_max_eff_prio) ? new_max_eff_prio->comm : "nil",
477 (new_max_eff_prio) ? new_max_eff_prio->pid : -1, 485 (new_max_eff_prio) ? new_max_eff_prio->pid : -1,
@@ -532,7 +540,8 @@ static void ikglp_remove_donation_from_owner(struct binheap_node *n,
532 TRACE_CUR("Propagating decreased inheritance to holder of fq %d.\n", 540 TRACE_CUR("Propagating decreased inheritance to holder of fq %d.\n",
533 ikglp_get_idx(sem, fq)); 541 ikglp_get_idx(sem, fq));
534 542
535 if(__edf_higher_prio(new_max_eff_prio, BASE, owner, BASE)) { 543 //if(__edf_higher_prio(new_max_eff_prio, BASE, owner, BASE)) {
544 if(litmus->__compare(new_max_eff_prio, BASE, owner, BASE)) {
536 TRACE_CUR("has greater base priority than base priority of owner of fq %d.\n", 545 TRACE_CUR("has greater base priority than base priority of owner of fq %d.\n",
537 ikglp_get_idx(sem, fq)); 546 ikglp_get_idx(sem, fq));
538 decreased_prio = new_max_eff_prio; 547 decreased_prio = new_max_eff_prio;
@@ -573,7 +582,8 @@ static void ikglp_remove_donation_from_fq_waiter(struct task_struct *t,
573 // Need to set new effective_priority for owner 582 // Need to set new effective_priority for owner
574 struct task_struct *decreased_prio; 583 struct task_struct *decreased_prio;
575 584
576 if(__edf_higher_prio(new_max_eff_prio, BASE, t, BASE)) { 585 //if(__edf_higher_prio(new_max_eff_prio, BASE, t, BASE)) {
586 if(litmus->__compare(new_max_eff_prio, BASE, t, BASE)) {
577 decreased_prio = new_max_eff_prio; 587 decreased_prio = new_max_eff_prio;
578 } 588 }
579 else { 589 else {
@@ -803,7 +813,8 @@ static void ikglp_enqueue_on_donor(struct ikglp_semaphore *sem,
803 813
804 if(new_max_eff_prio != old_max_eff_prio) { 814 if(new_max_eff_prio != old_max_eff_prio) {
805 if ((effective_priority(donee) == old_max_eff_prio) || 815 if ((effective_priority(donee) == old_max_eff_prio) ||
806 (__edf_higher_prio(new_max_eff_prio, BASE, donee, EFFECTIVE))){ 816 //(__edf_higher_prio(new_max_eff_prio, BASE, donee, EFFECTIVE))){
817 (litmus->__compare(new_max_eff_prio, BASE, donee, EFFECTIVE))){
807 TRACE_TASK(t, "Donation increases %s/%d's effective priority\n", 818 TRACE_TASK(t, "Donation increases %s/%d's effective priority\n",
808 donee->comm, donee->pid); 819 donee->comm, donee->pid);
809 new_prio = new_max_eff_prio; 820 new_prio = new_max_eff_prio;
@@ -907,7 +918,8 @@ int ikglp_lock(struct litmus_lock* l)
907 918
908 // no room in fifos. Go to PQ or donors. 919 // no room in fifos. Go to PQ or donors.
909 920
910 if(__edf_higher_prio(ikglp_mth_highest(sem), BASE, t, BASE)) { 921 //if(__edf_higher_prio(ikglp_mth_highest(sem), BASE, t, BASE)) {
922 if(litmus->__compare(ikglp_mth_highest(sem), BASE, t, BASE)) {
911 // enqueue on PQ 923 // enqueue on PQ
912 ikglp_enqueue_on_pq(sem, &wait); 924 ikglp_enqueue_on_pq(sem, &wait);
913 unlock_fine_irqrestore(&sem->lock, flags); 925 unlock_fine_irqrestore(&sem->lock, flags);
@@ -994,7 +1006,8 @@ static ikglp_wait_state_t* ikglp_find_hp_waiter_to_steal(
994 1006
995 for(i = 0; i < sem->nr_replicas; ++i) { 1007 for(i = 0; i < sem->nr_replicas; ++i) {
996 if( (sem->fifo_queues[i].count > 1) && 1008 if( (sem->fifo_queues[i].count > 1) &&
997 (!fq || edf_higher_prio(sem->fifo_queues[i].hp_waiter, fq->hp_waiter)) ) { 1009 //(!fq || edf_higher_prio(sem->fifo_queues[i].hp_waiter, fq->hp_waiter)) ) {
1010 (!fq || litmus->compare(sem->fifo_queues[i].hp_waiter, fq->hp_waiter)) ) {
998 1011
999 TRACE_CUR("hp_waiter on fq %d (%s/%d) has higher prio than hp_waiter on fq %d (%s/%d)\n", 1012 TRACE_CUR("hp_waiter on fq %d (%s/%d) has higher prio than hp_waiter on fq %d (%s/%d)\n",
1000 ikglp_get_idx(sem, &sem->fifo_queues[i]), 1013 ikglp_get_idx(sem, &sem->fifo_queues[i]),
@@ -1331,7 +1344,8 @@ int ikglp_unlock(struct litmus_lock* l)
1331 fq->hp_waiter->comm, fq->hp_waiter->pid); 1344 fq->hp_waiter->comm, fq->hp_waiter->pid);
1332 fq->nest.hp_waiter_eff_prio = effective_priority(fq->hp_waiter); // set this just to be sure... 1345 fq->nest.hp_waiter_eff_prio = effective_priority(fq->hp_waiter); // set this just to be sure...
1333 } 1346 }
1334 else if(edf_higher_prio(new_on_fq, fq->hp_waiter)) { 1347 //else if(edf_higher_prio(new_on_fq, fq->hp_waiter)) {
1348 else if(litmus->compare(new_on_fq, fq->hp_waiter)) {
1335 if(fq->hp_waiter) 1349 if(fq->hp_waiter)
1336 TRACE_TASK(t, "has higher prio than hp_waiter (%s/%d).\n", 1350 TRACE_TASK(t, "has higher prio than hp_waiter (%s/%d).\n",
1337 fq->hp_waiter->comm, fq->hp_waiter->pid); 1351 fq->hp_waiter->comm, fq->hp_waiter->pid);
@@ -1577,11 +1591,11 @@ struct litmus_lock* ikglp_new(int m,
1577 sem->top_m_size = 0; 1591 sem->top_m_size = 0;
1578 1592
1579 // init heaps 1593 // init heaps
1580 INIT_BINHEAP_HANDLE(&sem->top_m, ikglp_edf_min_heap_base_priority_order); 1594 INIT_BINHEAP_HANDLE(&sem->top_m, ikglp_min_heap_base_priority_order);
1581 INIT_BINHEAP_HANDLE(&sem->not_top_m, ikglp_edf_max_heap_base_priority_order); 1595 INIT_BINHEAP_HANDLE(&sem->not_top_m, ikglp_max_heap_base_priority_order);
1582 INIT_BINHEAP_HANDLE(&sem->donees, ikglp_edf_min_heap_donee_order); 1596 INIT_BINHEAP_HANDLE(&sem->donees, ikglp_min_heap_donee_order);
1583 INIT_BINHEAP_HANDLE(&sem->priority_queue, ikglp_edf_max_heap_base_priority_order); 1597 INIT_BINHEAP_HANDLE(&sem->priority_queue, ikglp_max_heap_base_priority_order);
1584 INIT_BINHEAP_HANDLE(&sem->donors, ikglp_donor_edf_max_heap_base_priority_order); 1598 INIT_BINHEAP_HANDLE(&sem->donors, ikglp_donor_max_heap_base_priority_order);
1585 1599
1586 return &sem->litmus_lock; 1600 return &sem->litmus_lock;
1587} 1601}
diff --git a/litmus/kfmlp_lock.c b/litmus/kfmlp_lock.c
index 37302064bd8c..f7bb17103383 100644
--- a/litmus/kfmlp_lock.c
+++ b/litmus/kfmlp_lock.c
@@ -5,7 +5,7 @@
5#include <litmus/sched_plugin.h> 5#include <litmus/sched_plugin.h>
6#include <litmus/kfmlp_lock.h> 6#include <litmus/kfmlp_lock.h>
7 7
8#include <litmus/edf_common.h> 8//#include <litmus/edf_common.h>
9 9
10static inline int kfmlp_get_idx(struct kfmlp_semaphore* sem, 10static inline int kfmlp_get_idx(struct kfmlp_semaphore* sem,
11 struct kfmlp_queue* queue) 11 struct kfmlp_queue* queue)
@@ -35,7 +35,8 @@ static struct task_struct* kfmlp_find_hp_waiter(struct kfmlp_queue *kqueue,
35 task_list)->private; 35 task_list)->private;
36 36
37 /* Compare task prios, find high prio task. */ 37 /* Compare task prios, find high prio task. */
38 if (queued != skip && edf_higher_prio(queued, found)) 38 //if (queued != skip && edf_higher_prio(queued, found))
39 if (queued != skip && litmus->compare(queued, found))
39 found = queued; 40 found = queued;
40 } 41 }
41 return found; 42 return found;
@@ -82,7 +83,8 @@ static struct task_struct* kfmlp_remove_hp_waiter(struct kfmlp_semaphore* sem)
82 { 83 {
83 if( (sem->queues[i].count > 1) && 84 if( (sem->queues[i].count > 1) &&
84 ((my_queue == NULL) || 85 ((my_queue == NULL) ||
85 (edf_higher_prio(sem->queues[i].hp_waiter, my_queue->hp_waiter))) ) 86 //(edf_higher_prio(sem->queues[i].hp_waiter, my_queue->hp_waiter))) )
87 (litmus->compare(sem->queues[i].hp_waiter, my_queue->hp_waiter))) )
86 { 88 {
87 my_queue = &sem->queues[i]; 89 my_queue = &sem->queues[i];
88 } 90 }
@@ -156,10 +158,12 @@ int kfmlp_lock(struct litmus_lock* l)
156 __add_wait_queue_tail_exclusive(&my_queue->wait, &wait); 158 __add_wait_queue_tail_exclusive(&my_queue->wait, &wait);
157 159
158 /* check if we need to activate priority inheritance */ 160 /* check if we need to activate priority inheritance */
159 if (edf_higher_prio(t, my_queue->hp_waiter)) 161 //if (edf_higher_prio(t, my_queue->hp_waiter))
162 if (litmus->compare(t, my_queue->hp_waiter))
160 { 163 {
161 my_queue->hp_waiter = t; 164 my_queue->hp_waiter = t;
162 if (edf_higher_prio(t, my_queue->owner)) 165 //if (edf_higher_prio(t, my_queue->owner))
166 if (litmus->compare(t, my_queue->owner))
163 { 167 {
164 litmus->increase_prio(my_queue->owner, my_queue->hp_waiter); 168 litmus->increase_prio(my_queue->owner, my_queue->hp_waiter);
165 } 169 }
diff --git a/litmus/litmus.c b/litmus/litmus.c
index 4a40c571d8c6..2f9079421ec7 100644
--- a/litmus/litmus.c
+++ b/litmus/litmus.c
@@ -374,6 +374,10 @@ static void reinit_litmus_state(struct task_struct* p, int restore)
374 up_and_set_stat(p, NOT_HELD, &p->rt_param.klitirqd_sem); 374 up_and_set_stat(p, NOT_HELD, &p->rt_param.klitirqd_sem);
375#endif 375#endif
376 376
377#ifdef CONFIG_LITMUS_NVIDIA
378 WARN_ON(p->rt_param.held_gpus != 0);
379#endif
380
377 /* Cleanup everything else. */ 381 /* Cleanup everything else. */
378 memset(&p->rt_param, 0, sizeof(p->rt_param)); 382 memset(&p->rt_param, 0, sizeof(p->rt_param));
379 383
diff --git a/litmus/nvidia_info.c b/litmus/nvidia_info.c
index d17152138c63..66181515186a 100644
--- a/litmus/nvidia_info.c
+++ b/litmus/nvidia_info.c
@@ -6,6 +6,10 @@
6#include <litmus/nvidia_info.h> 6#include <litmus/nvidia_info.h>
7#include <litmus/litmus.h> 7#include <litmus/litmus.h>
8 8
9#include <litmus/sched_plugin.h>
10
11#include <litmus/binheap.h>
12
9typedef unsigned char NvV8; /* "void": enumerated or multiple fields */ 13typedef unsigned char NvV8; /* "void": enumerated or multiple fields */
10typedef unsigned short NvV16; /* "void": enumerated or multiple fields */ 14typedef unsigned short NvV16; /* "void": enumerated or multiple fields */
11typedef unsigned char NvU8; /* 0 to 255 */ 15typedef unsigned char NvU8; /* 0 to 255 */
@@ -316,9 +320,13 @@ u32 get_work_nv_device_num(const struct work_struct *t)
316 320
317 321
318 322
323#define MAX_NR_OWNERS 3
324
319typedef struct { 325typedef struct {
320 raw_spinlock_t lock; 326 raw_spinlock_t lock;
321 struct task_struct *device_owner; 327 int nr_owners;
328 struct task_struct* max_prio_owner;
329 struct task_struct* owners[MAX_NR_OWNERS];
322}nv_device_registry_t; 330}nv_device_registry_t;
323 331
324static nv_device_registry_t NV_DEVICE_REG[NV_DEVICE_NUM]; 332static nv_device_registry_t NV_DEVICE_REG[NV_DEVICE_NUM];
@@ -327,12 +335,11 @@ int init_nv_device_reg(void)
327{ 335{
328 int i; 336 int i;
329 337
330 //memset(NV_DEVICE_REG, 0, sizeof(NV_DEVICE_REG)); 338 memset(NV_DEVICE_REG, 0, sizeof(NV_DEVICE_REG));
331 339
332 for(i = 0; i < NV_DEVICE_NUM; ++i) 340 for(i = 0; i < NV_DEVICE_NUM; ++i)
333 { 341 {
334 raw_spin_lock_init(&NV_DEVICE_REG[i].lock); 342 raw_spin_lock_init(&NV_DEVICE_REG[i].lock);
335 NV_DEVICE_REG[i].device_owner = NULL;
336 } 343 }
337 344
338 return(1); 345 return(1);
@@ -357,107 +364,148 @@ int get_nv_device_id(struct task_struct* owner)
357} 364}
358*/ 365*/
359 366
367static struct task_struct* find_hp_owner(nv_device_registry_t *reg, struct task_struct *skip) {
368 int i;
369 struct task_struct *found = NULL;
370 for(i = 0; i < reg->nr_owners; ++i) {
371 if(reg->owners[i] != skip && litmus->compare(reg->owners[i], found)) {
372 found = reg->owners[i];
373 }
374 }
375 return found;
376}
360 377
378#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
379void pai_check_priority_increase(struct task_struct *t, int reg_device_id)
380{
381 unsigned long flags;
382 nv_device_registry_t *reg = &NV_DEVICE_REG[reg_device_id];
383
384 if(reg->max_prio_owner != t) {
385
386 raw_spin_lock_irqsave(&reg->lock, flags);
387
388 if(reg->max_prio_owner != t) {
389 if(litmus->compare(t, reg->max_prio_owner)) {
390 litmus->change_prio_pai_tasklet(reg->max_prio_owner, t);
391 reg->max_prio_owner = t;
392 }
393 }
394
395 raw_spin_unlock_irqrestore(&reg->lock, flags);
396 }
397}
398
399
400void pai_check_priority_decrease(struct task_struct *t, int reg_device_id)
401{
402 unsigned long flags;
403 nv_device_registry_t *reg = &NV_DEVICE_REG[reg_device_id];
404
405 if(reg->max_prio_owner == t) {
406
407 raw_spin_lock_irqsave(&reg->lock, flags);
408
409 if(reg->max_prio_owner == t) {
410 reg->max_prio_owner = find_hp_owner(reg, NULL);
411 if(reg->max_prio_owner != t) {
412 litmus->change_prio_pai_tasklet(t, reg->max_prio_owner);
413 }
414 }
415
416 raw_spin_unlock_irqrestore(&reg->lock, flags);
417 }
418}
419#endif
361 420
362static int __reg_nv_device(int reg_device_id) 421static int __reg_nv_device(int reg_device_id)
363{ 422{
364 int ret = 0; 423 int ret = 0;
365 struct task_struct* old = 424 int i;
366 cmpxchg(&NV_DEVICE_REG[reg_device_id].device_owner, 425 struct task_struct *t = current;
367 NULL, 426 struct task_struct *old_max = NULL;
368 current); 427 unsigned long flags;
369 428 nv_device_registry_t *reg = &NV_DEVICE_REG[reg_device_id];
370 mb(); 429
371 430 raw_spin_lock_irqsave(&reg->lock, flags);
372 if(likely(old == NULL)) 431
373 { 432 if(reg->nr_owners < MAX_NR_OWNERS) {
433 for(i = 0; i < MAX_NR_OWNERS; ++i) {
434 if(reg->owners[i] == NULL) {
435 reg->owners[i] = t;
436
437 //if(edf_higher_prio(t, reg->max_prio_owner)) {
438 if(litmus->compare(t, reg->max_prio_owner)) {
439 old_max = reg->max_prio_owner;
440 reg->max_prio_owner = t;
441
442#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
443 litmus->change_prio_pai_tasklet(old_max, t);
444#endif
445 }
446
374#ifdef CONFIG_LITMUS_SOFTIRQD 447#ifdef CONFIG_LITMUS_SOFTIRQD
375 down_and_set_stat(current, HELD, &tsk_rt(current)->klitirqd_sem); 448 down_and_set_stat(t, HELD, &tsk_rt(t)->klitirqd_sem);
376#endif 449#endif
377 TRACE_CUR("%s: device %d registered.\n", __FUNCTION__, reg_device_id); 450 ++(reg->nr_owners);
378 } 451
452 break;
453 }
454 }
455 }
379 else 456 else
380 { 457 {
381 TRACE_CUR("%s: device %d is already in use!\n", __FUNCTION__, reg_device_id); 458 TRACE_CUR("%s: device %d is already in use!\n", __FUNCTION__, reg_device_id);
382 ret = -EBUSY; 459 ret = -EBUSY;
383 } 460 }
384 461
385 return(ret); 462 raw_spin_unlock_irqrestore(&reg->lock, flags);
386
387 463
464 __set_bit(reg_device_id, &tsk_rt(t)->held_gpus);
388 465
389#if 0 466 return(ret);
390 //unsigned long flags;
391 //raw_spin_lock_irqsave(&NV_DEVICE_REG[reg_device_id].lock, flags);
392 //lock_nv_registry(reg_device_id, &flags);
393
394 if(likely(NV_DEVICE_REG[reg_device_id].device_owner == NULL))
395 {
396 NV_DEVICE_REG[reg_device_id].device_owner = current;
397 mb(); // needed?
398
399 // release spin lock before chance of going to sleep.
400 //raw_spin_unlock_irqrestore(&NV_DEVICE_REG[reg_device_id].lock, flags);
401 //unlock_nv_registry(reg_device_id, &flags);
402
403 down_and_set_stat(current, HELD, &tsk_rt(current)->klitirqd_sem);
404 TRACE_CUR("%s: device %d registered.\n", __FUNCTION__, reg_device_id);
405 return(0);
406 }
407 else
408 {
409 //raw_spin_unlock_irqrestore(&NV_DEVICE_REG[reg_device_id].lock, flags);
410 //unlock_nv_registry(reg_device_id, &flags);
411
412 TRACE_CUR("%s: device %d is already in use!\n", __FUNCTION__, reg_device_id);
413 return(-EBUSY);
414 }
415#endif
416} 467}
417 468
418static int __clear_reg_nv_device(int de_reg_device_id) 469static int __clear_reg_nv_device(int de_reg_device_id)
419{ 470{
420 int ret = 0; 471 int ret = 0;
421 struct task_struct* old; 472 int i;
473 struct task_struct *t = current;
474 unsigned long flags;
475 nv_device_registry_t *reg = &NV_DEVICE_REG[de_reg_device_id];
422 476
423#ifdef CONFIG_LITMUS_SOFTIRQD 477#ifdef CONFIG_LITMUS_SOFTIRQD
424 unsigned long flags;
425 struct task_struct* klitirqd_th = get_klitirqd(de_reg_device_id); 478 struct task_struct* klitirqd_th = get_klitirqd(de_reg_device_id);
426 lock_nv_registry(de_reg_device_id, &flags);
427#endif 479#endif
428 480
429 old = cmpxchg(&NV_DEVICE_REG[de_reg_device_id].device_owner, 481 raw_spin_lock_irqsave(&reg->lock, flags);
430 current,
431 NULL);
432 482
433 mb(); 483 for(i = 0; i < reg->nr_owners; ++i) {
434 484 if(reg->owners[i] == t) {
435#ifdef CONFIG_LITMUS_SOFTIRQD 485#ifdef CONFIG_LITMUS_SOFTIRQD
436 if(likely(old == current)) 486 flush_pending(klitirqd_th, t);
437 { 487#endif
438 flush_pending(klitirqd_th, current); 488 if(reg->max_prio_owner == t) {
439 //unlock_nv_registry(de_reg_device_id, &flags); 489 reg->max_prio_owner = find_hp_owner(reg, t);
440 490#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
441 up_and_set_stat(current, NOT_HELD, &tsk_rt(current)->klitirqd_sem); 491 litmus->change_prio_pai_tasklet(t, reg->max_prio_owner);
442 492#endif
443 unlock_nv_registry(de_reg_device_id, &flags); 493 }
444 ret = 0; 494
445 495#ifdef CONFIG_LITMUS_SOFTIRQD
446 TRACE_CUR("%s: semaphore released.\n",__FUNCTION__); 496 up_and_set_stat(t, NOT_HELD, &tsk_rt(t)->klitirqd_sem);
447 }
448 else
449 {
450 unlock_nv_registry(de_reg_device_id, &flags);
451 ret = -EINVAL;
452
453 if(old)
454 TRACE_CUR("%s: device %d is not registered for this process's use! %s/%d is!\n",
455 __FUNCTION__, de_reg_device_id, old->comm, old->pid);
456 else
457 TRACE_CUR("%s: device %d is not registered for this process's use! No one is!\n",
458 __FUNCTION__, de_reg_device_id);
459 }
460#endif 497#endif
498
499 reg->owners[i] = NULL;
500 --(reg->nr_owners);
501
502 break;
503 }
504 }
505
506 raw_spin_unlock_irqrestore(&reg->lock, flags);
507
508 __clear_bit(de_reg_device_id, &tsk_rt(t)->held_gpus);
461 509
462 return(ret); 510 return(ret);
463} 511}
@@ -483,11 +531,11 @@ int reg_nv_device(int reg_device_id, int reg_action)
483} 531}
484 532
485/* use to get the owner of nv_device_id. */ 533/* use to get the owner of nv_device_id. */
486struct task_struct* get_nv_device_owner(u32 target_device_id) 534struct task_struct* get_nv_max_device_owner(u32 target_device_id)
487{ 535{
488 struct task_struct* owner; 536 struct task_struct *owner = NULL;
489 BUG_ON(target_device_id >= NV_DEVICE_NUM); 537 BUG_ON(target_device_id >= NV_DEVICE_NUM);
490 owner = NV_DEVICE_REG[target_device_id].device_owner; 538 owner = NV_DEVICE_REG[target_device_id].max_prio_owner;
491 return(owner); 539 return(owner);
492} 540}
493 541
@@ -516,21 +564,21 @@ void unlock_nv_registry(u32 target_device_id, unsigned long* flags)
516} 564}
517 565
518 566
519void increment_nv_int_count(u32 device) 567//void increment_nv_int_count(u32 device)
520{ 568//{
521 unsigned long flags; 569// unsigned long flags;
522 struct task_struct* owner; 570// struct task_struct* owner;
523 571//
524 lock_nv_registry(device, &flags); 572// lock_nv_registry(device, &flags);
525 573//
526 owner = NV_DEVICE_REG[device].device_owner; 574// owner = NV_DEVICE_REG[device].device_owner;
527 if(owner) 575// if(owner)
528 { 576// {
529 atomic_inc(&tsk_rt(owner)->nv_int_count); 577// atomic_inc(&tsk_rt(owner)->nv_int_count);
530 } 578// }
531 579//
532 unlock_nv_registry(device, &flags); 580// unlock_nv_registry(device, &flags);
533} 581//}
534EXPORT_SYMBOL(increment_nv_int_count); 582//EXPORT_SYMBOL(increment_nv_int_count);
535 583
536 584
diff --git a/litmus/rsm_lock.c b/litmus/rsm_lock.c
index 11d119210ef9..aaca93c1e5d1 100644
--- a/litmus/rsm_lock.c
+++ b/litmus/rsm_lock.c
@@ -5,7 +5,7 @@
5#include <litmus/sched_plugin.h> 5#include <litmus/sched_plugin.h>
6#include <litmus/rsm_lock.h> 6#include <litmus/rsm_lock.h>
7 7
8#include <litmus/edf_common.h> 8//#include <litmus/edf_common.h>
9 9
10 10
11/* caller is responsible for locking */ 11/* caller is responsible for locking */
@@ -41,7 +41,8 @@ static struct task_struct* rsm_mutex_find_hp_waiter(struct rsm_mutex *mutex,
41#endif 41#endif
42 42
43 /* Compare task prios, find high prio task. */ 43 /* Compare task prios, find high prio task. */
44 if (queued && queued != skip && edf_higher_prio(queued, found)) { 44 //if (queued && queued != skip && edf_higher_prio(queued, found)) {
45 if (queued && queued != skip && litmus->compare(queued, found)) {
45 found = queued; 46 found = queued;
46 } 47 }
47 } 48 }
@@ -107,7 +108,8 @@ void rsm_mutex_enable_priority(struct litmus_lock *l,
107 tsk_rt(t)->blocked_lock = l; 108 tsk_rt(t)->blocked_lock = l;
108 mb(); 109 mb();
109 110
110 if (edf_higher_prio(t, mutex->hp_waiter)) { 111 //if (edf_higher_prio(t, mutex->hp_waiter)) {
112 if (litmus->compare(t, mutex->hp_waiter)) {
111 113
112 struct task_struct *old_max_eff_prio; 114 struct task_struct *old_max_eff_prio;
113 struct task_struct *new_max_eff_prio; 115 struct task_struct *new_max_eff_prio;
@@ -132,7 +134,8 @@ void rsm_mutex_enable_priority(struct litmus_lock *l,
132 TRACE_TASK(t, "is new hp_waiter.\n"); 134 TRACE_TASK(t, "is new hp_waiter.\n");
133 135
134 if ((effective_priority(owner) == old_max_eff_prio) || 136 if ((effective_priority(owner) == old_max_eff_prio) ||
135 (__edf_higher_prio(new_max_eff_prio, BASE, owner, EFFECTIVE))){ 137 //(__edf_higher_prio(new_max_eff_prio, BASE, owner, EFFECTIVE))){
138 (litmus->__compare(new_max_eff_prio, BASE, owner, EFFECTIVE))){
136 new_prio = new_max_eff_prio; 139 new_prio = new_max_eff_prio;
137 } 140 }
138 } 141 }
@@ -215,7 +218,8 @@ int rsm_mutex_lock(struct litmus_lock* l)
215 __add_wait_queue_tail_exclusive(&mutex->wait, &wait); 218 __add_wait_queue_tail_exclusive(&mutex->wait, &wait);
216 219
217 /* check if we need to activate priority inheritance */ 220 /* check if we need to activate priority inheritance */
218 if (edf_higher_prio(t, mutex->hp_waiter)) { 221 //if (edf_higher_prio(t, mutex->hp_waiter)) {
222 if (litmus->compare(t, mutex->hp_waiter)) {
219 223
220 struct task_struct *old_max_eff_prio; 224 struct task_struct *old_max_eff_prio;
221 struct task_struct *new_max_eff_prio; 225 struct task_struct *new_max_eff_prio;
@@ -240,8 +244,8 @@ int rsm_mutex_lock(struct litmus_lock* l)
240 TRACE_TASK(t, "is new hp_waiter.\n"); 244 TRACE_TASK(t, "is new hp_waiter.\n");
241 245
242 if ((effective_priority(owner) == old_max_eff_prio) || 246 if ((effective_priority(owner) == old_max_eff_prio) ||
243 (__edf_higher_prio(new_max_eff_prio, BASE, 247 //(__edf_higher_prio(new_max_eff_prio, BASE, owner, EFFECTIVE))){
244 owner, EFFECTIVE))){ 248 (litmus->__compare(new_max_eff_prio, BASE, owner, EFFECTIVE))){
245 new_prio = new_max_eff_prio; 249 new_prio = new_max_eff_prio;
246 } 250 }
247 } 251 }
@@ -353,7 +357,8 @@ int rsm_mutex_unlock(struct litmus_lock* l)
353 { 357 {
354 // old_max_eff_prio > new_max_eff_prio 358 // old_max_eff_prio > new_max_eff_prio
355 359
356 if(__edf_higher_prio(new_max_eff_prio, BASE, t, EFFECTIVE)) { 360 //if(__edf_higher_prio(new_max_eff_prio, BASE, t, EFFECTIVE)) {
361 if(litmus->__compare(new_max_eff_prio, BASE, t, EFFECTIVE)) {
357 TRACE_TASK(t, "new_max_eff_prio > task's eff_prio-- new_max_eff_prio: %s/%d task: %s/%d [%s/%d]\n", 362 TRACE_TASK(t, "new_max_eff_prio > task's eff_prio-- new_max_eff_prio: %s/%d task: %s/%d [%s/%d]\n",
358 new_max_eff_prio->comm, new_max_eff_prio->pid, 363 new_max_eff_prio->comm, new_max_eff_prio->pid,
359 t->comm, t->pid, tsk_rt(t)->inh_task->comm, 364 t->comm, t->pid, tsk_rt(t)->inh_task->comm,
@@ -460,8 +465,8 @@ int rsm_mutex_unlock(struct litmus_lock* l)
460 { 465 {
461 if(dgl_wait && tsk_rt(next)->blocked_lock) { 466 if(dgl_wait && tsk_rt(next)->blocked_lock) {
462 BUG_ON(wake_up_task); 467 BUG_ON(wake_up_task);
463 if(__edf_higher_prio(l->nest.hp_waiter_eff_prio, BASE, 468 //if(__edf_higher_prio(l->nest.hp_waiter_eff_prio, BASE, next, EFFECTIVE)) {
464 next, EFFECTIVE)) { 469 if(litmus->__compare(l->nest.hp_waiter_eff_prio, BASE, next, EFFECTIVE)) {
465 litmus->nested_increase_prio(next, 470 litmus->nested_increase_prio(next,
466 l->nest.hp_waiter_eff_prio, &mutex->lock, flags); // unlocks lock && hp_blocked_tasks_lock. 471 l->nest.hp_waiter_eff_prio, &mutex->lock, flags); // unlocks lock && hp_blocked_tasks_lock.
467 goto out; // all spinlocks are released. bail out now. 472 goto out; // all spinlocks are released. bail out now.
@@ -532,7 +537,8 @@ void rsm_mutex_propagate_increase_inheritance(struct litmus_lock* l,
532 537
533 old_max_eff_prio = top_priority(&tsk_rt(owner)->hp_blocked_tasks); 538 old_max_eff_prio = top_priority(&tsk_rt(owner)->hp_blocked_tasks);
534 539
535 if((t != mutex->hp_waiter) && edf_higher_prio(t, mutex->hp_waiter)) { 540 //if((t != mutex->hp_waiter) && edf_higher_prio(t, mutex->hp_waiter)) {
541 if((t != mutex->hp_waiter) && litmus->compare(t, mutex->hp_waiter)) {
536 TRACE_TASK(t, "is new highest-prio waiter by propagation.\n"); 542 TRACE_TASK(t, "is new highest-prio waiter by propagation.\n");
537 mutex->hp_waiter = t; 543 mutex->hp_waiter = t;
538 } 544 }
@@ -554,8 +560,8 @@ void rsm_mutex_propagate_increase_inheritance(struct litmus_lock* l,
554 if(new_max_eff_prio != old_max_eff_prio) { 560 if(new_max_eff_prio != old_max_eff_prio) {
555 // new_max_eff_prio > old_max_eff_prio holds. 561 // new_max_eff_prio > old_max_eff_prio holds.
556 if ((effective_priority(owner) == old_max_eff_prio) || 562 if ((effective_priority(owner) == old_max_eff_prio) ||
557 (__edf_higher_prio(new_max_eff_prio, BASE, owner, EFFECTIVE))) { 563 //(__edf_higher_prio(new_max_eff_prio, BASE, owner, EFFECTIVE))) {
558 564 (litmus->__compare(new_max_eff_prio, BASE, owner, EFFECTIVE))) {
559 TRACE_CUR("Propagating inheritance to holder of lock %d.\n", 565 TRACE_CUR("Propagating inheritance to holder of lock %d.\n",
560 l->ident); 566 l->ident);
561 567
@@ -649,7 +655,8 @@ void rsm_mutex_propagate_decrease_inheritance(struct litmus_lock* l,
649 TRACE_CUR("Propagating decreased inheritance to holder of lock %d.\n", 655 TRACE_CUR("Propagating decreased inheritance to holder of lock %d.\n",
650 l->ident); 656 l->ident);
651 657
652 if(__edf_higher_prio(new_max_eff_prio, BASE, owner, BASE)) { 658 //if(__edf_higher_prio(new_max_eff_prio, BASE, owner, BASE)) {
659 if(litmus->__compare(new_max_eff_prio, BASE, owner, BASE)) {
653 TRACE_CUR("%s/%d has greater base priority than base priority of owner (%s/%d) of lock %d.\n", 660 TRACE_CUR("%s/%d has greater base priority than base priority of owner (%s/%d) of lock %d.\n",
654 (new_max_eff_prio) ? new_max_eff_prio->comm : "nil", 661 (new_max_eff_prio) ? new_max_eff_prio->comm : "nil",
655 (new_max_eff_prio) ? new_max_eff_prio->pid : -1, 662 (new_max_eff_prio) ? new_max_eff_prio->pid : -1,
diff --git a/litmus/sched_gsn_edf.c b/litmus/sched_gsn_edf.c
index e009b7f34aca..37f7821dca50 100644
--- a/litmus/sched_gsn_edf.c
+++ b/litmus/sched_gsn_edf.c
@@ -436,14 +436,6 @@ static void gsnedf_tick(struct task_struct* t)
436 436
437 437
438 438
439
440
441
442
443
444
445
446
447#ifdef CONFIG_LITMUS_PAI_SOFTIRQD 439#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
448 440
449 441
@@ -467,96 +459,117 @@ static void __do_lit_tasklet(struct tasklet_struct* tasklet, unsigned long flush
467 } 459 }
468} 460}
469 461
470
471static void flush_tasklets(struct task_struct* task)
472{
473 // lazy flushing.
474 // just change ownership to NULL and let idel processor
475 // take care of it. :P
476
477 struct tasklet_struct* step;
478 unsigned long flags;
479
480 raw_spin_lock_irqsave(&gsnedf_lock, flags);
481 for(step = gsnedf_pending_tasklets.head; step != NULL; step = step->next) {
482 if(step->owner == task) {
483 TRACE("%s: Found tasklet to flush: %d\n", __FUNCTION__, step->owner->pid);
484 step->owner = NULL;
485 }
486 }
487 raw_spin_unlock_irqrestore(&gsnedf_lock, flags);
488}
489
490
491static void do_lit_tasklets(struct task_struct* sched_task) 462static void do_lit_tasklets(struct task_struct* sched_task)
492{ 463{
493 int work_to_do = 1; 464 int work_to_do = 1;
494 struct tasklet_struct *tasklet = NULL; 465 struct tasklet_struct *tasklet = NULL;
495 //struct tasklet_struct *step;
496 unsigned long flags; 466 unsigned long flags;
497 467
498 while(work_to_do) { 468 while(work_to_do) {
499 469
500 TS_NV_SCHED_BOTISR_START; 470 TS_NV_SCHED_BOTISR_START;
501 471
502 // remove tasklet at head of list if it has higher priority. 472 // execute one tasklet that has higher priority
503 raw_spin_lock_irqsave(&gsnedf_lock, flags); 473 raw_spin_lock_irqsave(&gsnedf_lock, flags);
504 474
505 if(gsnedf_pending_tasklets.head != NULL) { 475 if(gsnedf_pending_tasklets.head != NULL) {
506 // remove tasklet at head. 476 struct tasklet_struct *prev = NULL;
507 tasklet = gsnedf_pending_tasklets.head; 477 tasklet = gsnedf_pending_tasklets.head;
478
479 while(tasklet && edf_higher_prio(sched_task, tasklet->owner)) {
480 prev = tasklet;
481 tasklet = tasklet->next;
482 }
508 483
509 if(edf_higher_prio(tasklet->owner, sched_task)) { 484 // remove the tasklet from the queue
510 485 if(prev) {
511 if(NULL == tasklet->next) { 486 prev->next = tasklet->next;
512 // tasklet is at the head, list only has one element 487 if(prev->next == NULL) {
513 TRACE("%s: Tasklet for %d is the last element in tasklet queue.\n", __FUNCTION__, tasklet->owner->pid); 488 TRACE("%s: Tasklet for %d is the last element in tasklet queue.\n", __FUNCTION__, tasklet->owner->pid);
514 gsnedf_pending_tasklets.tail = &(gsnedf_pending_tasklets.head); 489 gsnedf_pending_tasklets.tail = &(prev);
515 } 490 }
516
517 // remove the tasklet from the queue
518 gsnedf_pending_tasklets.head = tasklet->next;
519
520 TRACE("%s: Removed tasklet for %d from tasklet queue.\n", __FUNCTION__, tasklet->owner->pid);
521 } 491 }
522 else { 492 else {
523 TRACE("%s: Pending tasklet (%d) does not have priority to run on this CPU (%d).\n", __FUNCTION__, tasklet->owner->pid, smp_processor_id()); 493 gsnedf_pending_tasklets.head = tasklet->next;
524 tasklet = NULL; 494 if(tasklet->next == NULL) {
495 TRACE("%s: Tasklet for %d is the last element in tasklet queue.\n", __FUNCTION__, tasklet->owner->pid);
496 gsnedf_pending_tasklets.tail = &(gsnedf_pending_tasklets.head);
497 }
525 } 498 }
526 } 499 }
527 else { 500 else {
528 TRACE("%s: Tasklet queue is empty.\n", __FUNCTION__); 501 TRACE("%s: Tasklet queue is empty.\n", __FUNCTION__);
529 } 502 }
530 503
531 raw_spin_unlock_irqrestore(&gsnedf_lock, flags); 504 raw_spin_unlock_irqrestore(&gsnedf_lock, flags);
532 505
533 TS_NV_SCHED_BOTISR_END;
534
535 if(tasklet) { 506 if(tasklet) {
536 __do_lit_tasklet(tasklet, 0ul); 507 __do_lit_tasklet(tasklet, 0ul);
537 tasklet = NULL; 508 tasklet = NULL;
538 } 509 }
539 else { 510 else {
540 work_to_do = 0; 511 work_to_do = 0;
541 } 512 }
513
514 TS_NV_SCHED_BOTISR_END;
542 } 515 }
543
544 //TRACE("%s: exited.\n", __FUNCTION__);
545}
546
547
548static void run_tasklets(struct task_struct* sched_task)
549{
550 preempt_disable();
551
552 if(gsnedf_pending_tasklets.head != NULL) {
553 TRACE("%s: There are tasklets to process.\n", __FUNCTION__);
554 do_lit_tasklets(sched_task);
555 }
556
557 preempt_enable_no_resched();
558} 516}
559 517
518//static void do_lit_tasklets(struct task_struct* sched_task)
519//{
520// int work_to_do = 1;
521// struct tasklet_struct *tasklet = NULL;
522// //struct tasklet_struct *step;
523// unsigned long flags;
524//
525// while(work_to_do) {
526//
527// TS_NV_SCHED_BOTISR_START;
528//
529// // remove tasklet at head of list if it has higher priority.
530// raw_spin_lock_irqsave(&gsnedf_lock, flags);
531//
532// if(gsnedf_pending_tasklets.head != NULL) {
533// // remove tasklet at head.
534// tasklet = gsnedf_pending_tasklets.head;
535//
536// if(edf_higher_prio(tasklet->owner, sched_task)) {
537//
538// if(NULL == tasklet->next) {
539// // tasklet is at the head, list only has one element
540// TRACE("%s: Tasklet for %d is the last element in tasklet queue.\n", __FUNCTION__, tasklet->owner->pid);
541// gsnedf_pending_tasklets.tail = &(gsnedf_pending_tasklets.head);
542// }
543//
544// // remove the tasklet from the queue
545// gsnedf_pending_tasklets.head = tasklet->next;
546//
547// TRACE("%s: Removed tasklet for %d from tasklet queue.\n", __FUNCTION__, tasklet->owner->pid);
548// }
549// else {
550// TRACE("%s: Pending tasklet (%d) does not have priority to run on this CPU (%d).\n", __FUNCTION__, tasklet->owner->pid, smp_processor_id());
551// tasklet = NULL;
552// }
553// }
554// else {
555// TRACE("%s: Tasklet queue is empty.\n", __FUNCTION__);
556// }
557//
558// raw_spin_unlock_irqrestore(&gsnedf_lock, flags);
559//
560// TS_NV_SCHED_BOTISR_END;
561//
562// if(tasklet) {
563// __do_lit_tasklet(tasklet, 0ul);
564// tasklet = NULL;
565// }
566// else {
567// work_to_do = 0;
568// }
569// }
570//
571// //TRACE("%s: exited.\n", __FUNCTION__);
572//}
560 573
561static void __add_pai_tasklet(struct tasklet_struct* tasklet) 574static void __add_pai_tasklet(struct tasklet_struct* tasklet)
562{ 575{
@@ -604,7 +617,19 @@ static void __add_pai_tasklet(struct tasklet_struct* tasklet)
604 } 617 }
605} 618}
606 619
607static int enqueue_pai_tasklet(struct tasklet_struct* tasklet) 620static void gsnedf_run_tasklets(struct task_struct* sched_task)
621{
622 preempt_disable();
623
624 if(gsnedf_pending_tasklets.head != NULL) {
625 TRACE("%s: There are tasklets to process.\n", __FUNCTION__);
626 do_lit_tasklets(sched_task);
627 }
628
629 preempt_enable_no_resched();
630}
631
632static int gsnedf_enqueue_pai_tasklet(struct tasklet_struct* tasklet)
608{ 633{
609 cpu_entry_t *targetCPU = NULL; 634 cpu_entry_t *targetCPU = NULL;
610 int thisCPU; 635 int thisCPU;
@@ -692,6 +717,23 @@ static int enqueue_pai_tasklet(struct tasklet_struct* tasklet)
692 return(1); // success 717 return(1); // success
693} 718}
694 719
720static void gsnedf_change_prio_pai_tasklet(struct task_struct *old_prio,
721 struct task_struct *new_prio)
722{
723 struct tasklet_struct* step;
724 unsigned long flags;
725
726 raw_spin_lock_irqsave(&gsnedf_lock, flags);
727
728 for(step = gsnedf_pending_tasklets.head; step != NULL; step = step->next) {
729 if(step->owner == old_prio) {
730 TRACE("%s: Found tasklet to flush: %d\n", __FUNCTION__, step->owner->pid);
731 step->owner = new_prio;
732 }
733 }
734 raw_spin_unlock_irqrestore(&gsnedf_lock, flags);
735}
736
695#endif // end PAI 737#endif // end PAI
696 738
697 739
@@ -954,7 +996,7 @@ static void gsnedf_task_exit(struct task_struct * t)
954 unsigned long flags; 996 unsigned long flags;
955 997
956#ifdef CONFIG_LITMUS_PAI_SOFTIRQD 998#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
957 flush_tasklets(t); 999 gsnedf_change_prio_pai_tasklet(t, NULL);
958#endif 1000#endif
959 1001
960 /* unlink if necessary */ 1002 /* unlink if necessary */
@@ -1072,6 +1114,8 @@ static void __increase_priority_inheritance(struct task_struct* t,
1072/* called with IRQs off */ 1114/* called with IRQs off */
1073static void increase_priority_inheritance(struct task_struct* t, struct task_struct* prio_inh) 1115static void increase_priority_inheritance(struct task_struct* t, struct task_struct* prio_inh)
1074{ 1116{
1117 int i = 0;
1118
1075 raw_spin_lock(&gsnedf_lock); 1119 raw_spin_lock(&gsnedf_lock);
1076 1120
1077 __increase_priority_inheritance(t, prio_inh); 1121 __increase_priority_inheritance(t, prio_inh);
@@ -1087,6 +1131,14 @@ static void increase_priority_inheritance(struct task_struct* t, struct task_str
1087#endif 1131#endif
1088 1132
1089 raw_spin_unlock(&gsnedf_lock); 1133 raw_spin_unlock(&gsnedf_lock);
1134
1135#if defined(CONFIG_LITMUS_PAI_SOFTIRQD) && defined(CONFIG_LITMUS_NVIDIA)
1136 for(i = find_first_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus));
1137 i < NV_DEVICE_NUM;
1138 i = find_next_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus), i)) {
1139 pai_check_priority_increase(t, i);
1140 }
1141#endif
1090} 1142}
1091 1143
1092 1144
@@ -1147,6 +1199,8 @@ static void __decrease_priority_inheritance(struct task_struct* t,
1147static void decrease_priority_inheritance(struct task_struct* t, 1199static void decrease_priority_inheritance(struct task_struct* t,
1148 struct task_struct* prio_inh) 1200 struct task_struct* prio_inh)
1149{ 1201{
1202 int i;
1203
1150 raw_spin_lock(&gsnedf_lock); 1204 raw_spin_lock(&gsnedf_lock);
1151 __decrease_priority_inheritance(t, prio_inh); 1205 __decrease_priority_inheritance(t, prio_inh);
1152 1206
@@ -1160,7 +1214,15 @@ static void decrease_priority_inheritance(struct task_struct* t,
1160 } 1214 }
1161#endif 1215#endif
1162 1216
1163 raw_spin_unlock(&gsnedf_lock); 1217 raw_spin_unlock(&gsnedf_lock);
1218
1219#if defined(CONFIG_LITMUS_PAI_SOFTIRQD) && defined(CONFIG_LITMUS_NVIDIA)
1220 for(i = find_first_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus));
1221 i < NV_DEVICE_NUM;
1222 i = find_next_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus), i)) {
1223 pai_check_priority_decrease(t, i);
1224 }
1225#endif
1164} 1226}
1165 1227
1166 1228
@@ -1687,8 +1749,9 @@ static struct sched_plugin gsn_edf_plugin __cacheline_aligned_in_smp = {
1687 .decrease_prio_klitirqd = decrease_priority_inheritance_klitirqd, 1749 .decrease_prio_klitirqd = decrease_priority_inheritance_klitirqd,
1688#endif 1750#endif
1689#ifdef CONFIG_LITMUS_PAI_SOFTIRQD 1751#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
1690 .enqueue_pai_tasklet = enqueue_pai_tasklet, 1752 .enqueue_pai_tasklet = gsnedf_enqueue_pai_tasklet,
1691 .run_tasklets = run_tasklets, 1753 .change_prio_pai_tasklet = gsnedf_change_prio_pai_tasklet,
1754 .run_tasklets = gsnedf_run_tasklets,
1692#endif 1755#endif
1693}; 1756};
1694 1757
diff --git a/litmus/sched_plugin.c b/litmus/sched_plugin.c
index 75694350a9ad..24326ce4657e 100644
--- a/litmus/sched_plugin.c
+++ b/litmus/sched_plugin.c
@@ -110,6 +110,11 @@ static long litmus_dummy_deactivate_plugin(void)
110 return 0; 110 return 0;
111} 111}
112 112
113static int litmus_dummy_compare(struct task_struct* a, struct task_struct* b)
114{
115 return 0;
116}
117
113#ifdef CONFIG_LITMUS_LOCKING 118#ifdef CONFIG_LITMUS_LOCKING
114static long litmus_dummy_allocate_lock(struct litmus_lock **lock, int type, 119static long litmus_dummy_allocate_lock(struct litmus_lock **lock, int type,
115 void* __user config) 120 void* __user config)
@@ -146,6 +151,12 @@ static int litmus_dummy_enqueue_pai_tasklet(struct tasklet_struct* t)
146 return(0); // failure. 151 return(0); // failure.
147} 152}
148 153
154static void litmus_dummy_change_prio_pai_tasklet(struct task_struct *old_prio,
155 struct task_struct *new_prio)
156{
157 TRACE("%s: PAI Tasklet unsupported in this plugin!!!!!!\n", __FUNCTION__);
158}
159
149static void litmus_dummy_run_tasklets(struct task_struct* t) 160static void litmus_dummy_run_tasklets(struct task_struct* t)
150{ 161{
151 //TRACE("%s: PAI Tasklet unsupported in this plugin!!!!!!\n", __FUNCTION__); 162 //TRACE("%s: PAI Tasklet unsupported in this plugin!!!!!!\n", __FUNCTION__);
@@ -162,6 +173,12 @@ static void litmus_dummy_nested_decrease_prio(struct task_struct* t, struct task
162 raw_spinlock_t *to_unlock, unsigned long irqflags) 173 raw_spinlock_t *to_unlock, unsigned long irqflags)
163{ 174{
164} 175}
176
177static int litmus_dummy___compare(struct task_struct* a, comparison_mode_t a_mod,
178 struct task_struct* b, comparison_mode_t b_mode)
179{
180 return 0;
181}
165#endif 182#endif
166 183
167#ifdef CONFIG_LITMUS_DGL_SUPPORT 184#ifdef CONFIG_LITMUS_DGL_SUPPORT
@@ -188,6 +205,7 @@ struct sched_plugin linux_sched_plugin = {
188 .finish_switch = litmus_dummy_finish_switch, 205 .finish_switch = litmus_dummy_finish_switch,
189 .activate_plugin = litmus_dummy_activate_plugin, 206 .activate_plugin = litmus_dummy_activate_plugin,
190 .deactivate_plugin = litmus_dummy_deactivate_plugin, 207 .deactivate_plugin = litmus_dummy_deactivate_plugin,
208 .compare = litmus_dummy_compare,
191#ifdef CONFIG_LITMUS_LOCKING 209#ifdef CONFIG_LITMUS_LOCKING
192 .allocate_lock = litmus_dummy_allocate_lock, 210 .allocate_lock = litmus_dummy_allocate_lock,
193 .increase_prio = litmus_dummy_increase_prio, 211 .increase_prio = litmus_dummy_increase_prio,
@@ -196,6 +214,7 @@ struct sched_plugin linux_sched_plugin = {
196#ifdef CONFIG_LITMUS_NESTED_LOCKING 214#ifdef CONFIG_LITMUS_NESTED_LOCKING
197 .nested_increase_prio = litmus_dummy_nested_increase_prio, 215 .nested_increase_prio = litmus_dummy_nested_increase_prio,
198 .nested_decrease_prio = litmus_dummy_nested_decrease_prio, 216 .nested_decrease_prio = litmus_dummy_nested_decrease_prio,
217 .__compare = litmus_dummy___compare,
199#endif 218#endif
200#ifdef CONFIG_LITMUS_SOFTIRQD 219#ifdef CONFIG_LITMUS_SOFTIRQD
201 .increase_prio_klitirqd = litmus_dummy_increase_prio_klitirqd, 220 .increase_prio_klitirqd = litmus_dummy_increase_prio_klitirqd,
@@ -203,6 +222,7 @@ struct sched_plugin linux_sched_plugin = {
203#endif 222#endif
204#ifdef CONFIG_LITMUS_PAI_SOFTIRQD 223#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
205 .enqueue_pai_tasklet = litmus_dummy_enqueue_pai_tasklet, 224 .enqueue_pai_tasklet = litmus_dummy_enqueue_pai_tasklet,
225 .change_prio_pai_tasklet = litmus_dummy_change_prio_pai_tasklet,
206 .run_tasklets = litmus_dummy_run_tasklets, 226 .run_tasklets = litmus_dummy_run_tasklets,
207#endif 227#endif
208#ifdef CONFIG_LITMUS_DGL_SUPPORT 228#ifdef CONFIG_LITMUS_DGL_SUPPORT
@@ -243,6 +263,7 @@ int register_sched_plugin(struct sched_plugin* plugin)
243 CHECK(complete_job); 263 CHECK(complete_job);
244 CHECK(activate_plugin); 264 CHECK(activate_plugin);
245 CHECK(deactivate_plugin); 265 CHECK(deactivate_plugin);
266 CHECK(compare);
246#ifdef CONFIG_LITMUS_LOCKING 267#ifdef CONFIG_LITMUS_LOCKING
247 CHECK(allocate_lock); 268 CHECK(allocate_lock);
248 CHECK(increase_prio); 269 CHECK(increase_prio);
@@ -251,6 +272,7 @@ int register_sched_plugin(struct sched_plugin* plugin)
251#ifdef CONFIG_LITMUS_NESTED_LOCKING 272#ifdef CONFIG_LITMUS_NESTED_LOCKING
252 CHECK(nested_increase_prio); 273 CHECK(nested_increase_prio);
253 CHECK(nested_decrease_prio); 274 CHECK(nested_decrease_prio);
275 CHECK(__compare);
254#endif 276#endif
255#ifdef CONFIG_LITMUS_SOFTIRQD 277#ifdef CONFIG_LITMUS_SOFTIRQD
256 CHECK(increase_prio_klitirqd); 278 CHECK(increase_prio_klitirqd);
@@ -258,6 +280,7 @@ int register_sched_plugin(struct sched_plugin* plugin)
258#endif 280#endif
259#ifdef CONFIG_LITMUS_PAI_SOFTIRQD 281#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
260 CHECK(enqueue_pai_tasklet); 282 CHECK(enqueue_pai_tasklet);
283 CHECK(change_prio_pai_tasklet);
261 CHECK(run_tasklets); 284 CHECK(run_tasklets);
262#endif 285#endif
263#ifdef CONFIG_LITMUS_DGL_SUPPORT 286#ifdef CONFIG_LITMUS_DGL_SUPPORT
generated by cgit v1.2.2 (git 2.25.0) at 2024-09-19 21:32:41 -0400