PRIOQ_MUTEX dgl progress

8 files changed, 348 insertions, 220 deletions

diff --git a/include/litmus/locking.h b/include/litmus/locking.h
index cbc99ee54020..b1024e397f67 100644
--- a/include/litmus/locking.h
+++ b/include/litmus/locking.h
@@ -74,9 +74,24 @@ int dgl_wake_up(wait_queue_t *wq_node, unsigned mode, int sync, void *key);
 struct task_struct* __waitqueue_dgl_remove_first(wait_queue_head_t *wq, dgl_wait_state_t** dgl_wait);
 
 int __attempt_atomic_dgl_acquire(struct litmus_lock *cur_lock, dgl_wait_state_t *dgl_wait);
-#endif
 
 
+static inline struct task_struct* get_queued_task_and_dgl_wait(wait_queue_t* q, dgl_wait_state_t** dgl_wait_ptr)
+{
+        struct task_struct *queued;
+
+        if(q->func == dgl_wake_up) {
+                *dgl_wait_ptr = (dgl_wait_state_t*) q->private;
+                queued = (*dgl_wait_ptr)->task;
+        }
+        else {
+                *dgl_wait_ptr = NULL;
+                queued = (struct task_struct*) q->private;
+        }
+
+        return queued;
+}
+#endif
 
 
 static inline struct task_struct* get_queued_task(wait_queue_t* q)
@@ -206,7 +221,7 @@ struct litmus_lock_ops {
 
 
 void suspend_for_lock(void);
-
+int wake_up_for_lock(struct task_struct* t);
 
 #endif
 
diff --git a/include/litmus/rt_domain.h b/include/litmus/rt_domain.h
index 89a8850ecf9d..6d3c4672dca9 100644
--- a/include/litmus/rt_domain.h
+++ b/include/litmus/rt_domain.h
@@ -88,12 +88,12 @@ static inline struct task_struct* __take_ready(rt_domain_t* rt)
         if (hn) {
                 struct task_struct* taken = bheap2task(hn);
                 TRACE("rt: taking %s/%d [inh_task: %s/%d]\n",
-                        (taken) ? taken->comm : "nil",
-                        (taken) ? taken->pid : -1,
+                        (taken) ? taken->comm : "null",
+                        (taken) ? taken->pid : 0,
                         (taken && tsk_rt(taken)->inh_task) ?
-                                tsk_rt(taken)->inh_task->comm : "nil",
+                                tsk_rt(taken)->inh_task->comm : "null",
                         (taken && tsk_rt(taken)->inh_task) ?
-                                tsk_rt(taken)->inh_task->pid : -1);
+                                tsk_rt(taken)->inh_task->pid : 0);
 
                 return taken;
         }
diff --git a/litmus/fifo_lock.c b/litmus/fifo_lock.c
index cc3e1ab5a965..dfe56bface6d 100644
--- a/litmus/fifo_lock.c
+++ b/litmus/fifo_lock.c
@@ -486,14 +486,7 @@ int fifo_mutex_unlock(struct litmus_lock* l)
                         tsk_rt(next)->blocked_lock = NULL;
                         mb();
 
-#if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA)
-                        // re-enable tracking
-                        if(tsk_rt(next)->held_gpus) {
-                                tsk_rt(next)->suspend_gpu_tracker_on_block = 0;
-                        }
-#endif
-
-                        wake_up_process(next);
+                        wake_up_for_lock(next);
                 }
                 else {
                         TRACE_TASK(next, "is still blocked.\n");
diff --git a/litmus/ikglp_lock.c b/litmus/ikglp_lock.c
index 59d7ee4f9fd1..3d79e41b42df 100644
--- a/litmus/ikglp_lock.c
+++ b/litmus/ikglp_lock.c
@@ -1598,9 +1598,8 @@ wake_kludge:
                         raw_spin_unlock(&tsk_rt(next)->hp_blocked_tasks_lock);
                 }
 
-
                 // wake up the new resource holder!
-                wake_up_process(next);
+                wake_up_for_lock(next);
         }
         if(fq_of_new_on_fq && fq_of_new_on_fq != fq && fq_of_new_on_fq->count == 1) {
                 // The guy we promoted when to an empty FQ. (Why didn't stealing pick this up?)
diff --git a/litmus/kfmlp_lock.c b/litmus/kfmlp_lock.c
index 377e5a8f7456..041561839976 100644
--- a/litmus/kfmlp_lock.c
+++ b/litmus/kfmlp_lock.c
@@ -391,7 +391,7 @@ RETRY:
                 }
 
                 /* wake up next */
-                wake_up_process(next);
+                wake_up_for_lock(next);
         }
         else {
                 // TODO: put this stealing logic before we attempt to release
diff --git a/litmus/locking.c b/litmus/locking.c
index 0b5e162c0c02..4b8382cd77d1 100644
--- a/litmus/locking.c
+++ b/litmus/locking.c
@@ -365,6 +365,18 @@ static void snprintf_dgl(char* buf, size_t bsz, struct litmus_lock* dgl_locks[],
 #endif
 
 
+static int failed_owner(struct litmus_lock *cur_lock, struct task_struct *t)
+{
+        struct task_struct *cur_owner = cur_lock->ops->get_owner(cur_lock);
+        printk(KERN_EMERG "lock %d expected owner %s/%d but got %s/%d.\n",
+                                        cur_lock->ident,
+                                        (t) ? t->comm : "null",
+                                        (t) ? t->pid : 0,
+                                        (cur_owner) ? cur_owner->comm : "null",
+                                        (cur_owner) ? cur_owner->pid : 0);
+        BUG();
+}
+
 /* only valid when locks are prioq locks!!!
  * THE BIG DGL LOCK MUST BE HELD! */
 int __attempt_atomic_dgl_acquire(struct litmus_lock *cur_lock, dgl_wait_state_t *dgl_wait)
@@ -386,7 +398,9 @@ int __attempt_atomic_dgl_acquire(struct litmus_lock *cur_lock, dgl_wait_state_t
 
                 l->ops->dgl_quick_lock(l, cur_lock, dgl_wait->task, &dgl_wait->wq_nodes[i]);
 
-                BUG_ON(!(l->ops->is_owner(l, dgl_wait->task)));
+                if(!(l->ops->is_owner(l, dgl_wait->task)))
+                        failed_owner(l, dgl_wait->task);
+                //BUG_ON(!(l->ops->is_owner(l, dgl_wait->task)));
         }
 
         return 0; /* success */
@@ -564,7 +578,9 @@ all_acquired:
         // SANITY CHECK FOR TESTING
         for(i = 0; i < dgl_wait->size; ++i) {
                 struct litmus_lock *l = dgl_wait->locks[i];
-                BUG_ON(!l->ops->is_owner(l, dgl_wait->task));
+                if(!(l->ops->is_owner(l, dgl_wait->task)))
+                        failed_owner(l, dgl_wait->task);
+                //BUG_ON(!l->ops->is_owner(l, dgl_wait->task));
         }
 
         TRACE_CUR("Acquired entire DGL\n");
@@ -830,8 +846,25 @@ void suspend_for_lock(void)
         }
 #endif
 
+#if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA)
+        // disable tracking
+        if(tsk_rt(t)->held_gpus) {
+                tsk_rt(t)->suspend_gpu_tracker_on_block = 1;
+        }
+#endif
+        
         schedule();
 
+        
+        /* TODO: Move the following to wake_up_for_lock()? */
+        
+#if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA)
+        // re-enable tracking
+        if(tsk_rt(t)->held_gpus) {
+                tsk_rt(t)->suspend_gpu_tracker_on_block = 0;
+        }
+#endif
+        
 #ifdef CONFIG_LITMUS_NVIDIA
         if (gpu_restore) {
                 /* restore our state */
@@ -847,6 +880,15 @@ void suspend_for_lock(void)
 #endif
 }
 
+int wake_up_for_lock(struct task_struct* t)
+{
+        int ret;
+        
+        ret = wake_up_process(t);
+        
+        return ret;
+}
+
 
 #else  // CONFIG_LITMUS_LOCKING
 
diff --git a/litmus/prioq_lock.c b/litmus/prioq_lock.c
index ff6419ba1a13..faf8c15df542 100644
--- a/litmus/prioq_lock.c
+++ b/litmus/prioq_lock.c
@@ -128,6 +128,7 @@ static inline void __add_wait_queue_sorted_exclusive(wait_queue_head_t *q, wait_
         __add_wait_queue_sorted(q, wait);
 }
 
+
 static void __prioq_increase_pos(struct prioq_mutex *mutex, struct task_struct *t)
 {
         wait_queue_t            *q;
@@ -148,188 +149,310 @@ static void __prioq_increase_pos(struct prioq_mutex *mutex, struct task_struct *
         BUG();
 }
 
-
+#ifndef CONFIG_LITMUS_DGL_SUPPORT
 static void __prioq_decrease_pos(struct prioq_mutex *mutex, struct task_struct *t)
 {
+        /* TODO: Make this efficient instead of remove/add */
+        __prioq_increase_pos(mutex, t);
+}
+#endif
+
+
+/* caller is responsible for locking */
+static struct task_struct* __prioq_mutex_find_hp_waiter(struct prioq_mutex *mutex,
+                                                                                                                struct task_struct* skip)
+{
         wait_queue_t            *q;
         struct list_head        *pos;
-        struct task_struct  *queued;
-
-        /* TODO: Make this efficient instead of remove/add */
+        struct task_struct  *queued = NULL, *found = NULL;
+        
+        /* list in sorted order.  higher-prio tasks likely at the front. */
         list_for_each(pos, &mutex->wait.task_list) {
                 q = list_entry(pos, wait_queue_t, task_list);
                 queued = get_queued_task(q);
-                if (queued == t) {
-                        __remove_wait_queue(&mutex->wait, q);
-                        __add_wait_queue_sorted(&mutex->wait, q);
-                        return;
+                
+                /* Compare task prios, find high prio task. */
+                if (queued &&
+                        (queued != skip) &&
+                        (tsk_rt(queued)->blocked_lock == &mutex->litmus_lock) &&
+                        litmus->compare(queued, found)) {
+                        found = queued;
                 }
         }
-
-        BUG();
+        return found;
 }
 
+
+
 #ifdef CONFIG_LITMUS_DGL_SUPPORT
-static int __prioq_dgl_increase_pos(struct prioq_mutex *mutex, struct task_struct *t)
-{
-// TODO:
-//      (1) Increase position for 't' in all of it's DGLs.
-//  (2) Check to see if 't' can take the DGLs atomically
-//  (3) If it can take the DGLs, do so.
-//  (4) Cleanup?
-//  (5) Wake up 't'
 
+static int ___prioq_dgl_acquire_via_inheritance(struct prioq_mutex *mutex, struct task_struct *t, dgl_wait_state_t *dgl_wait)
+{
+        /* Any task that acquires a PRIOQ mutex via inheritance does not inheritance
+         * does not inherit priority from the hp_waiter, by defintion of the
+         * priority queue. */
+
+        struct litmus_lock *l;
+        BUG_ON(mutex->owner != NULL);
+        BUG_ON(list_empty(&mutex->wait.task_list));
+        
+        l = &mutex->litmus_lock;
+        
+        if (dgl_wait) {
+                BUG_ON(t != dgl_wait->task);
+                
+                /* we're a part of a DGL */
+                if(__attempt_atomic_dgl_acquire(NULL, dgl_wait)) {
+                        TRACE_CUR("%s/%d cannot take entire DGL via inheritance.\n",
+                                          t->comm, t->pid);
+                        /* it can't take the lock. nullify 't'. */
+                        t = NULL;
+                }
+                else {
+                        TRACE_CUR("%s/%d can take its entire DGL atomically via inheritance!\n",
+                                          t->comm, t->pid);
+                        /* __attempt_atomic_dgl_acquire() already cleaned up the state of acquired locks */
+                }
+        }
+        else {
+                /* we're a regular singular request. we can always take the lock if
+                 * there is no mutex owner. */
+                wait_queue_t *first;
+                
+                TRACE_CUR("%s/%d can take it's singular lock via inheritance!\n",
+                                  t->comm, t->pid);
+                
+                first = list_entry(mutex->wait.task_list.next, wait_queue_t, task_list);
+                
+                BUG_ON(get_queued_task(first) != t);
+                
+                __remove_wait_queue(&mutex->wait, first); /* remove the blocked task */
+                
+                /* update/cleanup the state of the lock */
+                
+                mutex->owner = t; /* take ownership!!! */
+                
+                mutex->hp_waiter = __prioq_mutex_find_hp_waiter(mutex, t);
+                l->nest.hp_waiter_eff_prio = (mutex->hp_waiter) ?
+                        effective_priority(mutex->hp_waiter) : NULL;
+                
+                if (mutex->hp_waiter)
+                        TRACE_CUR("%s/%d is new highest-prio waiter\n",
+                                          mutex->hp_waiter->comm, mutex->hp_waiter->pid);
+                else
+                        TRACE_CUR("no further waiters\n");
+                
+                raw_spin_lock(&tsk_rt(t)->hp_blocked_tasks_lock);
+                
+                binheap_add(&l->nest.hp_binheap_node,
+                                        &tsk_rt(t)->hp_blocked_tasks,
+                                        struct nested_info, hp_binheap_node);
+                
+                raw_spin_unlock(&tsk_rt(t)->hp_blocked_tasks_lock);
+        }
+        
+        if (t) {
+                BUG_ON(mutex->owner != t);
+                
+                TRACE_CUR("%s/%d waking up since it is no longer blocked.\n", t->comm, t->pid);
+                
+                tsk_rt(t)->blocked_lock = NULL;
+                mb();
+                
+                wake_up_for_lock(t);
+        }
+        
+        return (t != NULL);
+}
 
-        wait_queue_t            *q;
-        struct list_head        *pos;
-        struct task_struct  *queued;
-        int i;
-        int ret = 0;
+static inline struct task_struct* get_head_task(struct prioq_mutex *mutex)
+{
+        wait_queue_t *q_node = list_entry(mutex->wait.task_list.next, wait_queue_t, task_list);
+        return get_queued_task(q_node);
+}
 
+static int __prioq_dgl_increase_pos(struct prioq_mutex *mutex, struct task_struct *t)
+{
+//      (1) Increase position for 't' for all locks it is waiting.
+//  (2) Check to see if 't' can take the lock, DGL or singular lock.
+//  (3) If it can, do so and wake up 't'.
+        
+        struct list_head *pos;
+        struct task_struct *new_head;
+        struct task_struct *cur_head = NULL;
+        dgl_wait_state_t *dgl_wait = NULL;
+        int woke_up = 0;
+        int found = 0;
+
+        
+        BUG_ON(list_empty(&mutex->wait.task_list));
+        
+        /* note the task at the head of the queue */
+        if(mutex->owner == NULL) {
+                cur_head = get_head_task(mutex);
+        }
+        
         list_for_each(pos, &mutex->wait.task_list) {
-                q = list_entry(pos, wait_queue_t, task_list);
-                if(q->func == dgl_wake_up) {
-                        // we're looking at a dgl request
-                        dgl_wait_state_t *dgl_wait = (dgl_wait_state_t*) q->private;
-                        queued = dgl_wait->task;
-
-                        if (queued == t)  // is it the one we're looking for?
-                        {
+                dgl_wait_state_t *temp_dgl_state;
+                wait_queue_t *q = list_entry(pos, wait_queue_t, task_list);
+                struct task_struct *queued = get_queued_task_and_dgl_wait(q, &temp_dgl_state);
+                        
+                if (queued == t) {
+                        
+                        TRACE_CUR("found %s/%d in prioq of lock %d\n",
+                                          t->comm, t->pid,
+                                          mutex->litmus_lock.ident);
+                        
+                        if(temp_dgl_state) { /* it's a DGL request */
+                                int i;
+                                dgl_wait = temp_dgl_state;
+                                
+                                TRACE_CUR("found request for %s/%d is a DGL request of size %d.\n",
+                                                  t->comm, t->pid, dgl_wait->size);
+                                
                                 // reposition on the other mutexes
                                 for(i = 0; i < dgl_wait->size; ++i) {
+                                        // assume they're all PRIOQ_MUTEX
                                         struct prioq_mutex *pm = (struct prioq_mutex *) dgl_wait->locks[i];
                                         if (pm != mutex)
                                                 __prioq_increase_pos(pm, t);
                                 }
-                                // reposition on this mutex
-                                __remove_wait_queue(&mutex->wait, q);
-                                __add_wait_queue_sorted(&mutex->wait, q);
-
-
-                                if(__attempt_atomic_dgl_acquire(NULL, dgl_wait)) {
-                                        /* it can't take the lock. do nothing. */
-                                }
-                                else {
-                                        TRACE_CUR("%s/%d can take its entire DGL atomically via inheritance!\n",
-                                                          dgl_wait->task->comm, dgl_wait->task->pid);
-
-                                        /* we took the lock!  we've already been removed from mutex->wait.task_list */
-
-                                        TRACE_TASK(t, "waking up since it is no longer blocked.\n");
-
-                                        tsk_rt(t)->blocked_lock = NULL;
-                                        mb();
-
-#if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA)
-                                        // re-enable tracking
-                                        if(tsk_rt(t)->held_gpus) {
-                                                tsk_rt(t)->suspend_gpu_tracker_on_block = 0;
-                                        }
-#endif
-                                        wake_up_process(t);
-                                        ret = 1;
-                                }
-                                break;
                         }
+                        
+                        // reposition on this mutex
+                        __remove_wait_queue(&mutex->wait, q);
+                        __add_wait_queue_sorted(&mutex->wait, q);
+                        found = 1;
+                        break;
                 }
-                else {
-                        // not dgl request.
-                        queued = (struct task_struct*) q->private;
-                        if (queued == t) {  // is this the one we're looking for?
-                                                                // if so, reposition it.
-                                __remove_wait_queue(&mutex->wait, q);
-                                __add_wait_queue_sorted(&mutex->wait, q);
+        }
+        
+        BUG_ON(!found);
+        
+        if (mutex->owner == NULL) {
+                /* who is the new head? */
+                new_head = get_head_task(mutex);
+        
+                /* is the prioq mutex idle? */
+                if(cur_head != new_head) {
+                        /* the new head might be able to take the lock */
+                
+                        BUG_ON(new_head != t); /* the new head must be this task since our prio increased */
+                
+                        TRACE_CUR("Change in prioq head on idle prioq mutex %d: old = %s/%d new = %s/%d\n",
+                                          mutex->litmus_lock.ident,
+                                          cur_head->comm, cur_head->pid,
+                                          new_head->comm, new_head->pid);
+                        
+                        woke_up = ___prioq_dgl_acquire_via_inheritance(mutex, t, dgl_wait);
+                }
+        }
+
+        return woke_up;
+}
+
+static int ___prioq_dgl_decrease_pos_and_check_acquire(struct prioq_mutex *mutex, struct task_struct *t, wait_queue_t *q)
+{
+        struct list_head *pos;
+        struct task_struct *new_head;
+        struct task_struct *cur_head = NULL;
+        int woke_up = 0;
+        int found = 1;
+        
+        BUG_ON(list_empty(&mutex->wait.task_list));
+        
+        /* find the position of t in mutex's wait q if it's not provided */
+        if (q == NULL) {
+                found = 0;
+                list_for_each(pos, &mutex->wait.task_list) {
+                        q = list_entry(pos, wait_queue_t, task_list);
+                        if (t == get_queued_task(q)) {
+                                found = 1;
                                 break;
                         }
                 }
         }
 
-        return ret;
+        BUG_ON(!q);
+        BUG_ON(!found);
+        
+        if(mutex->owner == NULL) {
+                cur_head = get_head_task(mutex);
+        }
+        
+        // update the position
+        __remove_wait_queue(&mutex->wait, q);
+        __add_wait_queue_sorted(&mutex->wait, q);
+        
+        if(mutex->owner == NULL) {
+                // get a reference to dgl_wait of the new head is a DGL request
+                dgl_wait_state_t *dgl_wait;
+                q = list_entry(mutex->wait.task_list.next, wait_queue_t, task_list);
+                new_head = get_queued_task_and_dgl_wait(q, &dgl_wait);
+                
+                /* is the prioq mutex idle and did the head change? */
+                if(cur_head != new_head) {
+                        /* the new head might be able to take the lock */
+                        TRACE_CUR("Change in prioq head on idle prioq mutex %d: old = %s/%d new = %s/%d\n",
+                                          mutex->litmus_lock.ident,
+                                          cur_head->comm, cur_head->pid,
+                                          new_head->comm, new_head->pid);
+                        
+                        woke_up = ___prioq_dgl_acquire_via_inheritance(mutex, new_head, dgl_wait);
+                }
+        }
+        return woke_up;
 }
 
 static void __prioq_dgl_decrease_pos(struct prioq_mutex *mutex, struct task_struct *t)
 {
-        // TODO:
-        //      (1) Increase position for 't' in all of it's DGLs.
-        //  (2) Check to see if 't' can take the DGLs atomically
-        //  (3) If it can take the DGLs, do so.
-        //  (4) Cleanup?
-        //  (5) Wake up 't'
-
-
-        wait_queue_t            *q;
+//      (1) Decrease position for 't' for all locks it is waiting.
+//  (2) For every lock upon which 't' was the head AND that lock is idle:
+//  (3)    Can the new head take the lock?
+//  (4)    If it can, do so and wake up the new head.
+        
         struct list_head        *pos;
-        struct task_struct  *queued;
-        int i;
-
+        
+        BUG_ON(list_empty(&mutex->wait.task_list));
+        
         list_for_each(pos, &mutex->wait.task_list) {
-                q = list_entry(pos, wait_queue_t, task_list);
-                if(q->func == dgl_wake_up) {
-                        // we're looking at a dgl request
-                        dgl_wait_state_t *dgl_wait = (dgl_wait_state_t*) q->private;
-                        queued = dgl_wait->task;
-
-                        if (queued == t)  // is it the one we're looking for?
-                        {
-                                // reposition on the other mutexes
+                dgl_wait_state_t *dgl_wait;
+                wait_queue_t *q = list_entry(pos, wait_queue_t, task_list);
+                struct task_struct *queued = get_queued_task_and_dgl_wait(q, &dgl_wait);
+                
+                if (queued == t) {
+                        TRACE_CUR("found %s/%d in prioq of lock %d\n",
+                                          t->comm, t->pid,
+                                          mutex->litmus_lock.ident);                    
+                        
+                        if (dgl_wait) {
+                                // reposition on all mutexes and check for wakeup
+                                int i;
+                                
+                                TRACE_CUR("found request for %s/%d is a DGL request of size %d.\n",
+                                                  t->comm, t->pid, dgl_wait->size);
+                                
                                 for(i = 0; i < dgl_wait->size; ++i) {
-                                        struct prioq_mutex *pm = (struct prioq_mutex *)dgl_wait->locks[i];
+                                        // assume they're all PRIOQ_MUTEX
+                                        struct prioq_mutex *pm = (struct prioq_mutex *) dgl_wait->locks[i];
                                         if (pm != mutex)
-                                                __prioq_decrease_pos(pm, t);
+                                                ___prioq_dgl_decrease_pos_and_check_acquire(pm, t, NULL);
+                                        else
+                                                ___prioq_dgl_decrease_pos_and_check_acquire(pm, t, q);
                                 }
-                                // reposition on this mutex
-                                __remove_wait_queue(&mutex->wait, q);
-                                __add_wait_queue_sorted(&mutex->wait, q);
-                                return;
                         }
-                }
-                else {
-                        // not dgl request.
-                        queued = (struct task_struct*) q->private;
-                        if (queued == t) {  // is this the one we're looking for?
-                                                                // if so, reposition it.
-                                __remove_wait_queue(&mutex->wait, q);
-                                __add_wait_queue_sorted(&mutex->wait, q);
-                                return;
+                        else {
+                                ___prioq_dgl_decrease_pos_and_check_acquire(mutex, t, q);                               
                         }
+                        return;
                 }
         }
-
+        
         BUG();
 }
-#endif
 
 
 
-
-
-/* caller is responsible for locking */
-static struct task_struct* __prioq_mutex_find_hp_waiter(struct prioq_mutex *mutex,
-                                                                                         struct task_struct* skip)
-{
-        wait_queue_t            *q;
-        struct list_head        *pos;
-        struct task_struct  *queued = NULL, *found = NULL;
-
-        /* list in sorted order.  higher-prio tasks likely at the front. */
-        list_for_each(pos, &mutex->wait.task_list) {
-                q = list_entry(pos, wait_queue_t, task_list);
-                queued = get_queued_task(q);
-
-                /* Compare task prios, find high prio task. */
-                if (queued &&
-                        (queued != skip) &&
-                        (tsk_rt(queued)->blocked_lock == &mutex->litmus_lock) &&
-                        litmus->compare(queued, found)) {
-                        found = queued;
-                }
-        }
-        return found;
-}
-
-
-#ifdef CONFIG_LITMUS_DGL_SUPPORT
-
 int prioq_mutex_is_owner(struct litmus_lock *l, struct task_struct *t)
 {
         struct prioq_mutex *mutex = prioq_mutex_from_lock(l);
@@ -474,10 +597,12 @@ static void select_next_lock_if_primary(struct litmus_lock *l,
                                 effective_priority(mutex->hp_waiter) :
                                 NULL;
 
+                        
                         if (mutex->hp_waiter)
-                                TRACE_TASK(mutex->hp_waiter, "is new highest-prio waiter\n");
+                                TRACE_CUR("%s/%d is new highest-prio waiter\n",
+                                                  mutex->hp_waiter->comm, mutex->hp_waiter->pid);
                         else
-                                TRACE("no further waiters\n");
+                                TRACE_CUR("no further waiters\n");
                 }
 
                 select_next_lock(dgl_wait /*, l*/);  // pick the next lock to be blocked on
@@ -487,29 +612,6 @@ static void select_next_lock_if_primary(struct litmus_lock *l,
                                   l->ident, t->comm, t->pid);
         }
 }
-#endif
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-#ifdef CONFIG_LITMUS_DGL_SUPPORT
 
 int prioq_mutex_dgl_can_quick_lock(struct litmus_lock *l, struct task_struct *t)
 {
@@ -569,28 +671,6 @@ void prioq_mutex_dgl_quick_lock(struct litmus_lock *l, struct litmus_lock *cur_l
 
 
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
 int prioq_mutex_lock(struct litmus_lock* l)
 {
         struct task_struct *t = current;
@@ -819,12 +899,12 @@ int prioq_mutex_unlock(struct litmus_lock* l)
                  * DGL waiter can't acquire all locks at once.
                  */
                 wait_queue_t *q = list_entry(mutex->wait.task_list.next, wait_queue_t, task_list);
-                if(q->func == dgl_wake_up) {
-                        dgl_wait = (dgl_wait_state_t*) q->private;
-
+                get_queued_task_and_dgl_wait(q, &dgl_wait);
+                
+                if (dgl_wait) {
                         TRACE_CUR("Checking to see if DGL waiter %s/%d can take its locks\n",
                                           dgl_wait->task->comm, dgl_wait->task->pid);
-
+                        
                         if(__attempt_atomic_dgl_acquire(l, dgl_wait)) {
                                 /* failed. can't take this lock yet. we remain at head of prioq
                                  * allow hp requests in the future to go ahead of us. */
@@ -839,6 +919,8 @@ int prioq_mutex_unlock(struct litmus_lock* l)
 
                 /* remove the first */
                 next = __waitqueue_dgl_remove_first(&mutex->wait, &dgl_wait);
+                
+                BUG_ON(dgl_wait && (next != dgl_wait->task));
         }
 #else
         /* check if there are jobs waiting for this resource */
@@ -852,7 +934,8 @@ int prioq_mutex_unlock(struct litmus_lock* l)
                 /* determine new hp_waiter if necessary */
                 if (next == mutex->hp_waiter) {
 
-                        TRACE_TASK(next, "was highest-prio waiter\n");
+                        TRACE_CUR("%s/%d was highest-prio waiter\n", next->comm, next->pid);
+                        
                         /* next has the highest priority --- it doesn't need to
                          * inherit.  However, we need to make sure that the
                          * next-highest priority in the queue is reflected in
@@ -862,10 +945,13 @@ int prioq_mutex_unlock(struct litmus_lock* l)
                                 effective_priority(mutex->hp_waiter) :
                                 NULL;
 
+                        
                         if (mutex->hp_waiter)
-                                TRACE_TASK(mutex->hp_waiter, "is new highest-prio waiter\n");
+                                TRACE_CUR("%s/%d is new highest-prio waiter\n",
+                                                  mutex->hp_waiter->comm, mutex->hp_waiter->pid);
                         else
-                                TRACE("no further waiters\n");
+                                TRACE_CUR("no further waiters\n");
+                        
 
                         raw_spin_lock(&tsk_rt(next)->hp_blocked_tasks_lock);
 
@@ -879,7 +965,7 @@ int prioq_mutex_unlock(struct litmus_lock* l)
                         /* Well, if 'next' is not the highest-priority waiter,
                          * then it (probably) ought to inherit the highest-priority
                          * waiter's priority. */
-                        TRACE_TASK(next, "is not hp_waiter of lock %d.\n", l->ident);
+                        TRACE_CUR("%s/%d is not hp_waiter of lock %d.\n", next->comm, next->pid, l->ident);
 
                         raw_spin_lock(&tsk_rt(next)->hp_blocked_tasks_lock);
 
@@ -920,19 +1006,12 @@ int prioq_mutex_unlock(struct litmus_lock* l)
 #endif
                 }
 
-                TRACE_TASK(next, "waking up since it is no longer blocked.\n");
+                TRACE_CUR("%s/%d waking up since it is no longer blocked.\n", next->comm, next->pid);
 
                 tsk_rt(next)->blocked_lock = NULL;
                 mb();
 
-#if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA)
-                // re-enable tracking
-                if(tsk_rt(next)->held_gpus) {
-                        tsk_rt(next)->suspend_gpu_tracker_on_block = 0;
-                }
-#endif
-
-                wake_up_process(next);
+                wake_up_for_lock(next);
         }
 
         unlock_fine_irqrestore(&mutex->lock, flags);
diff --git a/litmus/sched_cedf.c b/litmus/sched_cedf.c
index 9b6d25f11c93..40daf8e16d74 100644
--- a/litmus/sched_cedf.c
+++ b/litmus/sched_cedf.c
@@ -1479,13 +1479,13 @@ static void nested_increase_priority_inheritance(struct task_struct* t,
                                                                                                                           irqflags);
                 }
                 else {
-                        TRACE_TASK(t, "Inheritor is blocked on lock (%d) that does not support nesting!\n",
+                        TRACE_TASK(t, "Inheritor is blocked on litmus lock (%d) that does not support nesting!\n",
                                            blocked_lock->ident);
                         unlock_fine_irqrestore(to_unlock, irqflags);
                 }
         }
         else {
-                TRACE_TASK(t, "is not blocked.  No propagation.\n");
+                TRACE_TASK(t, "is not blocked on litmus lock.  No propagation.\n");
                 unlock_fine_irqrestore(to_unlock, irqflags);
         }
 }