Make PRIOQ &DGLs work. hangs w/ ikglp though

7 files changed, 569 insertions, 182 deletions

diff --git a/include/litmus/fifo_lock.h b/include/litmus/fifo_lock.h
index 7f13863f11e6..0b2337b42155 100644
--- a/include/litmus/fifo_lock.h
+++ b/include/litmus/fifo_lock.h
@@ -30,6 +30,7 @@ static inline struct fifo_mutex* fifo_mutex_from_lock(struct litmus_lock* lock)
 
 #ifdef CONFIG_LITMUS_DGL_SUPPORT
 int fifo_mutex_is_owner(struct litmus_lock *l, struct task_struct *t);
+struct task_struct* fifo_mutex_get_owner(struct litmus_lock *l);
 int fifo_mutex_dgl_lock(struct litmus_lock *l, dgl_wait_state_t* dgl_wait, wait_queue_t* wq_node);
 void fifo_mutex_enable_priority(struct litmus_lock *l, dgl_wait_state_t* dgl_wait);
 #endif
diff --git a/include/litmus/locking.h b/include/litmus/locking.h
index 660bfc7f8174..cbc99ee54020 100644
--- a/include/litmus/locking.h
+++ b/include/litmus/locking.h
@@ -11,7 +11,7 @@ struct nested_info
         struct litmus_lock *lock;
         struct task_struct *hp_waiter_eff_prio;
         struct task_struct **hp_waiter_ptr;
-        struct task_struct **owner_ptr;
+//      struct task_struct **owner_ptr;
     struct binheap_node hp_binheap_node;
 };
 
@@ -71,8 +71,7 @@ void select_next_lock(dgl_wait_state_t* dgl_wait /*, struct litmus_lock* prev_lo
 
 void init_dgl_waitqueue_entry(wait_queue_t *wq_node, dgl_wait_state_t* dgl_wait);
 int dgl_wake_up(wait_queue_t *wq_node, unsigned mode, int sync, void *key);
-void __waitqueue_dgl_remove_first(wait_queue_head_t *wq, dgl_wait_state_t** dgl_wait, struct task_struct **task);
-
+struct task_struct* __waitqueue_dgl_remove_first(wait_queue_head_t *wq, dgl_wait_state_t** dgl_wait);
 
 int __attempt_atomic_dgl_acquire(struct litmus_lock *cur_lock, dgl_wait_state_t *dgl_wait);
 #endif
@@ -131,6 +130,7 @@ struct litmus_lock_ops {
         raw_spinlock_t* (*get_dgl_spin_lock)(struct litmus_lock *l);
         int (*dgl_lock)(struct litmus_lock *l, dgl_wait_state_t* dgl_wait, wait_queue_t* wq_node);
         int (*is_owner)(struct litmus_lock *l, struct task_struct *t);
+        struct task_struct* (*get_owner)(struct litmus_lock *l);
         void (*enable_priority)(struct litmus_lock *l, dgl_wait_state_t* dgl_wait);
         
         int (*dgl_can_quick_lock)(struct litmus_lock *l, struct task_struct *t);
diff --git a/include/litmus/prioq_lock.h b/include/litmus/prioq_lock.h
index f3c11d241991..5c135ef0bdc6 100644
--- a/include/litmus/prioq_lock.h
+++ b/include/litmus/prioq_lock.h
@@ -32,6 +32,7 @@ static inline struct prioq_mutex* prioq_mutex_from_lock(struct litmus_lock* lock
 
 #ifdef CONFIG_LITMUS_DGL_SUPPORT
 int prioq_mutex_is_owner(struct litmus_lock *l, struct task_struct *t);
+struct task_struct* prioq_mutex_get_owner(struct litmus_lock *l);
 int prioq_mutex_dgl_lock(struct litmus_lock *l, dgl_wait_state_t* dgl_wait, wait_queue_t* wq_node);
 void prioq_mutex_enable_priority(struct litmus_lock *l, dgl_wait_state_t* dgl_wait);
 void prioq_mutex_dgl_quick_lock(struct litmus_lock *l, struct litmus_lock *cur_lock,
diff --git a/litmus/fifo_lock.c b/litmus/fifo_lock.c
index be49cd3a58d2..cc3e1ab5a965 100644
--- a/litmus/fifo_lock.c
+++ b/litmus/fifo_lock.c
@@ -7,8 +7,6 @@
 
 #include <litmus/litmus_proc.h>
 
-//#include <litmus/edf_common.h>
-
 #if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA)
 #include <litmus/gpu_affinity.h>
 #endif
@@ -22,33 +20,16 @@ static struct task_struct* fifo_mutex_find_hp_waiter(struct fifo_mutex *mutex,
         struct list_head        *pos;
         struct task_struct  *queued = NULL, *found = NULL;
 
-#ifdef CONFIG_LITMUS_DGL_SUPPORT
-        dgl_wait_state_t        *dgl_wait = NULL;
-#endif
-
         list_for_each(pos, &mutex->wait.task_list) {
                 q = list_entry(pos, wait_queue_t, task_list);
 
-#ifdef CONFIG_LITMUS_DGL_SUPPORT
-                if(q->func == dgl_wake_up) {
-                        dgl_wait = (dgl_wait_state_t*) q->private;
-                        if(tsk_rt(dgl_wait->task)->blocked_lock == &mutex->litmus_lock) {
-                                queued = dgl_wait->task;
-                        }
-                        else {
-                                queued = NULL;  // skip it.
-                        }
-                }
-                else {
-                        queued = (struct task_struct*) q->private;
-                }
-#else
-                queued = (struct task_struct*) q->private;
-#endif
+                queued = get_queued_task(q);
 
                 /* Compare task prios, find high prio task. */
-                //if (queued && queued != skip && edf_higher_prio(queued, found)) {
-                if (queued && queued != skip && litmus->compare(queued, found)) {
+                if (queued &&
+                        (queued != skip) &&
+                        (tsk_rt(queued)->blocked_lock == &mutex->litmus_lock) &&
+                        litmus->compare(queued, found)) {
                         found = queued;
                 }
         }
@@ -64,6 +45,12 @@ int fifo_mutex_is_owner(struct litmus_lock *l, struct task_struct *t)
         return(mutex->owner == t);
 }
 
+struct task_struct* fifo_mutex_get_owner(struct litmus_lock *l)
+{
+        struct fifo_mutex *mutex = fifo_mutex_from_lock(l);
+        return(mutex->owner);
+}
+
 // return 1 if resource was immediatly acquired.
 // Assumes mutex->lock is held.
 // Must set task state to TASK_UNINTERRUPTIBLE if task blocks.
@@ -110,12 +97,16 @@ void fifo_mutex_enable_priority(struct litmus_lock *l,
         struct task_struct *owner = mutex->owner;
         unsigned long flags = 0;  // these are unused under DGL coarse-grain locking
 
+        /**************************************
+         * This code looks like it supports fine-grain locking, but it does not!
+         * TODO: Gaurantee that mutex->lock is held by the caller to support fine-grain locking.
+         **************************************/
+
         BUG_ON(owner == t);
 
         tsk_rt(t)->blocked_lock = l;
         mb();
 
-        //if (edf_higher_prio(t, mutex->hp_waiter)) {
         if (litmus->compare(t, mutex->hp_waiter)) {
 
                 struct task_struct *old_max_eff_prio;
@@ -141,7 +132,6 @@ void fifo_mutex_enable_priority(struct litmus_lock *l,
                         TRACE_TASK(t, "is new hp_waiter.\n");
 
                         if ((effective_priority(owner) == old_max_eff_prio) ||
-                                //(__edf_higher_prio(new_max_eff_prio, BASE, owner, EFFECTIVE))){
                                 (litmus->__compare(new_max_eff_prio, BASE, owner, EFFECTIVE))){
                                 new_prio = new_max_eff_prio;
                         }
@@ -234,7 +224,6 @@ int fifo_mutex_lock(struct litmus_lock* l)
                 __add_wait_queue_tail_exclusive(&mutex->wait, &wait);
 
                 /* check if we need to activate priority inheritance */
-                //if (edf_higher_prio(t, mutex->hp_waiter)) {
                 if (litmus->compare(t, mutex->hp_waiter)) {
 
                         struct task_struct *old_max_eff_prio;
@@ -260,7 +249,6 @@ int fifo_mutex_lock(struct litmus_lock* l)
                                 TRACE_TASK(t, "is new hp_waiter.\n");
 
                                 if ((effective_priority(owner) == old_max_eff_prio) ||
-                                        //(__edf_higher_prio(new_max_eff_prio, BASE, owner, EFFECTIVE))){
                                         (litmus->__compare(new_max_eff_prio, BASE, owner, EFFECTIVE))){
                                         new_prio = new_max_eff_prio;
                                 }
@@ -368,8 +356,6 @@ int fifo_mutex_unlock(struct litmus_lock* l)
                           (effective_priority(t) == old_max_eff_prio))  )
                 {
                         // old_max_eff_prio > new_max_eff_prio
-
-                        //if(__edf_higher_prio(new_max_eff_prio, BASE, t, EFFECTIVE)) {
                         if(litmus->__compare(new_max_eff_prio, BASE, t, EFFECTIVE)) {
                                 TRACE_TASK(t, "new_max_eff_prio > task's eff_prio-- new_max_eff_prio: %s/%d   task: %s/%d [%s/%d]\n",
                                                    new_max_eff_prio->comm, new_max_eff_prio->pid,
@@ -395,10 +381,7 @@ int fifo_mutex_unlock(struct litmus_lock* l)
 
         /* check if there are jobs waiting for this resource */
 #ifdef CONFIG_LITMUS_DGL_SUPPORT
-        __waitqueue_dgl_remove_first(&mutex->wait, &dgl_wait, &next);
-        if(dgl_wait) {
-                next = dgl_wait->task;
-        }
+        next = __waitqueue_dgl_remove_first(&mutex->wait, &dgl_wait);
 #else
         next = __waitqueue_remove_first(&mutex->wait);
 #endif
@@ -434,7 +417,6 @@ int fifo_mutex_unlock(struct litmus_lock* l)
 #ifdef CONFIG_LITMUS_DGL_SUPPORT
                         if(dgl_wait) {
                                 select_next_lock_if_primary(l, dgl_wait);
-                                //wake_up_task = atomic_dec_and_test(&dgl_wait->nr_remaining);
                                 --(dgl_wait->nr_remaining);
                                 wake_up_task = (dgl_wait->nr_remaining == 0);
                         }
@@ -476,7 +458,6 @@ int fifo_mutex_unlock(struct litmus_lock* l)
                         {
                                 if(dgl_wait && tsk_rt(next)->blocked_lock) {
                                         BUG_ON(wake_up_task);
-                                        //if(__edf_higher_prio(l->nest.hp_waiter_eff_prio, BASE, next, EFFECTIVE)) {
                                         if(litmus->__compare(l->nest.hp_waiter_eff_prio, BASE, next, EFFECTIVE)) {
                                                 litmus->nested_increase_prio(next,
                                                         l->nest.hp_waiter_eff_prio, &mutex->lock, flags);  // unlocks lock && hp_blocked_tasks_lock.
@@ -557,7 +538,6 @@ void fifo_mutex_propagate_increase_inheritance(struct litmus_lock* l,
 
                 old_max_eff_prio = top_priority(&tsk_rt(owner)->hp_blocked_tasks);
 
-                //if((t != mutex->hp_waiter) && edf_higher_prio(t, mutex->hp_waiter)) {
                 if((t != mutex->hp_waiter) && litmus->compare(t, mutex->hp_waiter)) {
                         TRACE_TASK(t, "is new highest-prio waiter by propagation.\n");
                         mutex->hp_waiter = t;
@@ -580,7 +560,6 @@ void fifo_mutex_propagate_increase_inheritance(struct litmus_lock* l,
                 if(new_max_eff_prio != old_max_eff_prio) {
                         // new_max_eff_prio > old_max_eff_prio holds.
                         if ((effective_priority(owner) == old_max_eff_prio) ||
-                                //(__edf_higher_prio(new_max_eff_prio, BASE, owner, EFFECTIVE))) {
                                 (litmus->__compare(new_max_eff_prio, BASE, owner, EFFECTIVE))) {
                                 TRACE_CUR("Propagating inheritance to holder of lock %d.\n",
                                                   l->ident);
@@ -675,7 +654,6 @@ void fifo_mutex_propagate_decrease_inheritance(struct litmus_lock* l,
                                 TRACE_CUR("Propagating decreased inheritance to holder of lock %d.\n",
                                                   l->ident);
 
-                                //if(__edf_higher_prio(new_max_eff_prio, BASE, owner, BASE)) {
                                 if(litmus->__compare(new_max_eff_prio, BASE, owner, BASE)) {
                                         TRACE_CUR("%s/%d has greater base priority than base priority of owner (%s/%d) of lock %d.\n",
                                                           (new_max_eff_prio) ? new_max_eff_prio->comm : "null",
diff --git a/litmus/locking.c b/litmus/locking.c
index 4fe572c28aea..0b5e162c0c02 100644
--- a/litmus/locking.c
+++ b/litmus/locking.c
@@ -234,8 +234,13 @@ void print_hp_waiters(struct binheap_node* n, int depth)
 
 #ifdef CONFIG_LITMUS_DGL_SUPPORT
 
+struct prioq_mutex;
+
 void select_next_lock(dgl_wait_state_t* dgl_wait /*, struct litmus_lock* prev_lock*/)
 {
+        int start = dgl_wait->last_primary;
+        extern void __dump_prioq_lock_info(struct prioq_mutex *mutex);
+
         /*
          We pick the next lock in reverse order. This causes inheritance propagation
          from locks received earlier to flow in the same direction as regular nested
@@ -244,30 +249,56 @@ void select_next_lock(dgl_wait_state_t* dgl_wait /*, struct litmus_lock* prev_lo
 
         BUG_ON(tsk_rt(dgl_wait->task)->blocked_lock);
 
-        //WARN_ON(dgl_wait->locks[dgl_wait->last_primary] != prev_lock);
+        // note reverse order
+        for(dgl_wait->last_primary = (dgl_wait->last_primary != 0) ?  dgl_wait->last_primary - 1 : dgl_wait->size-1;
+                dgl_wait->last_primary != start;
+                dgl_wait->last_primary = (dgl_wait->last_primary != 0) ?  dgl_wait->last_primary - 1 : dgl_wait->size-1)
+        {
+
+                struct litmus_lock *l = dgl_wait->locks[dgl_wait->last_primary];
+
+                if(!l->ops->is_owner(l, dgl_wait->task) &&
+                   l->ops->get_owner(l)) {
+
+                        tsk_rt(dgl_wait->task)->blocked_lock =
+                                        dgl_wait->locks[dgl_wait->last_primary];
+                        mb();
+
+                        TRACE_TASK(dgl_wait->task, "New blocked lock is %d\n", l->ident);
+
+                        l->ops->enable_priority(l, dgl_wait);
 
-        if (dgl_wait->last_primary == 0) {
-                /* loop around */
-                dgl_wait->last_primary = dgl_wait->size;
+                        return;
+                }
         }
 
+        // There was no one to push on.  This can happen if the blocked task is
+        // behind a task that is idling a prioq-mutex.
+
         // note reverse order
-        for(dgl_wait->last_primary = dgl_wait->last_primary - 1;
-                dgl_wait->last_primary >= 0;
-                --(dgl_wait->last_primary)){
-                if(!dgl_wait->locks[dgl_wait->last_primary]->ops->is_owner(
-                                dgl_wait->locks[dgl_wait->last_primary], dgl_wait->task)) {
+        dgl_wait->last_primary = start;
+        for(dgl_wait->last_primary = (dgl_wait->last_primary != 0) ?  dgl_wait->last_primary - 1 : dgl_wait->size-1;
+                dgl_wait->last_primary != start;
+                dgl_wait->last_primary = (dgl_wait->last_primary != 0) ?  dgl_wait->last_primary - 1 : dgl_wait->size-1)
+        {
+
+                struct litmus_lock *l = dgl_wait->locks[dgl_wait->last_primary];
+
+                if(!l->ops->is_owner(l, dgl_wait->task)) {
 
                         tsk_rt(dgl_wait->task)->blocked_lock =
-                                        dgl_wait->locks[dgl_wait->last_primary];
+                        dgl_wait->locks[dgl_wait->last_primary];
                         mb();
 
-                        TRACE_CUR("New blocked lock is %d\n",
-                                          dgl_wait->locks[dgl_wait->last_primary]->ident);
+                        TRACE_TASK(dgl_wait->task, "New blocked lock is %d\n", l->ident);
 
-                        break;
+                        l->ops->enable_priority(l, dgl_wait);
+
+                        return;
                 }
         }
+
+        BUG();
 }
 
 int dgl_wake_up(wait_queue_t *wq_node, unsigned mode, int sync, void *key)
@@ -277,14 +308,13 @@ int dgl_wake_up(wait_queue_t *wq_node, unsigned mode, int sync, void *key)
         return 1;
 }
 
-void __waitqueue_dgl_remove_first(wait_queue_head_t *wq,
-                                                                  dgl_wait_state_t** dgl_wait,
-                                                                  struct task_struct **task)
+struct task_struct* __waitqueue_dgl_remove_first(wait_queue_head_t *wq,
+                                                                  dgl_wait_state_t** dgl_wait)
 {
         wait_queue_t *q;
+        struct task_struct *task = NULL;
 
         *dgl_wait = NULL;
-        *task = NULL;
 
         if (waitqueue_active(wq)) {
                 q = list_entry(wq->task_list.next,
@@ -292,13 +322,15 @@ void __waitqueue_dgl_remove_first(wait_queue_head_t *wq,
 
                 if(q->func == dgl_wake_up) {
                         *dgl_wait = (dgl_wait_state_t*) q->private;
+                        task = (*dgl_wait)->task;
                 }
                 else {
-                        *task = (struct task_struct*) q->private;
+                        task = (struct task_struct*) q->private;
                 }
 
                 __remove_wait_queue(wq, q);
         }
+        return task;
 }
 
 void init_dgl_waitqueue_entry(wait_queue_t *wq_node, dgl_wait_state_t* dgl_wait)
@@ -354,7 +386,7 @@ int __attempt_atomic_dgl_acquire(struct litmus_lock *cur_lock, dgl_wait_state_t
 
                 l->ops->dgl_quick_lock(l, cur_lock, dgl_wait->task, &dgl_wait->wq_nodes[i]);
 
-                BUG_ON(dgl_wait->task != *(l->nest.owner_ptr));
+                BUG_ON(!(l->ops->is_owner(l, dgl_wait->task)));
         }
 
         return 0; /* success */
@@ -456,15 +488,108 @@ all_acquired:
         return 0;
 }
 
+
+
+static long do_litmus_dgl_atomic_lock(dgl_wait_state_t *dgl_wait)
+{
+        int i;
+        unsigned long irqflags; //, dummyflags;
+        raw_spinlock_t *dgl_lock;
+        struct litmus_lock *l;
+
+#ifdef CONFIG_SCHED_DEBUG_TRACE
+        char dglstr[CONFIG_LITMUS_MAX_DGL_SIZE*5];
+        snprintf_dgl(dglstr, sizeof(dglstr), dgl_wait->locks, dgl_wait->size);
+        TRACE_CUR("Atomic locking DGL with size %d: %s\n", dgl_wait->size, dglstr);
+#endif
+
+        dgl_lock = litmus->get_dgl_spinlock(dgl_wait->task);
+
+        BUG_ON(dgl_wait->task != current);
+
+        raw_spin_lock_irqsave(dgl_lock, irqflags);
+
+
+        dgl_wait->nr_remaining = dgl_wait->size;
+
+        for(i = 0; i < dgl_wait->size; ++i) {
+                struct litmus_lock *l = dgl_wait->locks[i];
+                l->ops->dgl_lock(l, dgl_wait, &dgl_wait->wq_nodes[i]);  // this should be a forced enqueue if atomic DGLs are needed.
+        }
+
+        if(__attempt_atomic_dgl_acquire(NULL, dgl_wait)) {
+                /* Failed to acquire all locks at once.
+                 * Pick a lock to push on and suspend. */
+                TRACE_CUR("Could not atomically acquire all locks.\n");
+
+
+#if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA)
+                // KLUDGE: don't count this suspension as time in the critical gpu
+                // critical section
+                if(tsk_rt(dgl_wait->task)->held_gpus) {
+                        tsk_rt(dgl_wait->task)->suspend_gpu_tracker_on_block = 1;
+                }
+#endif
+                // we are not the owner of any lock, so push on the last one in the DGL
+                // by default.
+
+                l = dgl_wait->locks[dgl_wait->size - 1];
+
+                TRACE_CUR("Activating priority inheritance on lock %d\n",
+                                  l->ident);
+
+                TS_DGL_LOCK_SUSPEND;
+
+                l->ops->enable_priority(l, dgl_wait);
+                dgl_wait->last_primary = dgl_wait->size - 1;
+
+                TRACE_CUR("Suspending for lock %d\n", l->ident);
+
+                raw_spin_unlock_irqrestore(dgl_lock, irqflags);  // free dgl_lock before suspending
+
+                suspend_for_lock(); // suspend!!!
+
+                TS_DGL_LOCK_RESUME;
+
+                TRACE_CUR("Woken up from DGL suspension.\n");
+
+                goto all_acquired;  // we should hold all locks when we wake up.
+        }
+        raw_spin_unlock_irqrestore(dgl_lock, irqflags);
+
+all_acquired:
+
+        dgl_wait->nr_remaining = 0;
+
+        // SANITY CHECK FOR TESTING
+        for(i = 0; i < dgl_wait->size; ++i) {
+                struct litmus_lock *l = dgl_wait->locks[i];
+                BUG_ON(!l->ops->is_owner(l, dgl_wait->task));
+        }
+
+        TRACE_CUR("Acquired entire DGL\n");
+
+        return 0;
+}
+
+
+
+
 static int supports_dgl(struct litmus_lock *l)
 {
         struct litmus_lock_ops* ops = l->ops;
-
         return (ops->dgl_lock                   &&
                         ops->is_owner                   &&
+                        ops->get_owner                  &&
                         ops->enable_priority);
 }
 
+static int needs_atomic_dgl(struct litmus_lock *l)
+{
+        struct litmus_lock_ops* ops = l->ops;
+        return (ops->dgl_quick_lock != NULL);
+}
+
 asmlinkage long sys_litmus_dgl_lock(void* __user usr_dgl_ods, int dgl_size)
 {
         struct task_struct *t = current;
@@ -472,6 +597,8 @@ asmlinkage long sys_litmus_dgl_lock(void* __user usr_dgl_ods, int dgl_size)
         int dgl_ods[MAX_DGL_SIZE];
         int i;
 
+        int num_need_atomic = 0;
+
         dgl_wait_state_t dgl_wait_state;  // lives on the stack until all resources in DGL are held.
 
         if(dgl_size > MAX_DGL_SIZE || dgl_size < 1)
@@ -503,6 +630,10 @@ asmlinkage long sys_litmus_dgl_lock(void* __user usr_dgl_ods, int dgl_size)
                                                           dgl_wait_state.locks[i]->ident);
                                         goto out;
                                 }
+
+                                if (needs_atomic_dgl(dgl_wait_state.locks[i])) {
+                                        ++num_need_atomic;
+                                }
                         }
                         else {
                                 TRACE_CUR("Invalid lock identifier\n");
@@ -510,11 +641,19 @@ asmlinkage long sys_litmus_dgl_lock(void* __user usr_dgl_ods, int dgl_size)
                         }
                 }
 
+                if (num_need_atomic && num_need_atomic != dgl_size) {
+                        TRACE_CUR("All locks in DGL must support atomic acquire if any one does.\n");
+                        goto out;
+                }
+
                 dgl_wait_state.task = t;
                 dgl_wait_state.size = dgl_size;
 
                 TS_DGL_LOCK_START;
-                err = do_litmus_dgl_lock(&dgl_wait_state);
+                if (!num_need_atomic)
+                        err = do_litmus_dgl_lock(&dgl_wait_state);
+                else
+                        err = do_litmus_dgl_atomic_lock(&dgl_wait_state);
 
                 /* Note: task my have been suspended or preempted in between!  Take
                  * this into account when computing overheads. */
diff --git a/litmus/prioq_lock.c b/litmus/prioq_lock.c
index 0091e4c1901e..ff6419ba1a13 100644
--- a/litmus/prioq_lock.c
+++ b/litmus/prioq_lock.c
@@ -12,10 +12,10 @@
 #include <litmus/gpu_affinity.h>
 #endif
 
-static void __attribute__((unused))
-__dump_lock_info(struct prioq_mutex *mutex)
+void __attribute__((unused))
+__dump_prioq_lock_info(struct prioq_mutex *mutex)
 {
-#ifdef CONFIG_SCHED_DEBUG_TRACE 
+#ifdef CONFIG_SCHED_DEBUG_TRACE
         TRACE_CUR("%s (mutex: %p):\n", mutex->litmus_lock.name, mutex);
         TRACE_CUR("owner: %s/%d (inh: %s/%d)\n",
                           (mutex->owner) ?
@@ -48,12 +48,12 @@ __dump_lock_info(struct prioq_mutex *mutex)
                         int enabled = 1;
 #endif
                         q = list_entry(pos, wait_queue_t, task_list);
-                                
+
 #ifdef CONFIG_LITMUS_DGL_SUPPORT
                         if(q->func == dgl_wake_up) {
                                 dgl_wait = (dgl_wait_state_t*) q->private;
                                 blocked_task = dgl_wait->task;
-                                        
+
                                 if(tsk_rt(blocked_task)->blocked_lock != &mutex->litmus_lock)
                                         enabled = 0;
                         }
@@ -92,14 +92,14 @@ static void __add_wait_queue_sorted(wait_queue_head_t *q, wait_queue_t *add_node
         struct task_struct *queued_task;
         struct task_struct *add_task;
         struct list_head *pos;
-        
+
         if (list_empty(pq)) {
                 list_add_tail(&add_node->task_list, pq);
                 return;
         }
-        
+
         add_task = get_queued_task(add_node);
-        
+
         /* less priority than tail?  if so, go to tail */
         q_node = list_entry(pq->prev, wait_queue_t, task_list);
         queued_task = get_queued_task(q_node);
@@ -107,7 +107,7 @@ static void __add_wait_queue_sorted(wait_queue_head_t *q, wait_queue_t *add_node
                 list_add_tail(&add_node->task_list, pq);
                 return;
         }
-        
+
         /* belongs at head or between nodes */
         list_for_each(pos, pq) {
                 q_node = list_entry(pos, wait_queue_t, task_list);
@@ -117,8 +117,9 @@ static void __add_wait_queue_sorted(wait_queue_head_t *q, wait_queue_t *add_node
                         return;
                 }
         }
-        
-        BUG();
+
+        WARN_ON(1);
+        list_add_tail(&add_node->task_list, pq);
 }
 
 static inline void __add_wait_queue_sorted_exclusive(wait_queue_head_t *q, wait_queue_t *wait)
@@ -132,7 +133,7 @@ static void __prioq_increase_pos(struct prioq_mutex *mutex, struct task_struct *
         wait_queue_t            *q;
         struct list_head        *pos;
         struct task_struct  *queued;
-        
+
         /* TODO: Make this efficient instead of remove/add */
         list_for_each(pos, &mutex->wait.task_list) {
                 q = list_entry(pos, wait_queue_t, task_list);
@@ -143,16 +144,17 @@ static void __prioq_increase_pos(struct prioq_mutex *mutex, struct task_struct *
                         return;
                 }
         }
-        
+
         BUG();
 }
 
+
 static void __prioq_decrease_pos(struct prioq_mutex *mutex, struct task_struct *t)
 {
         wait_queue_t            *q;
         struct list_head        *pos;
         struct task_struct  *queued;
-        
+
         /* TODO: Make this efficient instead of remove/add */
         list_for_each(pos, &mutex->wait.task_list) {
                 q = list_entry(pos, wait_queue_t, task_list);
@@ -163,10 +165,143 @@ static void __prioq_decrease_pos(struct prioq_mutex *mutex, struct task_struct *
                         return;
                 }
         }
-        
+
         BUG();
 }
 
+#ifdef CONFIG_LITMUS_DGL_SUPPORT
+static int __prioq_dgl_increase_pos(struct prioq_mutex *mutex, struct task_struct *t)
+{
+// TODO:
+//      (1) Increase position for 't' in all of it's DGLs.
+//  (2) Check to see if 't' can take the DGLs atomically
+//  (3) If it can take the DGLs, do so.
+//  (4) Cleanup?
+//  (5) Wake up 't'
+
+
+        wait_queue_t            *q;
+        struct list_head        *pos;
+        struct task_struct  *queued;
+        int i;
+        int ret = 0;
+
+        list_for_each(pos, &mutex->wait.task_list) {
+                q = list_entry(pos, wait_queue_t, task_list);
+                if(q->func == dgl_wake_up) {
+                        // we're looking at a dgl request
+                        dgl_wait_state_t *dgl_wait = (dgl_wait_state_t*) q->private;
+                        queued = dgl_wait->task;
+
+                        if (queued == t)  // is it the one we're looking for?
+                        {
+                                // reposition on the other mutexes
+                                for(i = 0; i < dgl_wait->size; ++i) {
+                                        struct prioq_mutex *pm = (struct prioq_mutex *) dgl_wait->locks[i];
+                                        if (pm != mutex)
+                                                __prioq_increase_pos(pm, t);
+                                }
+                                // reposition on this mutex
+                                __remove_wait_queue(&mutex->wait, q);
+                                __add_wait_queue_sorted(&mutex->wait, q);
+
+
+                                if(__attempt_atomic_dgl_acquire(NULL, dgl_wait)) {
+                                        /* it can't take the lock. do nothing. */
+                                }
+                                else {
+                                        TRACE_CUR("%s/%d can take its entire DGL atomically via inheritance!\n",
+                                                          dgl_wait->task->comm, dgl_wait->task->pid);
+
+                                        /* we took the lock!  we've already been removed from mutex->wait.task_list */
+
+                                        TRACE_TASK(t, "waking up since it is no longer blocked.\n");
+
+                                        tsk_rt(t)->blocked_lock = NULL;
+                                        mb();
+
+#if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA)
+                                        // re-enable tracking
+                                        if(tsk_rt(t)->held_gpus) {
+                                                tsk_rt(t)->suspend_gpu_tracker_on_block = 0;
+                                        }
+#endif
+                                        wake_up_process(t);
+                                        ret = 1;
+                                }
+                                break;
+                        }
+                }
+                else {
+                        // not dgl request.
+                        queued = (struct task_struct*) q->private;
+                        if (queued == t) {  // is this the one we're looking for?
+                                                                // if so, reposition it.
+                                __remove_wait_queue(&mutex->wait, q);
+                                __add_wait_queue_sorted(&mutex->wait, q);
+                                break;
+                        }
+                }
+        }
+
+        return ret;
+}
+
+static void __prioq_dgl_decrease_pos(struct prioq_mutex *mutex, struct task_struct *t)
+{
+        // TODO:
+        //      (1) Increase position for 't' in all of it's DGLs.
+        //  (2) Check to see if 't' can take the DGLs atomically
+        //  (3) If it can take the DGLs, do so.
+        //  (4) Cleanup?
+        //  (5) Wake up 't'
+
+
+        wait_queue_t            *q;
+        struct list_head        *pos;
+        struct task_struct  *queued;
+        int i;
+
+        list_for_each(pos, &mutex->wait.task_list) {
+                q = list_entry(pos, wait_queue_t, task_list);
+                if(q->func == dgl_wake_up) {
+                        // we're looking at a dgl request
+                        dgl_wait_state_t *dgl_wait = (dgl_wait_state_t*) q->private;
+                        queued = dgl_wait->task;
+
+                        if (queued == t)  // is it the one we're looking for?
+                        {
+                                // reposition on the other mutexes
+                                for(i = 0; i < dgl_wait->size; ++i) {
+                                        struct prioq_mutex *pm = (struct prioq_mutex *)dgl_wait->locks[i];
+                                        if (pm != mutex)
+                                                __prioq_decrease_pos(pm, t);
+                                }
+                                // reposition on this mutex
+                                __remove_wait_queue(&mutex->wait, q);
+                                __add_wait_queue_sorted(&mutex->wait, q);
+                                return;
+                        }
+                }
+                else {
+                        // not dgl request.
+                        queued = (struct task_struct*) q->private;
+                        if (queued == t) {  // is this the one we're looking for?
+                                                                // if so, reposition it.
+                                __remove_wait_queue(&mutex->wait, q);
+                                __add_wait_queue_sorted(&mutex->wait, q);
+                                return;
+                        }
+                }
+        }
+
+        BUG();
+}
+#endif
+
+
+
+
 
 /* caller is responsible for locking */
 static struct task_struct* __prioq_mutex_find_hp_waiter(struct prioq_mutex *mutex,
@@ -182,7 +317,10 @@ static struct task_struct* __prioq_mutex_find_hp_waiter(struct prioq_mutex *mute
                 queued = get_queued_task(q);
 
                 /* Compare task prios, find high prio task. */
-                if (queued && queued != skip && litmus->compare(queued, found)) {
+                if (queued &&
+                        (queued != skip) &&
+                        (tsk_rt(queued)->blocked_lock == &mutex->litmus_lock) &&
+                        litmus->compare(queued, found)) {
                         found = queued;
                 }
         }
@@ -198,6 +336,12 @@ int prioq_mutex_is_owner(struct litmus_lock *l, struct task_struct *t)
         return(mutex->owner == t);
 }
 
+struct task_struct* prioq_mutex_get_owner(struct litmus_lock *l)
+{
+        struct prioq_mutex *mutex = prioq_mutex_from_lock(l);
+        return(mutex->owner);
+}
+
 // return 1 if resource was immediatly acquired.
 // Assumes mutex->lock is held.
 // Must set task state to TASK_UNINTERRUPTIBLE if task blocks.
@@ -211,31 +355,16 @@ int prioq_mutex_dgl_lock(struct litmus_lock *l, dgl_wait_state_t* dgl_wait,
 
         BUG_ON(t != current);
 
-        if (mutex->owner) {
-                TRACE_TASK(t, "Enqueuing on lock %d (held by %s/%d).\n",
-                                        l->ident, mutex->owner->comm, mutex->owner->pid);
-
-                init_dgl_waitqueue_entry(wq_node, dgl_wait);
 
-                set_task_state(t, TASK_UNINTERRUPTIBLE);
-                __add_wait_queue_sorted_exclusive(&mutex->wait, wq_node);
-        } else {
-                TRACE_TASK(t, "Acquired lock %d with no blocking.\n", l->ident);
+        init_dgl_waitqueue_entry(wq_node, dgl_wait);
 
-                /* it's ours now */
-                mutex->owner = t;
-
-                raw_spin_lock(&tsk_rt(t)->hp_blocked_tasks_lock);
-                binheap_add(&l->nest.hp_binheap_node, &tsk_rt(t)->hp_blocked_tasks,
-                                        struct nested_info, hp_binheap_node);
-                raw_spin_unlock(&tsk_rt(t)->hp_blocked_tasks_lock);
-
-                acquired_immediatly = 1;
-        }
+        set_task_state(t, TASK_UNINTERRUPTIBLE);
+        __add_wait_queue_sorted_exclusive(&mutex->wait, wq_node);
 
         return acquired_immediatly;
 }
 
+
 void prioq_mutex_enable_priority(struct litmus_lock *l,
                                                            dgl_wait_state_t* dgl_wait)
 {
@@ -244,11 +373,23 @@ void prioq_mutex_enable_priority(struct litmus_lock *l,
         struct task_struct *owner = mutex->owner;
         unsigned long flags = 0;  // these are unused under DGL coarse-grain locking
 
+        /**************************************
+        * This code looks like it supports fine-grain locking, but it does not!
+        * TODO: Gaurantee that mutex->lock is held by the caller to support fine-grain locking.
+        **************************************/
+
         BUG_ON(owner == t);
 
         tsk_rt(t)->blocked_lock = l;
         mb();
 
+        TRACE_TASK(t, "Enabling prio on lock %d. I am %s/%d  :  cur hp_waiter is %s/%d.\n",
+                           l->ident,
+                           (t) ? t->comm : "null",
+                           (t) ? t->pid : 0,
+                           (mutex->hp_waiter) ? mutex->hp_waiter->comm : "null",
+                           (mutex->hp_waiter) ? mutex->hp_waiter->pid : 0);
+
         if (litmus->compare(t, mutex->hp_waiter)) {
                 struct task_struct *old_max_eff_prio;
                 struct task_struct *new_max_eff_prio;
@@ -260,6 +401,12 @@ void prioq_mutex_enable_priority(struct litmus_lock *l,
                 else
                         TRACE_TASK(t, "has higher prio than hp_waiter (NIL).\n");
 
+
+                if (!owner) {
+                        TRACE_TASK(t, "Enabling priority, but this lock %d is idle.\n", l->ident);
+                        goto out;
+                }
+
                 raw_spin_lock(&tsk_rt(owner)->hp_blocked_tasks_lock);
 
                 old_max_eff_prio = top_priority(&tsk_rt(owner)->hp_blocked_tasks);
@@ -289,26 +436,55 @@ void prioq_mutex_enable_priority(struct litmus_lock *l,
                         raw_spin_unlock(&tsk_rt(owner)->hp_blocked_tasks_lock);
                         unlock_fine_irqrestore(&mutex->lock, flags);
                 }
+
+                return;
         }
-        else {
-                TRACE_TASK(t, "no change in hp_waiter.\n");
-                unlock_fine_irqrestore(&mutex->lock, flags);
-        }
+
+        TRACE_TASK(t, "no change in hp_waiter.\n");
+
+out:
+        unlock_fine_irqrestore(&mutex->lock, flags);
 }
 
 static void select_next_lock_if_primary(struct litmus_lock *l,
                                                                                 dgl_wait_state_t *dgl_wait)
 {
-        if(tsk_rt(dgl_wait->task)->blocked_lock == l) {
+        struct task_struct *t = dgl_wait->task;
+
+        if(tsk_rt(t)->blocked_lock == l) {
+                struct prioq_mutex *mutex = prioq_mutex_from_lock(l);
+
                 TRACE_CUR("Lock %d in DGL was primary for %s/%d.\n",
-                                  l->ident, dgl_wait->task->comm, dgl_wait->task->pid);
-                tsk_rt(dgl_wait->task)->blocked_lock = NULL;
+                                  l->ident, t->comm, t->pid);
+
+                tsk_rt(t)->blocked_lock = NULL;
                 mb();
+
+
+                /* determine new hp_waiter if necessary */
+                if (t == mutex->hp_waiter) {
+
+                        TRACE_TASK(t, "Deciding to not be hp waiter on lock %d any more.\n", l->ident);
+                        /* next has the highest priority --- it doesn't need to
+                         * inherit.  However, we need to make sure that the
+                         * next-highest priority in the queue is reflected in
+                         * hp_waiter. */
+                        mutex->hp_waiter = __prioq_mutex_find_hp_waiter(mutex, t);
+                        l->nest.hp_waiter_eff_prio = (mutex->hp_waiter) ?
+                                effective_priority(mutex->hp_waiter) :
+                                NULL;
+
+                        if (mutex->hp_waiter)
+                                TRACE_TASK(mutex->hp_waiter, "is new highest-prio waiter\n");
+                        else
+                                TRACE("no further waiters\n");
+                }
+
                 select_next_lock(dgl_wait /*, l*/);  // pick the next lock to be blocked on
         }
         else {
                 TRACE_CUR("Got lock early! Lock %d in DGL was NOT primary for %s/%d.\n",
-                                  l->ident, dgl_wait->task->comm, dgl_wait->task->pid);
+                                  l->ident, t->comm, t->pid);
         }
 }
 #endif
@@ -316,13 +492,30 @@ static void select_next_lock_if_primary(struct litmus_lock *l,
 
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 #ifdef CONFIG_LITMUS_DGL_SUPPORT
 
 int prioq_mutex_dgl_can_quick_lock(struct litmus_lock *l, struct task_struct *t)
 {
         struct prioq_mutex *mutex = prioq_mutex_from_lock(l);
-        
-        if(!mutex->owner && mutex->hp_waiter == t) {
+
+        if(!mutex->owner) {
                 wait_queue_t *front = list_entry(mutex->wait.task_list.next, wait_queue_t, task_list);
                 struct task_struct *at_front = get_queued_task(front);
                 if(t == at_front) {
@@ -336,40 +529,68 @@ void prioq_mutex_dgl_quick_lock(struct litmus_lock *l, struct litmus_lock *cur_l
                                                                 struct task_struct* t, wait_queue_t *q)
 {
         struct prioq_mutex *mutex = prioq_mutex_from_lock(l);
-        
+
         BUG_ON(mutex->owner);
-        BUG_ON(mutex->hp_waiter != t);
         BUG_ON(t != get_queued_task(list_entry(mutex->wait.task_list.next, wait_queue_t, task_list)));
-        
+
 
         mutex->owner = t;
-        
+
         if (l != cur_lock) {
                 /* we have to update the state of the other lock for it */
                 __remove_wait_queue(&mutex->wait, q);
-                
+
                 mutex->hp_waiter = __prioq_mutex_find_hp_waiter(mutex, t);
                 l->nest.hp_waiter_eff_prio = (mutex->hp_waiter) ?
                         effective_priority(mutex->hp_waiter) :
                         NULL;
-                
+
                 if (mutex->hp_waiter)
                         TRACE_TASK(mutex->hp_waiter, "is new highest-prio waiter\n");
                 else
                         TRACE("no further waiters\n");
-                
+
                 raw_spin_lock(&tsk_rt(t)->hp_blocked_tasks_lock);
-                
+
                 binheap_add(&l->nest.hp_binheap_node,
                                         &tsk_rt(t)->hp_blocked_tasks,
                                         struct nested_info, hp_binheap_node);
-                
+
                 raw_spin_unlock(&tsk_rt(t)->hp_blocked_tasks_lock);
         }
+        else {
+                /* the unlock call that triggered the quick_lock call will handle
+                 * the acquire of cur_lock.
+                 */
+        }
 }
 #endif
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 int prioq_mutex_lock(struct litmus_lock* l)
 {
         struct task_struct *t = current;
@@ -394,9 +615,12 @@ int prioq_mutex_lock(struct litmus_lock* l)
 
         /* block if there is an owner, or if hp_waiter is blocked for DGL and
          * prio(t) < prio(hp_waiter) */
-        if (mutex->owner) {
+        if (mutex->owner ||
+                (waitqueue_active(&mutex->wait) && litmus->compare(mutex->hp_waiter, t))) {
                 TRACE_TASK(t, "Blocking on lock %d (held by %s/%d).\n",
-                                                l->ident, mutex->owner->comm, mutex->owner->pid);
+                                        l->ident,
+                                   (mutex->owner) ? mutex->owner->comm : "null",
+                                   (mutex->owner) ? mutex->owner->pid : 0);
 
 #if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA)
                 // KLUDGE: don't count this suspension as time in the critical gpu
@@ -421,7 +645,9 @@ int prioq_mutex_lock(struct litmus_lock* l)
                 __add_wait_queue_sorted_exclusive(&mutex->wait, &wait);
 
                 /* check if we need to activate priority inheritance */
-                if (litmus->compare(t, mutex->hp_waiter)) {
+                /* We can't be the hp waiter if there is no owner - task waiting for
+                 * the full DGL must be the hp_waiter. */
+                if (owner && litmus->compare(t, mutex->hp_waiter)) {
 
                         struct task_struct *old_max_eff_prio;
                         struct task_struct *new_max_eff_prio;
@@ -437,10 +663,10 @@ int prioq_mutex_lock(struct litmus_lock* l)
 
                         old_max_eff_prio = top_priority(&tsk_rt(owner)->hp_blocked_tasks);
                         mutex->hp_waiter = t;
-                        
+
                         TRACE_TASK(t, "prioq_mutex %d state after enqeue in priority queue\n", l->ident);
-                        __dump_lock_info(mutex);
-                        
+                        __dump_prioq_lock_info(mutex);
+
                         l->nest.hp_waiter_eff_prio = effective_priority(mutex->hp_waiter);
                         binheap_decrease(&l->nest.hp_binheap_node,
                                                          &tsk_rt(owner)->hp_blocked_tasks);
@@ -471,8 +697,8 @@ int prioq_mutex_lock(struct litmus_lock* l)
                         TRACE_TASK(t, "no change in hp_waiter.\n");
 
                         TRACE_TASK(t, "prioq_mutex %d state after enqeue in priority queue\n", l->ident);
-                        __dump_lock_info(mutex);
-                        
+                        __dump_prioq_lock_info(mutex);
+
                         unlock_fine_irqrestore(&mutex->lock, flags);
                 }
 
@@ -525,8 +751,6 @@ int prioq_mutex_unlock(struct litmus_lock* l)
 
         struct task_struct *old_max_eff_prio;
 
-        int wake_up_task = 1;
-
 #ifdef CONFIG_LITMUS_DGL_SUPPORT
         dgl_wait_state_t *dgl_wait = NULL;
         raw_spinlock_t *dgl_lock = litmus->get_dgl_spinlock(t);
@@ -584,14 +808,40 @@ int prioq_mutex_unlock(struct litmus_lock* l)
         raw_spin_unlock(&tsk_rt(t)->hp_blocked_tasks_lock);
 
 
-        /* check if there are jobs waiting for this resource */
+
+
+        mutex->owner = NULL;
+
 #ifdef CONFIG_LITMUS_DGL_SUPPORT
-        __waitqueue_dgl_remove_first(&mutex->wait, &dgl_wait, &next);
-        if(dgl_wait) {
-                next = dgl_wait->task;
+        if(waitqueue_active(&mutex->wait)) {
+                /* Priority queue-based locks must be _acquired_ atomically under DGLs
+                 * in order to avoid deadlock.  We leave this lock idle momentarily the
+                 * DGL waiter can't acquire all locks at once.
+                 */
+                wait_queue_t *q = list_entry(mutex->wait.task_list.next, wait_queue_t, task_list);
+                if(q->func == dgl_wake_up) {
+                        dgl_wait = (dgl_wait_state_t*) q->private;
+
+                        TRACE_CUR("Checking to see if DGL waiter %s/%d can take its locks\n",
+                                          dgl_wait->task->comm, dgl_wait->task->pid);
+
+                        if(__attempt_atomic_dgl_acquire(l, dgl_wait)) {
+                                /* failed. can't take this lock yet. we remain at head of prioq
+                                 * allow hp requests in the future to go ahead of us. */
+                                select_next_lock_if_primary(l, dgl_wait);
+                                goto out;
+                        }
+                        else {
+                                TRACE_CUR("%s/%d can take its entire DGL atomically.\n",
+                                                  dgl_wait->task->comm, dgl_wait->task->pid);
+                        }
+                }
+
+                /* remove the first */
+                next = __waitqueue_dgl_remove_first(&mutex->wait, &dgl_wait);
         }
 #else
-
+        /* check if there are jobs waiting for this resource */
         next = __waitqueue_remove_first(&mutex->wait);
 #endif
         if (next) {
@@ -623,14 +873,6 @@ int prioq_mutex_unlock(struct litmus_lock* l)
                                                 &tsk_rt(next)->hp_blocked_tasks,
                                                 struct nested_info, hp_binheap_node);
 
-#ifdef CONFIG_LITMUS_DGL_SUPPORT
-                        if(dgl_wait) {
-                                select_next_lock_if_primary(l, dgl_wait);
-                                --(dgl_wait->nr_remaining);
-                                wake_up_task = (dgl_wait->nr_remaining == 0);
-                        }
-#endif
-
                         raw_spin_unlock(&tsk_rt(next)->hp_blocked_tasks_lock);
                 }
                 else {
@@ -678,28 +920,19 @@ int prioq_mutex_unlock(struct litmus_lock* l)
 #endif
                 }
 
-                if(wake_up_task) {
-                        TRACE_TASK(next, "waking up since it is no longer blocked.\n");
+                TRACE_TASK(next, "waking up since it is no longer blocked.\n");
 
-                        tsk_rt(next)->blocked_lock = NULL;
-                        mb();
+                tsk_rt(next)->blocked_lock = NULL;
+                mb();
 
 #if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA)
-                        // re-enable tracking
-                        if(tsk_rt(next)->held_gpus) {
-                                tsk_rt(next)->suspend_gpu_tracker_on_block = 0;
-                        }
+                // re-enable tracking
+                if(tsk_rt(next)->held_gpus) {
+                        tsk_rt(next)->suspend_gpu_tracker_on_block = 0;
+                }
 #endif
 
-                        wake_up_process(next);
-                }
-                else {
-                        TRACE_TASK(next, "is still blocked.\n");
-                }
-        }
-        else {
-                /* becomes available */
-                mutex->owner = NULL;
+                wake_up_process(next);
         }
 
         unlock_fine_irqrestore(&mutex->lock, flags);
@@ -726,14 +959,34 @@ void prioq_mutex_propagate_increase_inheritance(struct litmus_lock* l,
         lock_fine(&mutex->lock);
         unlock_fine(to_unlock);
 
+#ifdef CONFIG_LITMUS_DGL_SUPPORT
+        {
+                int woke_up = __prioq_dgl_increase_pos(mutex, t);
+                if (woke_up) {
+                        /* t got the DGL. it is not blocked anywhere. just return. */
+                        unlock_fine_irqrestore(&mutex->lock, irqflags);
+                        return;
+                }
+        }
+#else
         __prioq_increase_pos(mutex, t);
-        
+#endif
+
         if(tsk_rt(t)->blocked_lock == l) {  // prevent race on tsk_rt(t)->blocked
                 struct task_struct *owner = mutex->owner;
 
                 struct task_struct *old_max_eff_prio;
                 struct task_struct *new_max_eff_prio;
-                
+
+                if (!owner) {
+                        TRACE_TASK(t, "Owner on PRIOQ lock %d is null. Don't propagate.\n", l->ident);
+                        if(t == mutex->hp_waiter) {
+                                // reflect the changed prio.
+                                l->nest.hp_waiter_eff_prio = effective_priority(mutex->hp_waiter);
+                        }
+                        return;
+                }
+
                 raw_spin_lock(&tsk_rt(owner)->hp_blocked_tasks_lock);
 
                 old_max_eff_prio = top_priority(&tsk_rt(owner)->hp_blocked_tasks);
@@ -741,17 +994,17 @@ void prioq_mutex_propagate_increase_inheritance(struct litmus_lock* l,
                 if((t != mutex->hp_waiter) && litmus->compare(t, mutex->hp_waiter)) {
                         TRACE_TASK(t, "is new highest-prio waiter by propagation.\n");
                         mutex->hp_waiter = t;
-                        
+
                         TRACE_TASK(t, "prioq_mutex %d state after prio increase in priority queue\n", l->ident);
-                        __dump_lock_info(mutex);
+                        __dump_prioq_lock_info(mutex);
                 }
                 else {
                         TRACE_TASK(t, "prioq_mutex %d state after prio increase in priority queue\n", l->ident);
-                        __dump_lock_info(mutex);
+                        __dump_prioq_lock_info(mutex);
                 }
-                
+
                 if(t == mutex->hp_waiter) {
-                        // reflect the decreased priority in the heap node.
+                        // reflect the increased priority in the heap node.
                         l->nest.hp_waiter_eff_prio = effective_priority(mutex->hp_waiter);
 
                         BUG_ON(!binheap_is_in_heap(&l->nest.hp_binheap_node));
@@ -791,10 +1044,10 @@ void prioq_mutex_propagate_increase_inheritance(struct litmus_lock* l,
         }
         else {
                 struct litmus_lock *still_blocked;
-                
+
                 TRACE_TASK(t, "prioq_mutex %d state after prio increase in priority queue\n", l->ident);
-                __dump_lock_info(mutex);
-                
+                __dump_prioq_lock_info(mutex);
+
                 still_blocked = tsk_rt(t)->blocked_lock;
 
                 TRACE_TASK(t, "is not blocked on lock %d.\n", l->ident);
@@ -836,8 +1089,12 @@ void prioq_mutex_propagate_decrease_inheritance(struct litmus_lock* l,
         lock_fine(&mutex->lock);
         unlock_fine(to_unlock);
 
+#ifdef CONFIG_LITMUS_DGL_SUPPORT
+        __prioq_dgl_decrease_pos(mutex, t);
+#else
         __prioq_decrease_pos(mutex, t);
-        
+#endif
+
         if(tsk_rt(t)->blocked_lock == l) {  // prevent race on tsk_rt(t)->blocked
                 if(t == mutex->hp_waiter) {
                         struct task_struct *owner = mutex->owner;
@@ -845,16 +1102,23 @@ void prioq_mutex_propagate_decrease_inheritance(struct litmus_lock* l,
                         struct task_struct *old_max_eff_prio;
                         struct task_struct *new_max_eff_prio;
 
+                        if (!owner) {
+                                TRACE_TASK(t, "Owner on PRIOQ lock %d is null. Don't propagate.\n", l->ident);
+                                // reflect the changed prio.
+                                l->nest.hp_waiter_eff_prio = effective_priority(mutex->hp_waiter);
+                                return;
+                        }
+
                         raw_spin_lock(&tsk_rt(owner)->hp_blocked_tasks_lock);
 
                         old_max_eff_prio = top_priority(&tsk_rt(owner)->hp_blocked_tasks);
 
                         binheap_delete(&l->nest.hp_binheap_node, &tsk_rt(owner)->hp_blocked_tasks);
                         mutex->hp_waiter = __prioq_mutex_find_hp_waiter(mutex, NULL);
-                        
+
                         TRACE_TASK(t, "prioq_mutex %d state after prio decrease in priority queue\n", l->ident);
-                        __dump_lock_info(mutex);
-                        
+                        __dump_prioq_lock_info(mutex);
+
                         l->nest.hp_waiter_eff_prio = (mutex->hp_waiter) ?
                                 effective_priority(mutex->hp_waiter) : NULL;
                         binheap_add(&l->nest.hp_binheap_node,
@@ -904,18 +1168,18 @@ void prioq_mutex_propagate_decrease_inheritance(struct litmus_lock* l,
                 }
                 else {
                         TRACE_TASK(t, "prioq_mutex %d state after prio decrease in priority queue\n", l->ident);
-                        __dump_lock_info(mutex);
-                        
+                        __dump_prioq_lock_info(mutex);
+
                         TRACE_TASK(t, "is not hp_waiter.  No propagation.\n");
                         unlock_fine_irqrestore(&mutex->lock, irqflags);
                 }
         }
         else {
                 struct litmus_lock *still_blocked;
-                
+
                 TRACE_TASK(t, "prioq_mutex %d state after prio decrease in priority queue\n", l->ident);
-                __dump_lock_info(mutex);
-                
+                __dump_prioq_lock_info(mutex);
+
                 still_blocked = tsk_rt(t)->blocked_lock;
 
                 TRACE_TASK(t, "is not blocked on lock %d.\n", l->ident);
@@ -1153,8 +1417,7 @@ struct litmus_lock* prioq_mutex_new(struct litmus_lock_ops* ops)
 #endif
 
         ((struct litmus_lock*)mutex)->nest.hp_waiter_ptr = &mutex->hp_waiter;
-        ((struct litmus_lock*)mutex)->nest.owner_ptr = &mutex->owner;
-        
+
         ((struct litmus_lock*)mutex)->proc = &prioq_proc_ops;
 
         return &mutex->litmus_lock;
diff --git a/litmus/sched_cedf.c b/litmus/sched_cedf.c
index 61d682d9f415..9b6d25f11c93 100644
--- a/litmus/sched_cedf.c
+++ b/litmus/sched_cedf.c
@@ -1542,7 +1542,11 @@ static struct litmus_lock_ops cedf_fifo_mutex_lock_ops = {
 #ifdef CONFIG_LITMUS_DGL_SUPPORT
         .dgl_lock = fifo_mutex_dgl_lock,
         .is_owner = fifo_mutex_is_owner,
+        .get_owner = fifo_mutex_get_owner,
         .enable_priority = fifo_mutex_enable_priority,
+
+        .dgl_can_quick_lock = NULL,
+        .dgl_quick_lock = NULL,
 #endif
 };
 
@@ -1558,15 +1562,16 @@ static struct litmus_lock_ops cedf_prioq_mutex_lock_ops = {
         .unlock = prioq_mutex_unlock,
         .close  = prioq_mutex_close,
         .deallocate = prioq_mutex_free,
-        
+
         .propagate_increase_inheritance = prioq_mutex_propagate_increase_inheritance,
         .propagate_decrease_inheritance = prioq_mutex_propagate_decrease_inheritance,
-        
+
 #ifdef CONFIG_LITMUS_DGL_SUPPORT
         .dgl_lock = prioq_mutex_dgl_lock,
         .is_owner = prioq_mutex_is_owner,
+        .get_owner = prioq_mutex_get_owner,
         .enable_priority = prioq_mutex_enable_priority,
-        
+
         .dgl_can_quick_lock = prioq_mutex_dgl_can_quick_lock,
         .dgl_quick_lock = prioq_mutex_dgl_quick_lock,
 #endif
@@ -1637,7 +1642,7 @@ static long cedf_allocate_lock(struct litmus_lock **lock, int type,
                 case PRIOQ_MUTEX:
                         *lock = cedf_new_prioq_mutex();
                         break;
-                        
+
                 case IKGLP_SEM:
                         *lock = cedf_new_ikglp(args);
                         break;