BUG FIX: Support DGLs with PRIOQ_MUTEXwip-prioq-dgl

First 'working' implementation of DGLs with PRIOQ_MUTEX.
(All other code prior was work-in-progress.)

General approach:
Because of priority queue order, PRIOQ_MUTEX DGLs must be
*acquired* atomically. This means that a task cannot
acquire an available PRIOQ_MUTEX if another PRIOQ_MUTEX
is not available at the same time.  Requests are buffered
in PRIOQ_MUTEX and the resource 'idles'-- that is, the
mutex owner is NULL, but there are waiting tasks for the
resource.

Several notes/side-effects:
1) A high-priority task that idles a resource can
effectively block lower-priority tasks from acquiring
that resource.  This is because the low-prio task
cannot skip ahead of the high-prio task in the priority
queue.
2) Priority inheritance from nesting can cause the
low-prioity task in #1 to jump over the high-priority
task and acquire the resource.  This means that
any task blocked on a DGL that receives an increase
in priority while blocked on the DGL must trigger
a re-eval of the locks it can take.  If the resources
can be acquired, then the task needs to be woken
up!  <<<<< Lock acquisition via inheritance is entirely
new and weird! >>>>>
3) A similar case for #2 exists for priorty decreases
(example: this can happen when a task loses a donor)
while it is blocked on a PRIOQ_MUTEX. The high-priority
task described in #1 can change and become a lower-
priority task.  Every idle lock (mutex owner is NULL)
on which the task losing priority must be revaluated---
it is possible that the (possible) new head on the
priority queue can take the lock.  Note: This affects
BOTH singular and DGL resource requests, while
the case described in #2 only affects DGL requests
(because a singular request at the head of the priority
queue will never idle a resource).

6 files changed, 152 insertions, 152 deletions

diff --git a/include/litmus/locking.h b/include/litmus/locking.h
index b1024e397f67..fc437811d2b6 100644
--- a/include/litmus/locking.h
+++ b/include/litmus/locking.h
@@ -11,7 +11,6 @@ struct nested_info
         struct litmus_lock *lock;
         struct task_struct *hp_waiter_eff_prio;
         struct task_struct **hp_waiter_ptr;
-//      struct task_struct **owner_ptr;
     struct binheap_node hp_binheap_node;
 };
 
@@ -134,24 +133,31 @@ struct litmus_lock_ops {
         /* The lock is no longer being referenced (mandatory method). */
         lock_free_t deallocate;
 
-        
+
 #ifdef CONFIG_LITMUS_NESTED_LOCKING
         void (*propagate_increase_inheritance)(struct litmus_lock* l, struct task_struct* t, raw_spinlock_t* to_unlock, unsigned long irqflags);
         void (*propagate_decrease_inheritance)(struct litmus_lock* l, struct task_struct* t, raw_spinlock_t* to_unlock, unsigned long irqflags);
 #endif
-
-        
 #ifdef CONFIG_LITMUS_DGL_SUPPORT
         raw_spinlock_t* (*get_dgl_spin_lock)(struct litmus_lock *l);
         int (*dgl_lock)(struct litmus_lock *l, dgl_wait_state_t* dgl_wait, wait_queue_t* wq_node);
         int (*is_owner)(struct litmus_lock *l, struct task_struct *t);
         struct task_struct* (*get_owner)(struct litmus_lock *l);
         void (*enable_priority)(struct litmus_lock *l, dgl_wait_state_t* dgl_wait);
-        
+
         int (*dgl_can_quick_lock)(struct litmus_lock *l, struct task_struct *t);
         void (*dgl_quick_lock)(struct litmus_lock *l, struct litmus_lock *cur_lock,
                   struct task_struct* t, wait_queue_t *q);
 #endif
+
+        /* all flags at the end */
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+        int supports_nesting:1;
+#endif
+#ifdef CONFIG_LITMUS_DGL_SUPPORT
+        int supports_dgl:1;
+        int requires_atomic_dgl:1;
+#endif
 };
 
 
diff --git a/include/litmus/prioq_lock.h b/include/litmus/prioq_lock.h
index 5c135ef0bdc6..1b0a591ef1a6 100644
--- a/include/litmus/prioq_lock.h
+++ b/include/litmus/prioq_lock.h
@@ -34,6 +34,7 @@ static inline struct prioq_mutex* prioq_mutex_from_lock(struct litmus_lock* lock
 int prioq_mutex_is_owner(struct litmus_lock *l, struct task_struct *t);
 struct task_struct* prioq_mutex_get_owner(struct litmus_lock *l);
 int prioq_mutex_dgl_lock(struct litmus_lock *l, dgl_wait_state_t* dgl_wait, wait_queue_t* wq_node);
+int prioq_mutex_dgl_unlock(struct litmus_lock *l);
 void prioq_mutex_enable_priority(struct litmus_lock *l, dgl_wait_state_t* dgl_wait);
 void prioq_mutex_dgl_quick_lock(struct litmus_lock *l, struct litmus_lock *cur_lock,
                                                                 struct task_struct* t, wait_queue_t *q);
diff --git a/litmus/ikglp_lock.c b/litmus/ikglp_lock.c
index 3d79e41b42df..3fd760799a75 100644
--- a/litmus/ikglp_lock.c
+++ b/litmus/ikglp_lock.c
@@ -1401,7 +1401,9 @@ int ikglp_unlock(struct litmus_lock* l)
                                                                 struct nested_info, hp_binheap_node);
                         ++count;
                 }
-                litmus->decrease_prio(t, NULL);
+                if (count) {
+                        litmus->decrease_prio(t, NULL);
+                }
                 WARN_ON(count > 2); // should not be greater than 2.  only local fq inh and donation can be possible.
         }
         raw_spin_unlock(&tsk_rt(t)->hp_blocked_tasks_lock);
diff --git a/litmus/locking.c b/litmus/locking.c
index 4b8382cd77d1..eddc67a4d36a 100644
--- a/litmus/locking.c
+++ b/litmus/locking.c
@@ -365,18 +365,6 @@ static void snprintf_dgl(char* buf, size_t bsz, struct litmus_lock* dgl_locks[],
 #endif
 
 
-static int failed_owner(struct litmus_lock *cur_lock, struct task_struct *t)
-{
-        struct task_struct *cur_owner = cur_lock->ops->get_owner(cur_lock);
-        printk(KERN_EMERG "lock %d expected owner %s/%d but got %s/%d.\n",
-                                        cur_lock->ident,
-                                        (t) ? t->comm : "null",
-                                        (t) ? t->pid : 0,
-                                        (cur_owner) ? cur_owner->comm : "null",
-                                        (cur_owner) ? cur_owner->pid : 0);
-        BUG();
-}
-
 /* only valid when locks are prioq locks!!!
  * THE BIG DGL LOCK MUST BE HELD! */
 int __attempt_atomic_dgl_acquire(struct litmus_lock *cur_lock, dgl_wait_state_t *dgl_wait)
@@ -395,12 +383,8 @@ int __attempt_atomic_dgl_acquire(struct litmus_lock *cur_lock, dgl_wait_state_t
         /* take the locks */
         for(i = 0; i < dgl_wait->size; ++i) {
                 struct litmus_lock *l = dgl_wait->locks[i];
-
                 l->ops->dgl_quick_lock(l, cur_lock, dgl_wait->task, &dgl_wait->wq_nodes[i]);
-
-                if(!(l->ops->is_owner(l, dgl_wait->task)))
-                        failed_owner(l, dgl_wait->task);
-                //BUG_ON(!(l->ops->is_owner(l, dgl_wait->task)));
+                BUG_ON(!(l->ops->is_owner(l, dgl_wait->task)));
         }
 
         return 0; /* success */
@@ -510,6 +494,7 @@ static long do_litmus_dgl_atomic_lock(dgl_wait_state_t *dgl_wait)
         unsigned long irqflags; //, dummyflags;
         raw_spinlock_t *dgl_lock;
         struct litmus_lock *l;
+        struct task_struct *t = current;
 
 #ifdef CONFIG_SCHED_DEBUG_TRACE
         char dglstr[CONFIG_LITMUS_MAX_DGL_SIZE*5];
@@ -519,7 +504,7 @@ static long do_litmus_dgl_atomic_lock(dgl_wait_state_t *dgl_wait)
 
         dgl_lock = litmus->get_dgl_spinlock(dgl_wait->task);
 
-        BUG_ON(dgl_wait->task != current);
+        BUG_ON(dgl_wait->task != t);
 
         raw_spin_lock_irqsave(dgl_lock, irqflags);
 
@@ -528,7 +513,8 @@ static long do_litmus_dgl_atomic_lock(dgl_wait_state_t *dgl_wait)
 
         for(i = 0; i < dgl_wait->size; ++i) {
                 struct litmus_lock *l = dgl_wait->locks[i];
-                l->ops->dgl_lock(l, dgl_wait, &dgl_wait->wq_nodes[i]);  // this should be a forced enqueue if atomic DGLs are needed.
+                // this should be a forced enqueue if atomic DGLs are needed.
+                l->ops->dgl_lock(l, dgl_wait, &dgl_wait->wq_nodes[i]);
         }
 
         if(__attempt_atomic_dgl_acquire(NULL, dgl_wait)) {
@@ -536,27 +522,26 @@ static long do_litmus_dgl_atomic_lock(dgl_wait_state_t *dgl_wait)
                  * Pick a lock to push on and suspend. */
                 TRACE_CUR("Could not atomically acquire all locks.\n");
 
+                /* we set the uninterruptible state here since
+                 * __attempt_atomic_dgl_acquire() may actually succeed. */
+                set_task_state(t, TASK_UNINTERRUPTIBLE);
 
 #if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA)
                 // KLUDGE: don't count this suspension as time in the critical gpu
                 // critical section
-                if(tsk_rt(dgl_wait->task)->held_gpus) {
-                        tsk_rt(dgl_wait->task)->suspend_gpu_tracker_on_block = 1;
+                if(tsk_rt(t)->held_gpus) {
+                        tsk_rt(t)->suspend_gpu_tracker_on_block = 1;