1 files changed, 160 insertions, 21 deletions

diff --git a/litmus/locking.c b/litmus/locking.c
index 4fe572c28aea..0b5e162c0c02 100644
--- a/litmus/locking.c
+++ b/litmus/locking.c
@@ -234,8 +234,13 @@ void print_hp_waiters(struct binheap_node* n, int depth)
 
 #ifdef CONFIG_LITMUS_DGL_SUPPORT
 
+struct prioq_mutex;
+
 void select_next_lock(dgl_wait_state_t* dgl_wait /*, struct litmus_lock* prev_lock*/)
 {
+        int start = dgl_wait->last_primary;
+        extern void __dump_prioq_lock_info(struct prioq_mutex *mutex);
+
         /*
          We pick the next lock in reverse order. This causes inheritance propagation
          from locks received earlier to flow in the same direction as regular nested
@@ -244,30 +249,56 @@ void select_next_lock(dgl_wait_state_t* dgl_wait /*, struct litmus_lock* prev_lo
 
         BUG_ON(tsk_rt(dgl_wait->task)->blocked_lock);
 
-        //WARN_ON(dgl_wait->locks[dgl_wait->last_primary] != prev_lock);
+        // note reverse order
+        for(dgl_wait->last_primary = (dgl_wait->last_primary != 0) ?  dgl_wait->last_primary - 1 : dgl_wait->size-1;
+                dgl_wait->last_primary != start;
+                dgl_wait->last_primary = (dgl_wait->last_primary != 0) ?  dgl_wait->last_primary - 1 : dgl_wait->size-1)
+        {
+
+                struct litmus_lock *l = dgl_wait->locks[dgl_wait->last_primary];
+
+                if(!l->ops->is_owner(l, dgl_wait->task) &&
+                   l->ops->get_owner(l)) {
+
+                        tsk_rt(dgl_wait->task)->blocked_lock =
+                                        dgl_wait->locks[dgl_wait->last_primary];
+                        mb();
+
+                        TRACE_TASK(dgl_wait->task, "New blocked lock is %d\n", l->ident);
+
+                        l->ops->enable_priority(l, dgl_wait);
 
-        if (dgl_wait->last_primary == 0) {
-                /* loop around */
-                dgl_wait->last_primary = dgl_wait->size;
+                        return;
+                }
         }
 
+        // There was no one to push on.  This can happen if the blocked task is
+        // behind a task that is idling a prioq-mutex.
+
         // note reverse order
-        for(dgl_wait->last_primary = dgl_wait->last_primary - 1;
-                dgl_wait->last_primary >= 0;
-                --(dgl_wait->last_primary)){
-                if(!dgl_wait->locks[dgl_wait->last_primary]->ops->is_owner(
-                                dgl_wait->locks[dgl_wait->last_primary], dgl_wait->task)) {
+        dgl_wait->last_primary = start;
+        for(dgl_wait->last_primary = (dgl_wait->last_primary != 0) ?  dgl_wait->last_primary - 1 : dgl_wait->size-1;
+                dgl_wait->last_primary != start;
+                dgl_wait->last_primary = (dgl_wait->last_primary != 0) ?  dgl_wait->last_primary - 1 : dgl_wait->size-1)
+        {
+
+                struct litmus_lock *l = dgl_wait->locks[dgl_wait->last_primary];
+
+                if(!l->ops->is_owner(l, dgl_wait->task)) {
 
                         tsk_rt(dgl_wait->task)->blocked_lock =
-                                        dgl_wait->locks[dgl_wait->last_primary];
+                        dgl_wait->locks[dgl_wait->last_primary];
                         mb();
 
-                        TRACE_CUR("New blocked lock is %d\n",
-                                          dgl_wait->locks[dgl_wait->last_primary]->ident);
+                        TRACE_TASK(dgl_wait->task, "New blocked lock is %d\n", l->ident);
 
-                        break;
+                        l->ops->enable_priority(l, dgl_wait);
+
+                        return;
                 }
         }
+
+        BUG();
 }
 
 int dgl_wake_up(wait_queue_t *wq_node, unsigned mode, int sync, void *key)
@@ -277,14 +308,13 @@ int dgl_wake_up(wait_queue_t *wq_node, unsigned mode, int sync, void *key)
         return 1;
 }
 
-void __waitqueue_dgl_remove_first(wait_queue_head_t *wq,
-                                                                  dgl_wait_state_t** dgl_wait,
-                                                                  struct task_struct **task)
+struct task_struct* __waitqueue_dgl_remove_first(wait_queue_head_t *wq,
+                                                                  dgl_wait_state_t** dgl_wait)
 {
         wait_queue_t *q;
+        struct task_struct *task = NULL;
 
         *dgl_wait = NULL;
-        *task = NULL;
 
         if (waitqueue_active(wq)) {
                 q = list_entry(wq->task_list.next,
@@ -292,13 +322,15 @@ void __waitqueue_dgl_remove_first(wait_queue_head_t *wq,
 
                 if(q->func == dgl_wake_up) {
                         *dgl_wait = (dgl_wait_state_t*) q->private;
+                        task = (*dgl_wait)->task;
                 }
                 else {
-                        *task = (struct task_struct*) q->private;
+                        task = (struct task_struct*) q->private;
                 }
 
                 __remove_wait_queue(wq, q);
         }
+        return task;
 }
 
 void init_dgl_waitqueue_entry(wait_queue_t *wq_node, dgl_wait_state_t* dgl_wait)
@@ -354,7 +386,7 @@ int __attempt_atomic_dgl_acquire(struct litmus_lock *cur_lock, dgl_wait_state_t
 
                 l->ops->dgl_quick_lock(l, cur_lock, dgl_wait->task, &dgl_wait->wq_nodes[i]);
 
-                BUG_ON(dgl_wait->task != *(l->nest.owner_ptr));
+                BUG_ON(!(l->ops->is_owner(l, dgl_wait->task)));
         }
 
         return 0; /* success */
@@ -456,15 +488,108 @@ all_acquired:
         return 0;
 }
 
+
+
+static long do_litmus_dgl_atomic_lock(dgl_wait_state_t *dgl_wait)
+{
+        int i;
+        unsigned long irqflags; //, dummyflags;
+        raw_spinlock_t *dgl_lock;
+        struct litmus_lock *l;
+
+#ifdef CONFIG_SCHED_DEBUG_TRACE
+        char dglstr[CONFIG_LITMUS_MAX_DGL_SIZE*5];
+        snprintf_dgl(dglstr, sizeof(dglstr), dgl_wait->locks, dgl_wait->size);
+        TRACE_CUR("Atomic locking DGL with size %d: %s\n", dgl_wait->size, dglstr);
+#endif
+
+        dgl_lock = litmus->get_dgl_spinlock(dgl_wait->task);
+
+        BUG_ON(dgl_wait->task != current);
+
+        raw_spin_lock_irqsave(dgl_lock, irqflags);
+
+
+        dgl_wait->nr_remaining = dgl_wait->size;
+
+        for(i = 0; i < dgl_wait->size; ++i) {
+                struct litmus_lock *l = dgl_wait->locks[i];
+                l->ops->dgl_lock(l, dgl_wait, &dgl_wait->wq_nodes[i]);  // this should be a forced enqueue if atomic DGLs are needed.
+        }
+
+        if(__attempt_atomic_dgl_acquire(NULL, dgl_wait)) {
+                /* Failed to acquire all locks at once.
+                 * Pick a lock to push on and suspend. */
+                TRACE_CUR("Could not atomically acquire all locks.\n");
+
+
+#if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA)
+                // KLUDGE: don't count this suspension as time in the critical gpu
+                // critical section
+                if(tsk_rt(dgl_wait->task)->held_gpus) {
+                        tsk_rt(dgl_wait->task)->suspend_gpu_tracker_on_block = 1;
+                }
+#endif
+                // we are not the owner of any lock, so push on the last one in the DGL
+                // by default.
+
+                l = dgl_wait->locks[dgl_wait->size - 1];
+
+                TRACE_CUR("Activating priority inheritance on lock %d\n",
+                                  l->ident);
+
+                TS_DGL_LOCK_SUSPEND;
+
+                l->ops->enable_priority(l, dgl_wait);
+                dgl_wait->last_primary = dgl_wait->size - 1;
+
+                TRACE_CUR("Suspending for lock %d\n", l->ident);
+
+                raw_spin_unlock_irqrestore(dgl_lock, irqflags);  // free dgl_lock before suspending
+
+                suspend_for_lock(); // suspend!!!
+
+                TS_DGL_LOCK_RESUME;
+
+                TRACE_CUR("Woken up from DGL suspension.\n");
+
+                goto all_acquired;  // we should hold all locks when we wake up.
+        }
+        raw_spin_unlock_irqrestore(dgl_lock, irqflags);
+
+all_acquired:
+
+        dgl_wait->nr_remaining = 0;
+
+        // SANITY CHECK FOR TESTING
+        for(i = 0; i < dgl_wait->size; ++i) {
+                struct litmus_lock *l = dgl_wait->locks[i];
+                BUG_ON(!l->ops->is_owner(l, dgl_wait->task));
+        }
+
+        TRACE_CUR("Acquired entire DGL\n");
+
+        return 0;
+}
+
+
+
+
 static int supports_dgl(struct litmus_lock *l)
 {
         struct litmus_lock_ops* ops = l->ops;
-
         return (ops->dgl_lock                   &&
                         ops->is_owner                   &&
+                        ops->get_owner                  &&
                         ops->enable_priority);
 }
 
+static int needs_atomic_dgl(struct litmus_lock *l)
+{
+        struct litmus_lock_ops* ops = l->ops;
+        return (ops->dgl_quick_lock != NULL);
+}
+
 asmlinkage long sys_litmus_dgl_lock(void* __user usr_dgl_ods, int dgl_size)
 {
         struct task_struct *t = current;
@@ -472,6 +597,8 @@ asmlinkage long sys_litmus_dgl_lock(void* __user usr_dgl_ods, int dgl_size)
         int dgl_ods[MAX_DGL_SIZE];
         int i;
 
+        int num_need_atomic = 0;
+
         dgl_wait_state_t dgl_wait_state;  // lives on the stack until all resources in DGL are held.
 
         if(dgl_size > MAX_DGL_SIZE || dgl_size < 1)
@@ -503,6 +630,10 @@ asmlinkage long sys_litmus_dgl_lock(void* __user usr_dgl_ods, int dgl_size)
                                                           dgl_wait_state.locks[i]->ident);
                                         goto out;
                                 }
+
+                                if (needs_atomic_dgl(dgl_wait_state.locks[i])) {
+                                        ++num_need_atomic;
+                                }
                         }
                         else {
                                 TRACE_CUR("Invalid lock identifier\n");
@@ -510,11 +641,19 @@ asmlinkage long sys_litmus_dgl_lock(void* __user usr_dgl_ods, int dgl_size)
                         }
                 }
 
+                if (num_need_atomic && num_need_atomic != dgl_size) {
+                        TRACE_CUR("All locks in DGL must support atomic acquire if any one does.\n");
+                        goto out;
+                }
+
                 dgl_wait_state.task = t;
                 dgl_wait_state.size = dgl_size;
 
                 TS_DGL_LOCK_START;
-                err = do_litmus_dgl_lock(&dgl_wait_state);
+                if (!num_need_atomic)
+                        err = do_litmus_dgl_lock(&dgl_wait_state);
+                else
+                        err = do_litmus_dgl_atomic_lock(&dgl_wait_state);
 
                 /* Note: task my have been suspended or preempted in between!  Take
                  * this into account when computing overheads. */