1 files changed, 31 insertions, 61 deletions

diff --git a/litmus/locking.c b/litmus/locking.c
index 4b8382cd77d1..eddc67a4d36a 100644
--- a/litmus/locking.c
+++ b/litmus/locking.c
@@ -365,18 +365,6 @@ static void snprintf_dgl(char* buf, size_t bsz, struct litmus_lock* dgl_locks[],
 #endif
 
 
-static int failed_owner(struct litmus_lock *cur_lock, struct task_struct *t)
-{
-        struct task_struct *cur_owner = cur_lock->ops->get_owner(cur_lock);
-        printk(KERN_EMERG "lock %d expected owner %s/%d but got %s/%d.\n",
-                                        cur_lock->ident,
-                                        (t) ? t->comm : "null",
-                                        (t) ? t->pid : 0,
-                                        (cur_owner) ? cur_owner->comm : "null",
-                                        (cur_owner) ? cur_owner->pid : 0);
-        BUG();
-}
-
 /* only valid when locks are prioq locks!!!
  * THE BIG DGL LOCK MUST BE HELD! */
 int __attempt_atomic_dgl_acquire(struct litmus_lock *cur_lock, dgl_wait_state_t *dgl_wait)
@@ -395,12 +383,8 @@ int __attempt_atomic_dgl_acquire(struct litmus_lock *cur_lock, dgl_wait_state_t
         /* take the locks */
         for(i = 0; i < dgl_wait->size; ++i) {
                 struct litmus_lock *l = dgl_wait->locks[i];
-
                 l->ops->dgl_quick_lock(l, cur_lock, dgl_wait->task, &dgl_wait->wq_nodes[i]);
-
-                if(!(l->ops->is_owner(l, dgl_wait->task)))
-                        failed_owner(l, dgl_wait->task);
-                //BUG_ON(!(l->ops->is_owner(l, dgl_wait->task)));
+                BUG_ON(!(l->ops->is_owner(l, dgl_wait->task)));
         }
 
         return 0; /* success */
@@ -510,6 +494,7 @@ static long do_litmus_dgl_atomic_lock(dgl_wait_state_t *dgl_wait)
         unsigned long irqflags; //, dummyflags;
         raw_spinlock_t *dgl_lock;
         struct litmus_lock *l;
+        struct task_struct *t = current;
 
 #ifdef CONFIG_SCHED_DEBUG_TRACE
         char dglstr[CONFIG_LITMUS_MAX_DGL_SIZE*5];
@@ -519,7 +504,7 @@ static long do_litmus_dgl_atomic_lock(dgl_wait_state_t *dgl_wait)
 
         dgl_lock = litmus->get_dgl_spinlock(dgl_wait->task);
 
-        BUG_ON(dgl_wait->task != current);
+        BUG_ON(dgl_wait->task != t);
 
         raw_spin_lock_irqsave(dgl_lock, irqflags);
 
@@ -528,7 +513,8 @@ static long do_litmus_dgl_atomic_lock(dgl_wait_state_t *dgl_wait)
 
         for(i = 0; i < dgl_wait->size; ++i) {
                 struct litmus_lock *l = dgl_wait->locks[i];
-                l->ops->dgl_lock(l, dgl_wait, &dgl_wait->wq_nodes[i]);  // this should be a forced enqueue if atomic DGLs are needed.
+                // this should be a forced enqueue if atomic DGLs are needed.
+                l->ops->dgl_lock(l, dgl_wait, &dgl_wait->wq_nodes[i]);
         }
 
         if(__attempt_atomic_dgl_acquire(NULL, dgl_wait)) {
@@ -536,27 +522,26 @@ static long do_litmus_dgl_atomic_lock(dgl_wait_state_t *dgl_wait)
                  * Pick a lock to push on and suspend. */
                 TRACE_CUR("Could not atomically acquire all locks.\n");
 
+                /* we set the uninterruptible state here since
+                 * __attempt_atomic_dgl_acquire() may actually succeed. */
+                set_task_state(t, TASK_UNINTERRUPTIBLE);
 
 #if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA)
                 // KLUDGE: don't count this suspension as time in the critical gpu
                 // critical section
-                if(tsk_rt(dgl_wait->task)->held_gpus) {
-                        tsk_rt(dgl_wait->task)->suspend_gpu_tracker_on_block = 1;
+                if(tsk_rt(t)->held_gpus) {
+                        tsk_rt(t)->suspend_gpu_tracker_on_block = 1;
                 }
 #endif
-                // we are not the owner of any lock, so push on the last one in the DGL
-                // by default.
 
-                l = dgl_wait->locks[dgl_wait->size - 1];
+                // select a lock to push priority on
+                dgl_wait->last_primary = 0;  // default
+                select_next_lock(dgl_wait);  // may change value of last_primary
 
-                TRACE_CUR("Activating priority inheritance on lock %d\n",
-                                  l->ident);
+                l = dgl_wait->locks[dgl_wait->last_primary];
 
                 TS_DGL_LOCK_SUSPEND;
 
-                l->ops->enable_priority(l, dgl_wait);
-                dgl_wait->last_primary = dgl_wait->size - 1;
-
                 TRACE_CUR("Suspending for lock %d\n", l->ident);
 
                 raw_spin_unlock_irqrestore(dgl_lock, irqflags);  // free dgl_lock before suspending
@@ -578,9 +563,7 @@ all_acquired:
         // SANITY CHECK FOR TESTING
         for(i = 0; i < dgl_wait->size; ++i) {
                 struct litmus_lock *l = dgl_wait->locks[i];
-                if(!(l->ops->is_owner(l, dgl_wait->task)))
-                        failed_owner(l, dgl_wait->task);
-                //BUG_ON(!l->ops->is_owner(l, dgl_wait->task));
+                BUG_ON(!l->ops->is_owner(l, dgl_wait->task));
         }
 
         TRACE_CUR("Acquired entire DGL\n");
@@ -589,23 +572,6 @@ all_acquired:
 }
 
 
-
-
-static int supports_dgl(struct litmus_lock *l)
-{
-        struct litmus_lock_ops* ops = l->ops;
-        return (ops->dgl_lock                   &&
-                        ops->is_owner                   &&
-                        ops->get_owner                  &&
-                        ops->enable_priority);
-}
-
-static int needs_atomic_dgl(struct litmus_lock *l)
-{
-        struct litmus_lock_ops* ops = l->ops;
-        return (ops->dgl_quick_lock != NULL);
-}
-
 asmlinkage long sys_litmus_dgl_lock(void* __user usr_dgl_ods, int dgl_size)
 {
         struct task_struct *t = current;
@@ -641,13 +607,13 @@ asmlinkage long sys_litmus_dgl_lock(void* __user usr_dgl_ods, int dgl_size)
                         struct od_table_entry *entry = get_entry_for_od(dgl_ods[i]);
                         if(entry && is_lock(entry)) {
                                 dgl_wait_state.locks[i] = get_lock(entry);
-                                if(!supports_dgl(dgl_wait_state.locks[i])) {
+                                if(!dgl_wait_state.locks[i]->ops->supports_dgl) {
                                         TRACE_CUR("Lock %d does not support all required DGL operations.\n",
                                                           dgl_wait_state.locks[i]->ident);
                                         goto out;
                                 }
 
-                                if (needs_atomic_dgl(dgl_wait_state.locks[i])) {
+                                if(dgl_wait_state.locks[i]->ops->requires_atomic_dgl) {
                                         ++num_need_atomic;
                                 }
                         }
@@ -686,9 +652,13 @@ static long do_litmus_dgl_unlock(struct litmus_lock* dgl_locks[], int dgl_size)
         long err = 0;
 
 #ifdef CONFIG_SCHED_DEBUG_TRACE
-        char dglstr[CONFIG_LITMUS_MAX_DGL_SIZE*5];
-        snprintf_dgl(dglstr, sizeof(dglstr), dgl_locks, dgl_size);
-        TRACE_CUR("Unlocking a DGL with size %d: %s\n", dgl_size, dglstr);
+        {
+                char dglstr[CONFIG_LITMUS_MAX_DGL_SIZE*5];
+                snprintf_dgl(dglstr, sizeof(dglstr), dgl_locks, dgl_size);
+                TRACE_CUR("Unlocking a DGL with size %d: %s\n",
+                                  dgl_size,
+                                  dglstr);
+        }
 #endif
 
         for(i = dgl_size - 1; i >= 0; --i) {  // unlock in reverse order
@@ -740,7 +710,7 @@ asmlinkage long sys_litmus_dgl_unlock(void* __user usr_dgl_ods, int dgl_size)
                         entry = get_entry_for_od(dgl_ods[i]);
                         if(entry && is_lock(entry)) {
                                 dgl_locks[i] = get_lock(entry);
-                                if(!supports_dgl(dgl_locks[i])) {
+                                if(!dgl_locks[i]->ops->supports_dgl) {
                                         TRACE_CUR("Lock %d does not support all required DGL operations.\n",
                                                           dgl_locks[i]->ident);
                                         goto out;
@@ -852,19 +822,19 @@ void suspend_for_lock(void)
                 tsk_rt(t)->suspend_gpu_tracker_on_block = 1;
         }
 #endif
-        
+
         schedule();
 
-        
+
         /* TODO: Move the following to wake_up_for_lock()? */
-        
+
 #if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA)
         // re-enable tracking
         if(tsk_rt(t)->held_gpus) {
                 tsk_rt(t)->suspend_gpu_tracker_on_block = 0;
         }
 #endif
-        
+
 #ifdef CONFIG_LITMUS_NVIDIA
         if (gpu_restore) {
                 /* restore our state */
@@ -883,9 +853,9 @@ void suspend_for_lock(void)
 int wake_up_for_lock(struct task_struct* t)
 {
         int ret;
-        
+
         ret = wake_up_process(t);
-        
+
         return ret;
 }