1 files changed, 118 insertions, 55 deletions

diff --git a/litmus/kfmlp_lock.c b/litmus/kfmlp_lock.c
index b30e5b589882..7cdca1b7b50a 100644
--- a/litmus/kfmlp_lock.c
+++ b/litmus/kfmlp_lock.c
@@ -99,12 +99,19 @@ static struct task_struct* kfmlp_select_hp_steal(struct kfmlp_semaphore* sem, wa
         if(*to_steal_from)
         {
                 struct list_head *pos;
+                struct task_struct *target = (*to_steal_from)->hp_waiter;
+                
+                TRACE_CUR("want to steal hp_waiter (%s/%d) from queue %d\n",
+                                  target->comm,
+                                  target->pid,
+                                  kfmlp_get_idx(sem, *to_steal_from));
+                
                 list_for_each(pos, &(*to_steal_from)->wait.task_list)
                 {
                         wait_queue_t *node = list_entry(pos, wait_queue_t, task_list);
                         struct task_struct *queued = (struct task_struct*) node->private;
                         /* Compare task prios, find high prio task. */
-                        if (queued == (*to_steal_from)->hp_waiter)
+                        if (queued == target)
                         {
                                 *to_steal = node;
                                 
@@ -115,6 +122,11 @@ static struct task_struct* kfmlp_select_hp_steal(struct kfmlp_semaphore* sem, wa
                                 return queued;
                         }
                 }
+                
+                TRACE_CUR("Could not find %s/%d in queue %d!!!  THIS IS A BUG!\n",
+                                  target->comm,
+                                  target->pid,
+                                  kfmlp_get_idx(sem, *to_steal_from));                            
         }
         
         return NULL;
@@ -133,6 +145,11 @@ static void kfmlp_steal_node(struct kfmlp_semaphore *sem,
         if(t == src->hp_waiter) {
                 src->hp_waiter = kfmlp_find_hp_waiter(src, NULL);
                 
+                TRACE_CUR("queue %d: %s/%d is new hp_waiter\n",
+                                  kfmlp_get_idx(sem, src),
+                                  (src->hp_waiter) ? src->hp_waiter->comm : "nil",
+                                  (src->hp_waiter) ? src->hp_waiter->pid : -1);
+
                 if(src->owner && tsk_rt(src->owner)->inh_task == t) {
                         litmus->decrease_prio(src->owner, src->hp_waiter);
                 }
@@ -140,6 +157,7 @@ static void kfmlp_steal_node(struct kfmlp_semaphore *sem,
         
         if(sem->shortest_queue->count > src->count) {
                 sem->shortest_queue = src;
+                TRACE_CUR("queue %d is the shortest\n", kfmlp_get_idx(sem, sem->shortest_queue));
         }
         
 #ifdef CONFIG_LITMUS_AFFINITY_LOCKING
@@ -155,6 +173,10 @@ static void kfmlp_steal_node(struct kfmlp_semaphore *sem,
         if(litmus->compare(t, dst->hp_waiter)) {
                 dst->hp_waiter = t;
                 
+                TRACE_CUR("queue %d: %s/%d is new hp_waiter\n",
+                                  kfmlp_get_idx(sem, dst),
+                                  t->comm, t->pid);             
+                
                 if(dst->owner && litmus->compare(t, dst->owner))
                 {
                         litmus->increase_prio(dst->owner, t);
@@ -264,8 +286,9 @@ int kfmlp_lock(struct litmus_lock* l)
         
         if (my_queue->owner) {
                 /* resource is not free => must suspend and wait */
-                TRACE_CUR("queue %d: Resource is not free => must suspend and wait.\n",
-                                  kfmlp_get_idx(sem, my_queue));
+                TRACE_CUR("queue %d: Resource is not free => must suspend and wait. (queue size = %d)\n",
+                                  kfmlp_get_idx(sem, my_queue),
+                                  my_queue->count);
                 
                 init_waitqueue_entry(&wait, t);
                 
@@ -274,29 +297,37 @@ int kfmlp_lock(struct litmus_lock* l)
                 
                 __add_wait_queue_tail_exclusive(&my_queue->wait, &wait);
                 
+                TRACE_CUR("queue %d: hp_waiter is currently %s/%d\n",
+                                  kfmlp_get_idx(sem, my_queue),
+                                  (my_queue->hp_waiter) ? my_queue->hp_waiter->comm : "nil",
+                                  (my_queue->hp_waiter) ? my_queue->hp_waiter->pid : -1);
+                
                 /* check if we need to activate priority inheritance */
                 //if (edf_higher_prio(t, my_queue->hp_waiter))
-                if (litmus->compare(t, my_queue->hp_waiter))
-                {
+                if (litmus->compare(t, my_queue->hp_waiter)) {
                         my_queue->hp_waiter = t;
+                        TRACE_CUR("queue %d: %s/%d is new hp_waiter\n",
+                                          kfmlp_get_idx(sem, my_queue),
+                                          t->comm, t->pid);
+                        
                         //if (edf_higher_prio(t, my_queue->owner))
-                        if (litmus->compare(t, my_queue->owner))
-                        {
+                        if (litmus->compare(t, my_queue->owner)) {
                                 litmus->increase_prio(my_queue->owner, my_queue->hp_waiter);
                         }
                 }
                 
                 ++(my_queue->count);
                 
-#ifdef CONFIG_LITMUS_AFFINITY_LOCKING           
                 if(my_queue == sem->shortest_queue) {
                         sem->shortest_queue = kfmlp_find_shortest(sem, my_queue);
+                        TRACE_CUR("queue %d is the shortest\n",
+                                          kfmlp_get_idx(sem, sem->shortest_queue));
                 }
+                
+#ifdef CONFIG_LITMUS_AFFINITY_LOCKING           
                 if(sem->aff_obs) {
                         sem->aff_obs->ops->notify_enqueue(sem->aff_obs, my_queue, t);
                 }
-#else
-                sem->shortest_queue = kfmlp_find_shortest(sem, my_queue);
 #endif
                 
                 /* release lock before sleeping */
@@ -309,13 +340,11 @@ int kfmlp_lock(struct litmus_lock* l)
                 schedule();
                 
                 
-                if(my_queue->owner == t)
-                {
+                if(my_queue->owner == t) {
                         TRACE_CUR("queue %d: acquired through waiting\n",
                                           kfmlp_get_idx(sem, my_queue));
                 }
-                else
-                {
+                else {
                         /* this case may happen if our wait entry was stolen
                          between queues. record where we went. */
                         my_queue = kfmlp_get_queue(sem, t);
@@ -325,8 +354,7 @@ int kfmlp_lock(struct litmus_lock* l)
                                           kfmlp_get_idx(sem, my_queue));
                 }
         }
-        else
-        {
+        else {
                 TRACE_CUR("queue %d: acquired immediately\n",
                                   kfmlp_get_idx(sem, my_queue));
                 
@@ -334,16 +362,17 @@ int kfmlp_lock(struct litmus_lock* l)
                 
                 ++(my_queue->count);
                 
-#ifdef CONFIG_LITMUS_AFFINITY_LOCKING
                 if(my_queue == sem->shortest_queue) {
                         sem->shortest_queue = kfmlp_find_shortest(sem, my_queue);
+                        TRACE_CUR("queue %d is the shortest\n",
+                                          kfmlp_get_idx(sem, sem->shortest_queue));
                 }
+                
+#ifdef CONFIG_LITMUS_AFFINITY_LOCKING
                 if(sem->aff_obs) {
                         sem->aff_obs->ops->notify_enqueue(sem->aff_obs, my_queue, t);
                         sem->aff_obs->ops->notify_acquired(sem->aff_obs, my_queue, t);
                 }
-#else
-                sem->shortest_queue = kfmlp_find_shortest(sem, my_queue);
 #endif
                 
                 spin_unlock_irqrestore(&sem->lock, flags);
@@ -370,12 +399,16 @@ int kfmlp_unlock(struct litmus_lock* l)
                 goto out;
         }
         
+        TRACE_CUR("queue %d: unlocking\n", kfmlp_get_idx(sem, my_queue));       
+        
         my_queue->owner = NULL;  // clear ownership
         --(my_queue->count);
         
         if(my_queue->count < sem->shortest_queue->count)
         {
                 sem->shortest_queue = my_queue;
+                TRACE_CUR("queue %d is the shortest\n",
+                                  kfmlp_get_idx(sem, sem->shortest_queue));
         }
         
 #ifdef CONFIG_LITMUS_AFFINITY_LOCKING
@@ -424,13 +457,13 @@ RETRY:
                 /* wake up next */
                 wake_up_process(next);
         }
-        else
-        {
+        else {
                 // TODO: put this stealing logic before we attempt to release
                 // our resource.  (simplifies code and gets rid of ugly goto RETRY.
                 wait_queue_t *wait;
                 
-                TRACE_CUR("queue %d: looking to steal someone...\n", kfmlp_get_idx(sem, my_queue));
+                TRACE_CUR("queue %d: looking to steal someone...\n",
+                                  kfmlp_get_idx(sem, my_queue));
 
 #ifdef CONFIG_LITMUS_AFFINITY_LOCKING
                 next = (sem->aff_obs) ?
@@ -441,16 +474,18 @@ RETRY:
 #endif
                 
                 if(next) {
-                        kfmlp_steal_node(sem, my_queue, wait, to_steal_from);
-                        
-                        TRACE_CUR("queued %d: stole %s/%d from queue %d\n",
+                        TRACE_CUR("queue %d: stealing %s/%d from queue %d\n",
+                                          kfmlp_get_idx(sem, my_queue),
                                           next->comm, next->pid,
-                                          kfmlp_get_idx(sem, to_steal_from));
+                                          kfmlp_get_idx(sem, to_steal_from));                   
+                        
+                        kfmlp_steal_node(sem, my_queue, wait, to_steal_from);
                         
                         goto RETRY;  // will succeed this time.
                 }
                 else {          
-                        TRACE_CUR("queue %d: no one to steal.\n", kfmlp_get_idx(sem, my_queue));
+                        TRACE_CUR("queue %d: no one to steal.\n",
+                                          kfmlp_get_idx(sem, my_queue));
                 }
         }
         
@@ -597,7 +632,8 @@ static struct affinity_observer* kfmlp_aff_obs_new(struct affinity_observer_ops*
                 return(NULL);
         }
         
-        kfmlp_aff->obs.ops = ops;
+        affinity_observer_new(&kfmlp_aff->obs, ops, &aff_args.obs);
+        
         kfmlp_aff->ops = kfmlp_ops;
         kfmlp_aff->offset = aff_args.replica_to_gpu_offset;
         
@@ -645,34 +681,50 @@ struct kfmlp_queue* gpu_kfmlp_advise_enqueue(struct kfmlp_affinity* aff, struct
         
         // simply pick the shortest queue if, we have no affinity, or we have
         // affinity with the shortest
-        if((tsk_rt(t)->last_gpu < 0) ||
-           ((kfmlp_get_idx(sem, aff->shortest_queue->q) + aff->offset) == tsk_rt(t)->last_gpu)) {
+        if(unlikely(tsk_rt(t)->last_gpu < 0)) {
                 // we have affinity with the shorest queue.  pick it.
-                to_enqueue = aff->shortest_queue->q;
-                
-                TRACE_CUR("special case: no affinity or have affinity with shortest\n");
-                
+                shortest = aff->shortest_queue;
+                TRACE_CUR("special case: no affinity\n");
                 goto out;
         }
         
-        // enqueue where we will have the shortest time to completion
+        // all things being equal, let's start with the queue with which we have
+        // affinity.  this helps us maintain affinity even when we don't have
+        // an estiamte for local-affinity execution time (i.e., 2nd time on GPU)
+        shortest = &aff->q_info[tsk_rt(t)->last_gpu - aff->offset];
+        
+        if(shortest == aff->shortest_queue) {
+                TRACE_CUR("special case: have affinity with shortest queue\n");
+                goto out;
+        }
         
-        shortest = &aff->q_info[0];
-        min_len = shortest->estimated_len + get_gpu_estimate(t, gpu_migration_distance(tsk_rt(t)->last_gpu, 0 + aff->offset));
+        min_len = shortest->estimated_len + get_gpu_estimate(t, MIG_LOCAL);
         
-        for(i = 1; i < sem->num_resources; ++i) {
-                lt_t est_len =
-                        aff->q_info[i].estimated_len +
-                        get_gpu_estimate(t, gpu_migration_distance(tsk_rt(t)->last_gpu, i + aff->offset));
-                                                         
-                if(est_len < min_len) {
-                        shortest = &aff->q_info[i];
-                        min_len = est_len;
+        TRACE_CUR("cs is %llu on queue %d: est len = %llu\n",
+                          get_gpu_estimate(t, MIG_LOCAL),
+                          kfmlp_get_idx(sem, shortest->q),
+                          min_len);
+        
+        for(i = 0; i < sem->num_resources; ++i) {
+                if(&aff->q_info[i] != shortest) {
+                        
+                        lt_t est_len =
+                                aff->q_info[i].estimated_len +
+                                get_gpu_estimate(t, gpu_migration_distance(tsk_rt(t)->last_gpu, i + aff->offset));
+                        if(est_len < min_len) {
+                                shortest = &aff->q_info[i];
+                                min_len = est_len;
+                        }
+                        
+                        TRACE_CUR("cs is %llu on queue %d: est len = %llu\n",
+                                          get_gpu_estimate(t, gpu_migration_distance(tsk_rt(t)->last_gpu, i + aff->offset)),
+                                          kfmlp_get_idx(sem, aff->q_info[i].q),
+                                          est_len);                             
                 }
         }
-        to_enqueue = shortest->q;
         
-out:
+out:    
+        to_enqueue = shortest->q;
         TRACE_CUR("enqueue on fq %d (non-aff wanted fq %d)\n",
                           kfmlp_get_idx(sem, to_enqueue),
                           kfmlp_get_idx(sem, sem->shortest_queue));     
@@ -698,17 +750,20 @@ void gpu_kfmlp_notify_enqueue(struct kfmlp_affinity* aff, struct kfmlp_queue* fq
         int gpu = aff->offset + replica;
         struct kfmlp_queue_info *info = &aff->q_info[replica];
         lt_t est_time;
+        lt_t est_len_before;
         
         if(current == t) {
                 tsk_rt(t)->suspend_gpu_tracker_on_block = 1;
         }
-
+        
+        est_len_before = info->estimated_len;
         est_time = get_gpu_estimate(t, gpu_migration_distance(tsk_rt(t)->last_gpu, gpu));
         info->estimated_len += est_time;
         
-        TRACE_CUR("fq %d est len is now %llu\n",
-                          kfmlp_get_idx(sem, aff->shortest_queue->q),
-                          aff->shortest_queue->estimated_len);
+        TRACE_CUR("fq %d: q_len (%llu) + est_cs (%llu) = %llu\n",
+                          kfmlp_get_idx(sem, info->q),
+                          est_len_before, est_time,
+                          info->estimated_len);
         
         if(aff->shortest_queue == info) {
                 // we may no longer be the shortest
@@ -764,7 +819,7 @@ void gpu_kfmlp_notify_acquired(struct kfmlp_affinity* aff, struct kfmlp_queue* f
         TRACE_CUR("%s/%d acquired gpu %d.  migration type = %d\n",
                           t->comm, t->pid, gpu, tsk_rt(t)->gpu_migration);
         
-        reg_nv_device(gpu, 1);  // register
+        reg_nv_device(gpu, 1, t);  // register
         
         tsk_rt(t)->suspend_gpu_tracker_on_block = 0;
         reset_gpu_tracker(t);
@@ -782,7 +837,7 @@ void gpu_kfmlp_notify_freed(struct kfmlp_affinity* aff, struct kfmlp_queue* fq,
         est_time = get_gpu_estimate(t, gpu_migration_distance(tsk_rt(t)->last_gpu, gpu));
         
         tsk_rt(t)->last_gpu = gpu;
-        reg_nv_device(gpu, 0);  // unregister
+        reg_nv_device(gpu, 0, t);       // unregister
         
         // update estimates
         update_gpu_estimate(t, get_gpu_time(t));
@@ -822,21 +877,25 @@ struct affinity_observer* kfmlp_gpu_aff_obs_new(struct affinity_observer_ops* op
 struct kfmlp_queue* simple_gpu_kfmlp_advise_enqueue(struct kfmlp_affinity* aff, struct task_struct* t)
 {
         struct kfmlp_semaphore *sem = kfmlp_from_lock(aff->obs.lock);
+//      TRACE_CUR("Simple GPU KFMLP advise_enqueue invoked\n");
         return sem->shortest_queue;
 }
 
 struct task_struct* simple_gpu_kfmlp_advise_steal(struct kfmlp_affinity* aff, wait_queue_t** to_steal, struct kfmlp_queue** to_steal_from)
 {
         struct kfmlp_semaphore *sem = kfmlp_from_lock(aff->obs.lock);
+//      TRACE_CUR("Simple GPU KFMLP advise_steal invoked\n");
         return kfmlp_select_hp_steal(sem, to_steal, to_steal_from);
 }
 
 void simple_gpu_kfmlp_notify_enqueue(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t)
 {
+//      TRACE_CUR("Simple GPU KFMLP notify_enqueue invoked\n");
 }
 
 void simple_gpu_kfmlp_notify_dequeue(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t)
 {
+//      TRACE_CUR("Simple GPU KFMLP notify_dequeue invoked\n");
 }
 
 void simple_gpu_kfmlp_notify_acquired(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t)
@@ -844,7 +903,9 @@ void simple_gpu_kfmlp_notify_acquired(struct kfmlp_affinity* aff, struct kfmlp_q
         struct kfmlp_semaphore *sem = kfmlp_from_lock(aff->obs.lock);
         int gpu = kfmlp_get_idx(sem, fq) + aff->offset;
         
-        reg_nv_device(gpu, 1);  // register
+//      TRACE_CUR("Simple GPU KFMLP notify_acquired invoked\n");
+        
+        reg_nv_device(gpu, 1, t);  // register
 }
 
 void simple_gpu_kfmlp_notify_freed(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t)
@@ -852,7 +913,9 @@ void simple_gpu_kfmlp_notify_freed(struct kfmlp_affinity* aff, struct kfmlp_queu
         struct kfmlp_semaphore *sem = kfmlp_from_lock(aff->obs.lock);
         int gpu = kfmlp_get_idx(sem, fq) + aff->offset;
         
-        reg_nv_device(gpu, 0);  // unregister
+//      TRACE_CUR("Simple GPU KFMLP notify_freed invoked\n");
+        
+        reg_nv_device(gpu, 0, t);       // unregister
 }
 
 struct kfmlp_affinity_ops simple_gpu_kfmlp_affinity =