hoist task wakeups out of DGL crit csx's.

a very ugly kludge.  wake ups are queued in
a per-cpu buffer.  lock, unlock, and budget
operations that affect priority then have to
flush the wake queue.

7 files changed, 110 insertions, 10 deletions

diff --git a/include/litmus/budget.h b/include/litmus/budget.h
index 0ae9c9f30023..44a667c94720 100644
--- a/include/litmus/budget.h
+++ b/include/litmus/budget.h
@@ -10,7 +10,7 @@ struct enforcement_timer
 {
         raw_spinlock_t lock;
         struct hrtimer timer;
-        int armed:1;
+        unsigned int armed:1;
         unsigned int job_when_armed;
 };
 
diff --git a/include/litmus/locking.h b/include/litmus/locking.h
index 39e32d3f48c7..b03e22658298 100644
--- a/include/litmus/locking.h
+++ b/include/litmus/locking.h
@@ -241,6 +241,8 @@ struct litmus_lock_ops {
 
 void suspend_for_lock(void);
 int wake_up_for_lock(struct task_struct* t);
+int flush_pending_wakes(void);
+void init_wake_queues(void);
 
 /* thread safe?? */
 #ifndef CONFIG_LITMUS_NESTED_LOCKING
diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c
index 11e896903828..b836a4357fc3 100644
--- a/kernel/hrtimer.c
+++ b/kernel/hrtimer.c
@@ -1317,8 +1317,9 @@ static void __run_hrtimer(struct hrtimer *timer, ktime_t *now)
          * hrtimer_start_range_ns() or in hrtimer_interrupt()
          */
         if (restart != HRTIMER_NORESTART) {
-                BUG_ON(timer->state != HRTIMER_STATE_CALLBACK);
-                enqueue_hrtimer(timer, base);
+                if (likely(timer->state == HRTIMER_STATE_CALLBACK))
+                        enqueue_hrtimer(timer, base);
+        //      BUG_ON(timer->state != HRTIMER_STATE_CALLBACK);
         }
 
         WARN_ON_ONCE(!(timer->state & HRTIMER_STATE_CALLBACK));
diff --git a/litmus/ikglp_lock.c b/litmus/ikglp_lock.c
index d9316766c71b..71d3c3392b16 100644
--- a/litmus/ikglp_lock.c
+++ b/litmus/ikglp_lock.c
@@ -2823,6 +2823,7 @@ static ikglp_donee_heap_node_t* pick_donee(struct ikglp_affinity* aff,
         else if(waitqueue_active(&fq->wait)) {
                 struct list_head        *pos;
 
+                TRACE_CUR("searching fq %d for donee\n", ikglp_get_idx(sem, fq));
 
 //              TRACE_CUR("fq %d: owner: %s/%d, deadline = %d: (donor) = %s/%d "
 //                                "(mth_highest != fq->owner) = %d "
@@ -2997,7 +2998,8 @@ ikglp_donee_heap_node_t* gpu_ikglp_advise_donee_selection(
                 donee_node = default_donee;
 
                 TRACE_CUR("Could not find a donee. We have to steal one.\n");
-                WARN_ON(default_donee->donor_info == NULL);
+                // TODO: vv Is the below a bug when raised?
+                //WARN_ON(default_donee->donor_info == NULL);
         }
 
 out:
diff --git a/litmus/locking.c b/litmus/locking.c
index 37aaab5fb7e8..f1673535f114 100644
--- a/litmus/locking.c
+++ b/litmus/locking.c
@@ -122,8 +122,11 @@ asmlinkage long sys_litmus_lock(int lock_od)
 
         entry = get_entry_for_od(lock_od);
         if (entry && is_lock(entry)) {
+                unsigned long flags;
                 l = get_lock(entry);
                 TRACE_CUR("Attempts to lock %d\n", l->ident);
+
+                local_irq_save(flags);
                 err = l->ops->lock(l);
                 if (!err) {
                         sched_trace_lock(current, l->ident, 1);
@@ -135,6 +138,8 @@ asmlinkage long sys_litmus_lock(int lock_od)
                                 tsk_rt(current)->outermost_lock = l;
                         }
                 }
+                flush_pending_wakes();
+                local_irq_restore(flags);
         }
 
         /* Note: task my have been suspended or preempted in between!  Take
@@ -160,6 +165,7 @@ asmlinkage long sys_litmus_unlock(int lock_od)
 
         entry = get_entry_for_od(lock_od);
         if (entry && is_lock(entry)) {
+                unsigned long flags;
                 l = get_lock(entry);
 
                 if (l == tsk_rt(current)->outermost_lock) {
@@ -168,11 +174,14 @@ asmlinkage long sys_litmus_unlock(int lock_od)
                 }
 
                 TRACE_CUR("Attempts to unlock %d\n", l->ident);
+                local_irq_save(flags);
                 err = l->ops->unlock(l);
                 if (!err) {
                         sched_trace_lock(current, l->ident, 0);
                         TRACE_CUR("Unlocked %d\n", l->ident);
                 }
+                flush_pending_wakes();
+                local_irq_restore(flags);
         }
 
         /* Note: task my have been preempted in between!  Take this into
@@ -429,6 +438,7 @@ static long do_litmus_dgl_lock(dgl_wait_state_t *dgl_wait)
 {
         int i;
         unsigned long irqflags; //, dummyflags;
+        unsigned long kludge_flags;
         raw_spinlock_t *dgl_lock;
 
 #ifdef CONFIG_SCHED_DEBUG_TRACE
@@ -444,6 +454,8 @@ static long do_litmus_dgl_lock(dgl_wait_state_t *dgl_wait)
         dgl_wait->nr_remaining = dgl_wait->size;
 
         dgl_lock = litmus->get_dgl_spinlock(dgl_wait->task);
+        
+        local_irq_save(kludge_flags);
         raw_spin_lock_irqsave(dgl_lock, irqflags);
 
         // try to acquire each lock.  enqueue (non-blocking) if it is unavailable.
@@ -463,6 +475,7 @@ static long do_litmus_dgl_lock(dgl_wait_state_t *dgl_wait)
                 // acquired entire group immediatly
                 TRACE_CUR("Acquired all locks in DGL immediatly!\n");
                 raw_spin_unlock_irqrestore(dgl_lock, irqflags);
+                local_irq_restore(kludge_flags);
         }
         else {
                 struct litmus_lock *first_primary;
@@ -484,6 +497,8 @@ static long do_litmus_dgl_lock(dgl_wait_state_t *dgl_wait)
                 TS_DGL_LOCK_SUSPEND;
 
                 raw_spin_unlock_irqrestore(dgl_lock, irqflags);  // free dgl_lock before suspending
+                flush_pending_wakes();
+                local_irq_restore(kludge_flags);
                 suspend_for_lock();
 
                 TS_DGL_LOCK_RESUME;
@@ -508,6 +523,7 @@ static long do_litmus_dgl_atomic_lock(dgl_wait_state_t *dgl_wait)
 {
         int i;
         unsigned long irqflags; //, dummyflags;
+        unsigned long kludge_flags;
         raw_spinlock_t *dgl_lock;
         struct task_struct *t = current;
 
@@ -523,6 +539,7 @@ static long do_litmus_dgl_atomic_lock(dgl_wait_state_t *dgl_wait)
 
         BUG_ON(dgl_wait->task != t);
 
+        local_irq_save(kludge_flags);
         raw_spin_lock_irqsave(dgl_lock, irqflags);
 
 
@@ -556,6 +573,8 @@ static long do_litmus_dgl_atomic_lock(dgl_wait_state_t *dgl_wait)
                 TS_DGL_LOCK_SUSPEND;
 
                 raw_spin_unlock_irqrestore(dgl_lock, irqflags);  // free dgl_lock before suspending
+                flush_pending_wakes();
+                local_irq_restore(kludge_flags);
                 suspend_for_lock(); // suspend!!!
 
                 TS_DGL_LOCK_RESUME;
@@ -565,6 +584,8 @@ static long do_litmus_dgl_atomic_lock(dgl_wait_state_t *dgl_wait)
                 goto all_acquired;  // we should hold all locks when we wake up.
         }
         raw_spin_unlock_irqrestore(dgl_lock, irqflags);
+        flush_pending_wakes();
+        local_irq_restore(kludge_flags);
 
 all_acquired:
 
@@ -661,6 +682,7 @@ static long do_litmus_dgl_unlock(struct litmus_lock* dgl_locks[], int dgl_size)
 {
         int i;
         long err = 0;
+        unsigned long flags;
 
 #ifdef CONFIG_SCHED_DEBUG_TRACE
         {
@@ -670,6 +692,7 @@ static long do_litmus_dgl_unlock(struct litmus_lock* dgl_locks[], int dgl_size)
         }
 #endif
 
+        local_irq_save(flags);
         for(i = dgl_size - 1; i >= 0; --i) {  // unlock in reverse order
 
                 struct litmus_lock *l = dgl_locks[i];
@@ -685,6 +708,8 @@ static long do_litmus_dgl_unlock(struct litmus_lock* dgl_locks[], int dgl_size)
                         err = tmp_err;
                 }
         }
+        flush_pending_wakes();
+        local_irq_restore(flags);
 
         TRACE_CUR("DGL unlocked. err = %d\n", err);
 
@@ -878,15 +903,65 @@ void suspend_for_lock(void)
 #endif
 }
 
+#define WAKE_Q_SZ       32
+
+typedef struct wake_queue
+{
+        struct task_struct *to_wake[WAKE_Q_SZ];
+        int count;
+} wake_queue_t;
+
+DEFINE_PER_CPU(wake_queue_t, wqueues);
+
+void init_wake_queues()
+{
+        int cpu = 0;
+        for_each_online_cpu(cpu)
+        {
+                wake_queue_t *q = &per_cpu(wqueues, cpu);
+                memset(q, 0, sizeof(*q));
+        }
+}
+
 int wake_up_for_lock(struct task_struct* t)
 {
-        int ret;
+        int ret = 1; /* mimic success of wake_up_process() */
+        wake_queue_t *q;
 
-        ret = wake_up_process(t);
+        //ret = wake_up_process(t);
+        
+        TRACE_TASK(t, "is queued for wakeup\n");
+        q = &per_cpu(wqueues, smp_processor_id());
+        q->to_wake[q->count] = t;
+        ++(q->count);
+
+        BUG_ON(q->count >= WAKE_Q_SZ);
 
         return ret;
 }
 
+int flush_pending_wakes()
+{
+        int count = 0, i;
+        
+        wake_queue_t *q = &per_cpu(wqueues, smp_processor_id());
+        for(i = 0; i < q->count; ++i)
+        {
+                if (q->to_wake[i]) {
+                        struct task_struct *t = q->to_wake[i];
+                        q->to_wake[i] = NULL;
+                        
+                        TRACE_TASK(t, "is being woken up\n");
+                        wake_up_process(t);
+                        ++count;
+                }
+        }
+        WARN_ON(count != q->count);
+        q->count = 0;
+        
+        return count;
+}
+
 
 #else  // CONFIG_LITMUS_LOCKING
 
diff --git a/litmus/sched_cedf.c b/litmus/sched_cedf.c
index e9422077f9dd..00dabbf65af7 100644
--- a/litmus/sched_cedf.c
+++ b/litmus/sched_cedf.c
@@ -779,7 +779,7 @@ static enum hrtimer_restart cedf_simple_io_on_exhausted(struct task_struct *t, i
                 else {
                         lt_t remaining;
                         cedf_domain_t *cluster;
-                        unsigned long flags;
+                        unsigned long flags, kludge_flags;
 
                         BUG_ON(in_schedule);
 
@@ -792,6 +792,8 @@ static enum hrtimer_restart cedf_simple_io_on_exhausted(struct task_struct *t, i
                         /* force job completion */
                         TRACE_TASK(t, "blocked, postponing deadline\n");
 
+                        local_irq_save(kludge_flags);
+                        
                         /* Outermost lock of the cluster. Recursive lock calls are
                          * possible on this code path. This should be the _ONLY_
                          * scenario where recursive calls are made. */
@@ -866,6 +868,8 @@ static enum hrtimer_restart cedf_simple_io_on_exhausted(struct task_struct *t, i
 #else
                         raw_readyq_unlock_irqrestore(&cluster->cluster_lock, flags);
 #endif
+                        flush_pending_wakes();
+                        local_irq_restore(kludge_flags);
 
                         /* we need to set up the budget timer since we're within the callback. */
                         hrtimer_forward_now(&get_budget_timer(t).timer.timer,
@@ -971,7 +975,7 @@ static enum hrtimer_restart cedf_sobliv_on_exhausted(struct task_struct *t, int
                 else {
                         lt_t remaining;
                         cedf_domain_t *cluster;
-                        unsigned long flags;
+                        unsigned long flags, kludge_flags;
 
                         BUG_ON(in_schedule);
 
@@ -984,6 +988,8 @@ static enum hrtimer_restart cedf_sobliv_on_exhausted(struct task_struct *t, int
                         /* force job completion */
                         TRACE_TASK(t, "blocked, postponing deadline\n");
 
+                        local_irq_save(kludge_flags);
+                        
                         /* Outermost lock of the cluster. Recursive lock calls are
                          * possible on this code path. This should be the _ONLY_
                          * scenario where recursive calls are made. */
@@ -1061,6 +1067,8 @@ static enum hrtimer_restart cedf_sobliv_on_exhausted(struct task_struct *t, int
 #else
                         raw_readyq_unlock_irqrestore(&cluster->cluster_lock, flags);
 #endif
+                        flush_pending_wakes();
+                        local_irq_restore(kludge_flags);
 
                         /* we need to set up the budget timer since we're within the callback. */
                         if (bt_flag_is_set(t, BTF_IS_TOP_M)) {
@@ -2816,6 +2824,8 @@ static long cedf_activate_plugin(void)
 #ifdef CONFIG_LITMUS_NVIDIA
         init_nvidia_info();
 #endif
+        
+        init_wake_queues();
 
         free_cpumask_var(mask);
         clusters_allocated = 1;
diff --git a/litmus/sched_crm.c b/litmus/sched_crm.c
index b3f98f17ac0b..a60f819138d8 100644
--- a/litmus/sched_crm.c
+++ b/litmus/sched_crm.c
@@ -780,7 +780,7 @@ static enum hrtimer_restart crm_simple_io_on_exhausted(struct task_struct *t, in
                 else {
                         lt_t remaining;
                         crm_domain_t *cluster;
-                        unsigned long flags;
+                        unsigned long flags, kludge_flags;
 
                         BUG_ON(in_schedule);
 
@@ -793,6 +793,8 @@ static enum hrtimer_restart crm_simple_io_on_exhausted(struct task_struct *t, in
                         /* force job completion */
                         TRACE_TASK(t, "blocked, postponing deadline\n");
 
+                        local_irq_save(kludge_flags);
+                        
                         /* Outermost lock of the cluster. Recursive lock calls are
                          * possible on this code path. This should be the _ONLY_
                          * scenario where recursive calls are made. */
@@ -867,6 +869,8 @@ static enum hrtimer_restart crm_simple_io_on_exhausted(struct task_struct *t, in
 #else
                         raw_readyq_unlock_irqrestore(&cluster->cluster_lock, flags);
 #endif
+                        flush_pending_wakes();
+                        local_irq_restore(kludge_flags);
 
                         /* we need to set up the budget timer since we're within the callback. */
                         hrtimer_forward_now(&get_budget_timer(t).timer.timer,
@@ -972,7 +976,7 @@ static enum hrtimer_restart crm_sobliv_on_exhausted(struct task_struct *t, int i
                 else {
                         lt_t remaining;
                         crm_domain_t *cluster;
-                        unsigned long flags;
+                        unsigned long flags, kludge_flags;
 
                         BUG_ON(in_schedule);
 
@@ -988,6 +992,7 @@ static enum hrtimer_restart crm_sobliv_on_exhausted(struct task_struct *t, int i
                         /* Outermost lock of the cluster. Recursive lock calls are
                          * possible on this code path. This should be the _ONLY_
                          * scenario where recursive calls are made. */
+                        local_irq_save(kludge_flags);
 #ifdef CONFIG_LITMUS_DGL_SUPPORT
                         /* Unfortunately, we _might_ need to grab the DGL lock, so we
                          * must grab it every time since it must be take before the
@@ -1062,6 +1067,8 @@ static enum hrtimer_restart crm_sobliv_on_exhausted(struct task_struct *t, int i
 #else
                         raw_readyq_unlock_irqrestore(&cluster->cluster_lock, flags);
 #endif
+                        flush_pending_wakes();
+                        local_irq_restore(kludge_flags);
 
                         /* we need to set up the budget timer since we're within the callback. */
                         if (bt_flag_is_set(t, BTF_IS_TOP_M)) {
@@ -2677,6 +2684,7 @@ static long crm_activate_plugin(void)
 
         /* de-allocate old clusters, if any */
         cleanup_crm();
+        
 
         printk(KERN_INFO "C-RM: Activate Plugin, cluster configuration = %d\n",
                         cluster_config);
@@ -2818,6 +2826,8 @@ static long crm_activate_plugin(void)
         init_nvidia_info();
 #endif
 
+        init_wake_queues();
+        
         free_cpumask_var(mask);
         clusters_allocated = 1;
         return 0;