Fixed case where blocked tasks are released.

Fixed bug where AUX tasks were being added to the
ready queue while those AUX tasks were actually blocked.
Bug stems from the fact that the AUX tasks do not
make themselves realtime, but another thread does this
instead. Also fixed minor bugs elsewhere.

NOTE: ONLY FIXES C-EDF.  OTHER PLUGINS REMAIN TO BE FIXED.

8 files changed, 124 insertions, 106 deletions

diff --git a/include/litmus/litmus.h b/include/litmus/litmus.h
index 2da61fa58bdc..17d30326034c 100644
--- a/include/litmus/litmus.h
+++ b/include/litmus/litmus.h
@@ -85,6 +85,19 @@ static inline lt_t litmus_clock(void)
         return ktime_to_ns(ktime_get());
 }
 
+static inline int is_persistent(struct task_struct* t)
+{
+        int is_per = ( 0
+                #ifdef CONFIG_REALTIME_AUX_TASKS
+                        || t->rt_param.is_aux_task
+                #endif
+                #ifdef CONFIG_LITMUS_SOFTIRQD
+                        || t->rt_param.is_interrupt_thread
+                #endif
+                        );
+        return is_per;
+}
+
 /* A macro to convert from nanoseconds to ktime_t. */
 #define ns_to_ktime(t)          ktime_add_ns(ktime_set(0, 0), t)
 
diff --git a/litmus/aux_tasks.c b/litmus/aux_tasks.c
index aa851ab2655b..46217e78b1ec 100644
--- a/litmus/aux_tasks.c
+++ b/litmus/aux_tasks.c
@@ -67,11 +67,11 @@ int exit_aux_task(struct task_struct *t)
         return retval;
 }
 
+#ifdef CONFIG_REALTIME_AUX_TASK_PRIORITY_INHERITANCE
 static int aux_tasks_increase_priority(struct task_struct *leader, struct task_struct *hp)
 {
         int retval = 0;
 
-#ifdef CONFIG_REALTIME_AUX_TASK_PRIORITY_INHERITANCE
         struct list_head *pos;
 
         TRACE_CUR("Increasing priority of aux tasks in group %s/%d.\n", leader->comm, leader->pid);
@@ -94,7 +94,6 @@ static int aux_tasks_increase_priority(struct task_struct *leader, struct task_s
                         retval = litmus->__increase_prio(aux, hp);
                 }
         }
-#endif
 
         return retval;
 }
@@ -103,7 +102,6 @@ static int aux_tasks_decrease_priority(struct task_struct *leader, struct task_s
 {
         int retval = 0;
 
-#ifdef CONFIG_REALTIME_AUX_TASK_PRIORITY_INHERITANCE
         struct list_head *pos;
 
         TRACE_CUR("Decreasing priority of aux tasks in group %s/%d.\n", leader->comm, leader->pid);
@@ -121,11 +119,12 @@ static int aux_tasks_decrease_priority(struct task_struct *leader, struct task_s
                         retval = litmus->__decrease_prio(aux, hp);
                 }
         }
-#endif
 
         return retval;
 }
 
+#endif
+
 int aux_task_owner_increase_priority(struct task_struct *t)
 {
         int retval = 0;
@@ -167,9 +166,9 @@ int aux_task_owner_increase_priority(struct task_struct *t)
                 TRACE_CUR("%s/%d is new hp in group %s/%d.\n", t->comm, t->pid, leader->comm, leader->pid);
                 retval = aux_tasks_increase_priority(leader, hp_eff);
         }
-#endif
-
 out:
+
+#endif
         return retval;
 }
 
@@ -215,9 +214,9 @@ int aux_task_owner_decrease_priority(struct task_struct *t)
                         retval = aux_tasks_decrease_priority(leader, hp_eff);
                 }
         }
-#endif
-
 out:
+
+#endif
         return retval;
 }
 
@@ -305,9 +304,9 @@ long enable_aux_task_owner(struct task_struct *t)
                 retval = aux_tasks_increase_priority(leader,
                                            (tsk_rt(hp)->inh_task)? tsk_rt(hp)->inh_task : hp);
         }
-#endif
-
 out:
+
+#endif
         return retval;
 }
 
@@ -356,9 +355,9 @@ long disable_aux_task_owner(struct task_struct *t)
 
                 retval = aux_tasks_decrease_priority(leader, to_inh);
         }
-#endif
-
 out:
+
+#endif
         return retval;
 }
 
diff --git a/litmus/edf_common.c b/litmus/edf_common.c
index 441fbfddf0c2..ef22eb93dbf3 100644
--- a/litmus/edf_common.c
+++ b/litmus/edf_common.c
@@ -67,11 +67,14 @@ int edf_higher_prio(struct task_struct* first, struct task_struct* second)
                 return 0;
         }
 
-
         /* check for NULL tasks */
         if (!first || !second) {
                 return first && !second;
         }
+        /* check for non-realtime */
+        if (!is_realtime(first) || !is_realtime(second)) {
+                return is_realtime(first) && !is_realtime(second);
+        }
 
         /* There is some goofy stuff in this code here.  There are three subclasses
          * within the SCHED_LITMUS scheduling class:
@@ -92,49 +95,31 @@ int edf_higher_prio(struct task_struct* first, struct task_struct* second)
 
 #if defined(CONFIG_REALTIME_AUX_TASK_PRIORITY_BOOSTED)
         /* run aux tasks at max priority */
-        /* TODO: Actually use prio-boosting. */
-        if (first->rt_param.is_aux_task != second->rt_param.is_aux_task)
-        {
-                return (first->rt_param.is_aux_task > second->rt_param.is_aux_task);
-        }
-        else if(first->rt_param.is_aux_task && second->rt_param.is_aux_task)
-        {
-                if(first->group_leader == second->group_leader) {
-                        TRACE_CUR("aux tie break!\n");  // tie-break by BASE priority of the aux tasks
-                        goto aux_tie_break;
-                }
-                first = first->group_leader;
-                second = second->group_leader;
+        if (tsk_rt(first)->is_aux_task != tsk_rt(second)->is_aux_task) {
+                return (tsk_rt(first)->is_aux_task > tsk_rt(second)->is_aux_task);
         }
 #elif defined(CONFIG_REALTIME_AUX_TASK_PRIORITY_INHERITANCE)
         {
-                int first_lo_aux = first->rt_param.is_aux_task && !first->rt_param.inh_task;
-                int second_lo_aux = second->rt_param.is_aux_task && !second->rt_param.inh_task;
+                int first_lo_aux = tsk_rt(first)->is_aux_task && !tsk_rt(first)->inh_task;
+                int second_lo_aux = tsk_rt(second)->is_aux_task && !tsk_rt(second)->inh_task;
 
                 /* prioritize aux tasks without inheritance below real-time tasks */
                 if (first_lo_aux || second_lo_aux) {
                         // one of these is an aux task without inheritance.
-                        if(first_lo_aux && second_lo_aux) {
-                                TRACE_CUR("aux tie break!\n");  // tie-break by BASE priority of the aux tasks
-                                goto aux_tie_break;
-                        }
-                        else {
-
-                                // make the aux thread lowest priority real-time task
-                                int temp = 0;
-                                if (first_lo_aux && is_realtime(second)) {
-//                                      temp = 0;
-                                }
-                                else if(second_lo_aux && is_realtime(first)) {
-                                        temp = 1;
-                                }
+                        if (first_lo_aux != second_lo_aux) {
+                                int temp = (first_lo_aux < second_lo_aux);  // non-lo-aux has higher priority.
                                 TRACE_CUR("%s/%d >> %s/%d --- %d\n", first->comm, first->pid, second->comm, second->pid, temp);
                                 return temp;
                         }
+                        else {
+                                /* both MUST be lo_aux. tie-break. */
+                                TRACE_CUR("aux tie break!\n");
+                                goto aux_tie_break;
+                        }
                 }
 
-                if (first->rt_param.is_aux_task && second->rt_param.is_aux_task &&
-                        first->rt_param.inh_task == second->rt_param.inh_task) {
+                if (tsk_rt(first)->is_aux_task && tsk_rt(second)->is_aux_task &&
+                        tsk_rt(first)->inh_task == tsk_rt(second)->inh_task) {
                         // inh_task is !NULL for both tasks since neither was a lo_aux task.
                         // Both aux tasks inherit from the same task, so tie-break
                         // by base priority of the aux tasks.
@@ -146,33 +131,26 @@ int edf_higher_prio(struct task_struct* first, struct task_struct* second)
 
 #ifdef CONFIG_LITMUS_SOFTIRQD
         {
-                int first_lo_klmirqd = first->rt_param.is_interrupt_thread && !first->rt_param.inh_task;
-                int second_lo_klmirqd = second->rt_param.is_interrupt_thread && !second->rt_param.inh_task;
+                int first_lo_klmirqd = tsk_rt(first)->is_interrupt_thread && !tsk_rt(first)->inh_task;
+                int second_lo_klmirqd = tsk_rt(second)->is_interrupt_thread && !tsk_rt(second)->inh_task;
 
                 /* prioritize aux tasks without inheritance below real-time tasks */
                 if (first_lo_klmirqd || second_lo_klmirqd) {
                         // one of these is an klmirqd thread without inheritance.
-                        if(first_lo_klmirqd && second_lo_klmirqd) {
-                                TRACE_CUR("klmirqd tie break!\n");  // tie-break by BASE priority of the aux tasks
-                                goto klmirqd_tie_break;
-                        }
-                        else {
-                                // make the klmirqd thread the lowest-priority real-time task
-                                // but (above low-prio aux tasks and Linux tasks)
-                                int temp = 0;
-                                if (first_lo_klmirqd && is_realtime(second)) {
-//                                      temp = 0;
-                                }
-                                else if(second_lo_klmirqd && is_realtime(first)) {
-                                        temp = 1;
-                                }
+                        if (first_lo_klmirqd != second_lo_klmirqd) {
+                                int temp = (first_lo_klmirqd < second_lo_klmirqd);  // non-klmirqd has higher priority
                                 TRACE_CUR("%s/%d >> %s/%d --- %d\n", first->comm, first->pid, second->comm, second->pid, temp);
                                 return temp;
                         }
+                        else {
+                                /* both MUST be klmirqd. tie-break. */
+                                TRACE_CUR("klmirqd tie break!\n");
+                                goto klmirqd_tie_break;
+                        }
                 }
 
-                if (first->rt_param.is_interrupt_thread && second->rt_param.is_interrupt_thread &&
-                        first->rt_param.inh_task == second->rt_param.inh_task) {
+                if (tsk_rt(first)->is_interrupt_thread && tsk_rt(second)->is_interrupt_thread &&
+                        tsk_rt(first)->inh_task == tsk_rt(second)->inh_task) {
                         // inh_task is !NULL for both tasks since neither was a lo_klmirqd task.
                         // Both klmirqd tasks inherit from the same task, so tie-break
                         // by base priority of the klmirqd tasks.
@@ -187,19 +165,19 @@ int edf_higher_prio(struct task_struct* first, struct task_struct* second)
         /* Check for EFFECTIVE priorities. Change task
          * used for comparison in such a case.
          */
-        if (unlikely(first->rt_param.inh_task)
+        if (unlikely(tsk_rt(first)->inh_task)
 #ifdef CONFIG_LITMUS_NESTED_LOCKING
                 && (first_mode == EFFECTIVE)
 #endif
                 ) {
-                first_task = first->rt_param.inh_task;
+                first_task = tsk_rt(first)->inh_task;
         }
-        if (unlikely(second->rt_param.inh_task)
+        if (unlikely(tsk_rt(second)->inh_task)
 #ifdef CONFIG_LITMUS_NESTED_LOCKING
                 && (second_mode == EFFECTIVE)
 #endif
                 ) {
-                second_task = second->rt_param.inh_task;
+                second_task = tsk_rt(second)->inh_task;
         }
 
         /* Check for priority boosting. Tie-break by start of boosting.
@@ -222,17 +200,14 @@ int edf_higher_prio(struct task_struct* first, struct task_struct* second)
 
 #endif
 
-#ifdef CONFIG_REALTIME_AUX_TASKS
+#ifdef CONFIG_REALTIME_AUX_TASK_PRIORITY_INHERITANCE
 aux_tie_break:
 #endif
 #ifdef CONFIG_LITMUS_SOFTIRQD
 klmirqd_tie_break:
 #endif
 
-        if (!is_realtime(second_task)) {
-                return 1;
-        }
-        else if (earlier_deadline(first_task, second_task)) {
+        if (earlier_deadline(first_task, second_task)) {
                 return 1;
         }
         else if (get_deadline(first_task) == get_deadline(second_task)) {
@@ -297,10 +272,10 @@ klmirqd_tie_break:
                         }
                         else if (first_task->pid == second_task->pid) {
 #ifdef CONFIG_LITMUS_SOFTIRQD
-                                if (first_task->rt_param.is_interrupt_thread < second_task->rt_param.is_interrupt_thread) {
+                                if (tsk_rt(first_task)->is_interrupt_thread < tsk_rt(second_task)->is_interrupt_thread) {
                                         return 1;
                                 }
-                                else if (first_task->rt_param.is_interrupt_thread == second_task->rt_param.is_interrupt_thread) {
+                                else if (tsk_rt(first_task)->is_interrupt_thread == tsk_rt(second_task)->is_interrupt_thread) {
 #endif
 
 #if defined(CONFIG_REALTIME_AUX_TASK_PRIORITY_INHERITANCE)
@@ -311,7 +286,7 @@ klmirqd_tie_break:
 #endif
 
                                 /* Something could be wrong if you get this far. */
-                                if (unlikely(first->rt_param.inh_task == second->rt_param.inh_task)) {
+                                if (unlikely(tsk_rt(first)->inh_task == tsk_rt(second)->inh_task)) {
                                         /* Both tasks have the same inherited priority.
                                          * Likely in a bug-condition.
                                      */
@@ -324,22 +299,22 @@ klmirqd_tie_break:
                                 }
                                 else {
                                         /* At least one task must inherit */
-                                        BUG_ON(!first->rt_param.inh_task &&
-                                                   !second->rt_param.inh_task);
+                                        BUG_ON(!tsk_rt(first)->inh_task &&
+                                                   !tsk_rt(second)->inh_task);
 
                                         /* The task withOUT the inherited priority wins. */
-                                        if (second->rt_param.inh_task) {
+                                        if (tsk_rt(second)->inh_task) {
                                                 /*
                                                  * common with aux tasks.
                                                 TRACE_CUR("unusual comparison: "
                                                         "first = %s/%d  first_task = %s/%d  "
                                                         "second = %s/%d  second_task = %s/%d\n",
                                                         first->comm, first->pid,
-                                                        (first->rt_param.inh_task) ? first->rt_param.inh_task->comm : "(nil)",
-                                                        (first->rt_param.inh_task) ? first->rt_param.inh_task->pid : 0,
+                                                        (tsk_rt(first)->inh_task) ? tsk_rt(first)->inh_task->comm : "(nil)",
+                                                        (tsk_rt(first)->inh_task) ? tsk_rt(first)->inh_task->pid : 0,
                                                         second->comm, second->pid,
-                                                        (second->rt_param.inh_task) ? second->rt_param.inh_task->comm : "(nil)",
-                                                        (second->rt_param.inh_task) ? second->rt_param.inh_task->pid : 0);
+                                                        (tsk_rt(second)->inh_task) ? tsk_rt(second)->inh_task->comm : "(nil)",
+                                                        (tsk_rt(second)->inh_task) ? tsk_rt(second)->inh_task->pid : 0);
                                                  */
                                                 return 1;
                                         }
@@ -426,3 +401,4 @@ int edf_preemption_needed(rt_domain_t* rt, struct task_struct *t)
         /* make sure to get non-rt stuff out of the way */
         return !is_realtime(t) || edf_higher_prio(__next_ready(rt), t);
 }
+
diff --git a/litmus/ikglp_lock.c b/litmus/ikglp_lock.c
index aa6b659e437d..170c43fcc78e 100644
--- a/litmus/ikglp_lock.c
+++ b/litmus/ikglp_lock.c
@@ -870,7 +870,8 @@ static void ikglp_enqueue_on_donor(struct ikglp_semaphore *sem,
         }
         else {
                 TRACE_TASK(t, "No change in effective priority (it is %d/%s).  BUG?\n",
-                                   new_max_eff_prio->comm, new_max_eff_prio->pid);
+                                   (new_max_eff_prio) ? new_max_eff_prio->comm : "nil",
+                                   (new_max_eff_prio) ? new_max_eff_prio->pid : -1);
                 raw_spin_unlock(&tsk_rt(donee)->hp_blocked_tasks_lock);
                 unlock_fine_irqrestore(&sem->lock, flags);
         }
diff --git a/litmus/jobs.c b/litmus/jobs.c
index 596b48e699b1..bdfc41004d38 100644
--- a/litmus/jobs.c
+++ b/litmus/jobs.c
@@ -8,6 +8,14 @@
 
 static inline void setup_release(struct task_struct *t, lt_t release)
 {
+        /* Shift all tasks that are actually daemons that inherit
+         * priority to be released immediatly. They need to be ready to run
+         * at _all_ times.
+         */
+        if (unlikely(is_persistent(t))) {
+                release = litmus_clock();
+        }
+
         /* prepare next release */
         t->rt_param.job_params.release = release;
         t->rt_param.job_params.deadline = release + get_rt_relative_deadline(t);
@@ -40,7 +48,7 @@ void release_at(struct task_struct *t, lt_t start)
 {
         BUG_ON(!t);
         setup_release(t, start);
-        tsk_rt(t)->completed = 0;
+        t->rt_param.completed = 0;
 }
 
 
diff --git a/litmus/nvidia_info.c b/litmus/nvidia_info.c
index 7160bb85ac9d..dda863009fee 100644
--- a/litmus/nvidia_info.c
+++ b/litmus/nvidia_info.c
@@ -729,6 +729,7 @@ int gpu_owner_increase_priority(struct task_struct *t)
                 binheap_decrease(&tsk_rt(t)->gpu_owner_node, &reg->owners);
         }
 
+#ifdef CONFIG_LITMUS_SOFTIRQD
         hp = container_of(binheap_top_entry(&reg->owners, struct rt_param, gpu_owner_node),
                                           struct task_struct, rt_param);
 
@@ -738,6 +739,7 @@ int gpu_owner_increase_priority(struct task_struct *t)
 
                 retval = gpu_klmirqd_increase_priority(reg->thread, hp_eff);
         }
+#endif
 
 out:
         return retval;
@@ -774,6 +776,7 @@ int gpu_owner_decrease_priority(struct task_struct *t)
         binheap_add(&tsk_rt(t)->gpu_owner_node, &reg->owners,
                                 struct rt_param, gpu_owner_node);
 
+#ifdef CONFIG_LITMUS_SOFTIRQD
         if (hp == t) { /* t was originally the hp */
                 struct task_struct *new_hp =
                         container_of(binheap_top_entry(&reg->owners, struct rt_param, gpu_owner_node),
@@ -784,6 +787,7 @@ int gpu_owner_decrease_priority(struct task_struct *t)
                         retval = gpu_klmirqd_decrease_priority(reg->thread, hp_eff);
                 }
         }
+#endif
 
 out:
         return retval;
diff --git a/litmus/sched_cedf.c b/litmus/sched_cedf.c
index 2ec919dc850c..1feb2fbe42bc 100644
--- a/litmus/sched_cedf.c
+++ b/litmus/sched_cedf.c
@@ -211,6 +211,7 @@ static cpu_entry_t* lowest_prio_cpu(cedf_domain_t *cluster)
         return binheap_top_entry(&cluster->cpu_heap, cpu_entry_t, hn);
 }
 
+static noinline void unlink(struct task_struct* t);
 
 /* link_task_to_cpu - Update the link of a CPU.
  *                    Handles the case where the to-be-linked task is already
@@ -237,8 +238,28 @@ static noinline void link_task_to_cpu(struct task_struct* linked,
                 on_cpu = linked->rt_param.scheduled_on;
                 if (on_cpu != NO_CPU) {
                         sched = &per_cpu(cedf_cpu_entries, on_cpu);
-                        /* this should only happen if not linked already */
+
                         BUG_ON(sched->linked == linked);
+#if 0
+                        /* this should only happen if not linked already */
+                        if (sched->linked == linked) {
+                                printk(KERN_EMERG "%s/%d hit sched bug. is_aux = %d; has_aux = %d\n",
+                                                linked->comm, linked->pid, tsk_rt(linked)->is_aux_task, tsk_rt(linked)->has_aux_tasks);
+                                printk(KERN_EMERG "eCPU: %d, eLinked: %s/%d, sCPU: %d, sched: %s/%d\n",
+                                                entry->cpu,
+                                                (entry->linked) ?
+                                                        entry->linked->comm : "nil",
+                                                (entry->linked) ?
+                                                        entry->linked->pid : -1,
+                                                sched->cpu,
+                                                (sched->linked) ?
+                                                        sched->linked->comm : "nil",
+                                                (sched->linked) ?
+                                                        sched->linked->pid : -1);
+                                TRACE_TASK(linked, "LINK BUG!\n");
+                                unlink(linked);
+                        }
+#endif
 
                         /* If we are already scheduled on the CPU to which we
                          * wanted to link, we don't need to do the swap --
@@ -957,7 +978,7 @@ static void cedf_task_new(struct task_struct * t, int on_rq, int running)
         cpu_entry_t*            entry;
         cedf_domain_t*          cluster;
 
-        TRACE("c-edf: task new %d\n", t->pid);
+        TRACE("c-edf: task new %d (param running = %d, is_running = %d)\n", t->pid, running, is_running(t));
 
         /* the cluster doesn't change even if t is running */
         cluster = task_cpu_cluster(t);
@@ -986,44 +1007,41 @@ static void cedf_task_new(struct task_struct * t, int on_rq, int running)
         } else {
                 t->rt_param.scheduled_on = NO_CPU;
         }
-        t->rt_param.linked_on          = NO_CPU;
+        t->rt_param.linked_on = NO_CPU;
 
-        cedf_job_arrival(t);
+        if (is_running(t)) {
+                cedf_job_arrival(t);
+        }
+        else {
+                TRACE("Deferred job arrival because %d is blocked.\n", t->pid);
+                unlink(t); /* needed?? */
+        }
         raw_spin_unlock_irqrestore(&(cluster->cluster_lock), flags);
 }
 
 static void cedf_task_wake_up(struct task_struct *task)
 {
         unsigned long flags;
-        //lt_t now;
         cedf_domain_t *cluster;
+        lt_t now = litmus_clock();
 
-        TRACE_TASK(task, "wake_up at %llu\n", litmus_clock());
+        TRACE_TASK(task, "wake_up at %llu\n", now);
 
         cluster = task_cpu_cluster(task);
 
         raw_spin_lock_irqsave(&cluster->cluster_lock, flags);
 
-#if 0
-        /* sporadic task model. will increment job numbers automatically */
-        now = litmus_clock();
-        if (is_tardy(task, now)) {
-                /* new sporadic release */
+        if (unlikely(is_persistent(task) && is_tardy(task, now))) {
+                /* treat tardy perisistent tasks as if they were sporadic
+                   tasks by releasing a new job if they're tardy. */
                 release_at(task, now);
                 sched_trace_task_release(task);
         }
-        else {
-                if (task->rt.time_slice) {
-                        /* came back in time before deadline
-                        */
-                        tsk_rt(task)->completed = 0;
-                }
+        else if (task->rt.time_slice) {
+                /* periodic task model.  don't force job to end.
+                 * rely on user to say when jobs complete or when budget expires. */
+                tsk_rt(task)->completed = 0;
         }
-#else
-        /* periodic task model.  don't force job to end.
-         * rely on user to say when jobs complete or when budget expires. */
-        tsk_rt(task)->completed = 0;
-#endif
 
 #ifdef CONFIG_REALTIME_AUX_TASKS
         if (tsk_rt(task)->has_aux_tasks && !tsk_rt(task)->hide_from_aux_tasks) {
diff --git a/litmus/sched_trace.c b/litmus/sched_trace.c
index f4171fddbbb1..d52e2a75dd96 100644
--- a/litmus/sched_trace.c
+++ b/litmus/sched_trace.c
@@ -75,7 +75,6 @@ void sched_trace_log_message(const char* fmt, ...)
         va_end(args);
 }
 
-
 /*
  * log_read - Read the trace buffer
  *