8 files changed, 124 insertions, 106 deletions

diff --git a/include/litmus/litmus.h b/include/litmus/litmus.h
index 2da61fa58bdc..17d30326034c 100644
--- a/include/litmus/litmus.h
+++ b/include/litmus/litmus.h
@@ -85,6 +85,19 @@ static inline lt_t litmus_clock(void)
         return ktime_to_ns(ktime_get());
 }
 
+static inline int is_persistent(struct task_struct* t)
+{
+        int is_per = ( 0
+                #ifdef CONFIG_REALTIME_AUX_TASKS
+                        || t->rt_param.is_aux_task
+                #endif
+                #ifdef CONFIG_LITMUS_SOFTIRQD
+                        || t->rt_param.is_interrupt_thread
+                #endif
+                        );
+        return is_per;
+}
+
 /* A macro to convert from nanoseconds to ktime_t. */
 #define ns_to_ktime(t)          ktime_add_ns(ktime_set(0, 0), t)
 
diff --git a/litmus/aux_tasks.c b/litmus/aux_tasks.c
index aa851ab2655b..46217e78b1ec 100644
--- a/litmus/aux_tasks.c
+++ b/litmus/aux_tasks.c
@@ -67,11 +67,11 @@ int exit_aux_task(struct task_struct *t)
         return retval;
 }
 
+#ifdef CONFIG_REALTIME_AUX_TASK_PRIORITY_INHERITANCE
 static int aux_tasks_increase_priority(struct task_struct *leader, struct task_struct *hp)
 {
         int retval = 0;
 
-#ifdef CONFIG_REALTIME_AUX_TASK_PRIORITY_INHERITANCE
         struct list_head *pos;
 
         TRACE_CUR("Increasing priority of aux tasks in group %s/%d.\n", leader->comm, leader->pid);
@@ -94,7 +94,6 @@ static int aux_tasks_increase_priority(struct task_struct *leader, struct task_s
                         retval = litmus->__increase_prio(aux, hp);
                 }
         }
-#endif
 
         return retval;
 }
@@ -103,7 +102,6 @@ static int aux_tasks_decrease_priority(struct task_struct *leader, struct task_s
 {
         int retval = 0;
 
-#ifdef CONFIG_REALTIME_AUX_TASK_PRIORITY_INHERITANCE
         struct list_head *pos;
 
         TRACE_CUR("Decreasing priority of aux tasks in group %s/%d.\n", leader->comm, leader->pid);
@@ -121,11 +119,12 @@ static int aux_tasks_decrease_priority(struct task_struct *leader, struct task_s
                         retval = litmus->__decrease_prio(aux, hp);
                 }
         }
-#endif
 
         return retval;
 }
 
+#endif
+
 int aux_task_owner_increase_priority(struct task_struct *t)
 {
         int retval = 0;
@@ -167,9 +166,9 @@ int aux_task_owner_increase_priority(struct task_struct *t)
                 TRACE_CUR("%s/%d is new hp in group %s/%d.\n", t->comm, t->pid, leader->comm, leader->pid);
                 retval = aux_tasks_increase_priority(leader, hp_eff);
         }
-#endif
-
 out:
+
+#endif
         return retval;
 }
 
@@ -215,9 +214,9 @@ int aux_task_owner_decrease_priority(struct task_struct *t)
                         retval = aux_tasks_decrease_priority(leader, hp_eff);
                 }
         }
-#endif
-
 out:
+
+#endif
         return retval;
 }
 
@@ -305,9 +304,9 @@ long enable_aux_task_owner(struct task_struct *t)
                 retval = aux_tasks_increase_priority(leader,
                                            (tsk_rt(hp)->inh_task)? tsk_rt(hp)->inh_task : hp);
         }
-#endif
-
 out:
+
+#endif
         return retval;
 }
 
@@ -356,9 +355,9 @@ long disable_aux_task_owner(struct task_struct *t)
 
                 retval = aux_tasks_decrease_priority(leader, to_inh);
         }
-#endif
-
 out:
+
+#endif
         return retval;
 }
 
diff --git a/litmus/edf_common.c b/litmus/edf_common.c
index 441fbfddf0c2..ef22eb93dbf3 100644
--- a/litmus/edf_common.c
+++ b/litmus/edf_common.c
@@ -67,11 +67,14 @@ int edf_higher_prio(struct task_struct* first, struct task_struct* second)
                 return 0;
         }
 
-
         /* check for NULL tasks */
         if (!first || !second) {
                 return first && !second;
         }
+        /* check for non-realtime */
+        if (!is_realtime(first) || !is_realtime(second)) {
+                return is_realtime(first) && !is_realtime(second);
+        }
 
         /* There is some goofy stuff in this code here.  There are three subclasses
          * within the SCHED_LITMUS scheduling class:
@@ -92,49 +95,31 @@ int edf_higher_prio(struct task_struct* first, struct task_struct* second)
 
 #if defined(CONFIG_REALTIME_AUX_TASK_PRIORITY_BOOSTED)
         /* run aux tasks at max priority */
-        /* TODO: Actually use prio-boosting. */
-        if (first->rt_param.is_aux_task != second->rt_param.is_aux_task)
-        {
-                return (first->rt_param.is_aux_task > second->rt_param.is_aux_task);
-        }
-        else if(first->rt_param.is_aux_task && second->rt_param.is_aux_task)
-        {
-                if(first->group_leader == second->group_leader) {
-                        TRACE_CUR("aux tie break!\n");  // tie-break by BASE priority of the aux tasks
-                        goto aux_tie_break;
-                }
-                first = first->group_leader;
-                second = second->group_leader;
+        if (tsk_rt(first)->is_aux_task != tsk_rt(second)->is_aux_task) {
+                return (tsk_rt(first)->is_aux_task > tsk_rt(second)->is_aux_task);
         }
 #elif defined(CONFIG_REALTIME_AUX_TASK_PRIORITY_INHERITANCE)
         {
-                int first_lo_aux = first->rt_param.is_aux_task && !first->rt_param.inh_task;
-                int second_lo_aux = second->rt_param.is_aux_task && !second->rt_param.inh_task;
+                int first_lo_aux = tsk_rt(first)->is_aux_task && !tsk_rt(first)->inh_task;
+                int second_lo_aux = tsk_rt(second)->is_aux_task && !tsk_rt(second)->inh_task;
 
                 /* prioritize aux tasks without inheritance below real-time tasks */
                 if (first_lo_aux || second_lo_aux) {
                         // one of these is an aux task without inheritance.
-                        if(first_lo_aux && second_lo_aux) {
-                                TRACE_CUR("aux tie break!\n");  // tie-break by BASE priority of the aux tasks
-                                goto aux_tie_break;
-                        }
-                        else {
-
-                                // make the aux thread lowest priority real-time task
-                                int temp = 0;
-                                if (first_lo_aux && is_realtime(second)) {
-//                                      temp = 0;
-                                }
-                                else if(second_lo_aux && is_realtime(first)) {
-                                        temp = 1;
-                                }
+                        if (first_lo_aux != second_lo_aux) {
+                                int temp = (first_lo_aux < second_lo_aux);  // non-lo-aux has higher priority.
                                 TRACE_CUR("%s/%d >> %s/%d --- %d\n", first->comm, first->pid, second->comm, second->pid, temp);
                                 return temp;
                         }
+                        else {
+                                /* both MUST be lo_aux. tie-break. */
+                                TRACE_CUR("aux tie break!\n");
+                                goto aux_tie_break;
+                        }
                 }
 
-                if (first->rt_param.is_aux_task && second->rt_param.is_aux_task &&
-                        first->rt_param.inh_task == second->rt_param.inh_task) {
+                if (tsk_rt(first)->is_aux_task && tsk_rt(second)->is_aux_task &&
+                        tsk_rt(first)->inh_task == tsk_rt(second)->inh_task) {
                         // inh_task is !NULL for both tasks since neither was a lo_aux task.
                         // Both aux tasks inherit from the same task, so tie-break
                         // by base priority of the aux tasks.
@@ -146,33 +131,26 @@ int edf_higher_prio(struct task_struct* first, struct task_struct* second)
 
 #ifdef CONFIG_LITMUS_SOFTIRQD
         {
-                int first_lo_klmirqd = first->rt_param.is_interrupt_thread && !first->rt_param.inh_task;
-                int second_lo_klmirqd = second->rt_param.is_interrupt_thread && !second->rt_param.inh_task;
+                int first_lo_klmirqd = tsk_rt(first)->is_interrupt_thread && !tsk_rt(first)->inh_task;
+                int second_lo_klmirqd = tsk_rt(second)->is_interrupt_thread && !tsk_rt(second)->inh_task;
 
                 /* prioritize aux tasks without inheritance below real-time tasks */
                 if (first_lo_klmirqd || second_lo_klmirqd) {
                         // one of these is an klmirqd thread without inheritance.
-                        if(first_lo_klmirqd && second_lo_klmirqd) {
-                                TRACE_CUR("klmirqd tie break!\n");  // tie-break by BASE priority of the aux tasks
-                                goto klmirqd_tie_break;
-                        }
-                        else {
-                                // make the klmirqd thread the lowest-priority real-time task
-                                // but (above low-prio aux tasks and Linux tasks)
-                                int temp = 0;
-                                if (first_lo_klmirqd && is_realtime(second)) {
-//                                      temp = 0;
-                                }
-                                else if(second_lo_klmirqd && is_realtime(first)) {
-                                        temp = 1;
-                                }
+                        if (first_lo_klmirqd != second_lo_klmirqd) {
+                                int temp = (first_lo_klmirqd < second_lo_klmirqd);  // non-klmirqd has higher priority
                                 TRACE_CUR("%s/%d >> %s/%d --- %d\n", first->comm, first->pid, second->comm, second->pid, temp);
                                 return temp;
                         }
+                        else {
+                                /* both MUST be klmirqd. tie-break. */
+                                TRACE_CUR("klmirqd tie break!\n");
+                                goto klmirqd_tie_break;
+                        }
                 }
 
-                if (first->rt_param.is_interrupt_thread && second->rt_param.is_interrupt_thread &&
-                        first->rt_param.inh_task == second->rt_param.inh_task) {
+                if (tsk_rt(first)->is_interrupt_thread && tsk_rt(second)->is_interrupt_thread &&
+                        tsk_rt(first)->inh_task == tsk_rt(second)->inh_task) {
                         // inh_task is !NULL for both tasks since neither was a lo_klmirqd task.
                         // Both klmirqd tasks inherit from the same task, so tie-break
                         // by base priority of the klmirqd tasks.
@@ -187,19 +165,19 @@ int edf_higher_prio(struct task_struct* first, struct task_struct* second)
         /* Check for EFFECTIVE priorities. Change task
          * used for comparison in such a case.
          */
-        if (unlikely(first->rt_param.inh_task)
+        if (unlikely(tsk_rt(first)->inh_task)
 #ifdef CONFIG_LITMUS_NESTED_LOCKING
                 && (first_mode == EFFECTIVE)
 #endif
                 ) {
-                first_task = first->rt_param.inh_task;
+                first_task = tsk_rt(first)->inh_task;
         }
-        if (unlikely(second->rt_param.inh_task)
+        if (unlikely(tsk_rt(second)->inh_task)
 #ifdef CONFIG_LITMUS_NESTED_LOCKING
                 && (second_mode == EFFECTIVE)
 #endif
                 ) {
-                second_task = second->rt_param.inh_task;
+                second_task = tsk_rt(second)->inh_task;
         }
 
         /* Check for priority boosting. Tie-break by start of boosting.
@@ -222,17 +200,14 @@ int edf_higher_prio(struct task_struct* first, struct task_struct* second)
 
 #endif
 
-#ifdef CONFIG_REALTIME_AUX_TASKS
+#ifdef CONFIG_REALTIME_AUX_TASK_PRIORITY_INHERITANCE
 aux_tie_break:
 #endif
 #ifdef CONFIG_LITMUS_SOFTIRQD
 klmirqd_tie_break:
 #endif
 
-        if (!is_realtime(second_task)) {
-                return 1;
-        }
-        else if (earlier_deadline(first_task, second_task)) {
+        if (earlier_deadline(first_task, second_task)) {
                 return 1;
         }
         else if (get_deadline(first_task) == get_deadline(second_task)) {
@@ -297,10 +272,10 @@ klmirqd_tie_break:
                         }
                         else if (first_task->pid == second_task->pid) {
 #ifdef CONFIG_LITMUS_SOFTIRQD
-                                if (first_task->rt_param.is_interrupt_thread < second_task->rt_param.is_interrupt_thread) {
+                                if (tsk_rt(first_task)->is_interrupt_thread < tsk_rt(second_task)->is_interrupt_thread) {
                                         return 1;
                                 }
-                                else if (first_task->rt_param.is_interrupt_thread == second_task->rt_param.is_interrupt_thread) {
+                                else if (tsk_rt(first_task)->is_interrupt_thread == tsk_rt(second_task)->is_interrupt_thread) {
 #endif
 
 #if defined(CONFIG_REALTIME_AUX_TASK_PRIORITY_INHERITANCE)
@@ -311,7 +286,7 @@ klmirqd_tie_break:
 #endif
 
                                 /* Something could be wrong if you get this far. */
-                                if (unlikely(first->rt_param.inh_task == second->rt_param.inh_task)) {
+                                if (unlikely(tsk_rt(first)->inh_task == tsk_rt(second)->inh_task)) {
                                         /* Both tasks have the same inherited priority.
                                          * Likely in a bug-condition.
                                      */
@@ -324,22 +299,22 @@ klmirqd_tie_break:
                                 }
                                 else {
                                         /* At least one task must inherit */
-                                        BUG_ON(!first->rt_param.inh_task &&
-                                                   !second->rt_param.inh_task);
+                                        BUG_ON(!tsk_rt(first)->inh_task &&
+                                                   !tsk_rt(second)->inh_task);
 
                                         /* The task withOUT the inherited priority wins. */
-                                        if (second->rt_param.inh_task) {
+                                        if (tsk_rt(second)->inh_task) {
                                                 /*
                                                  * common with aux tasks.
                                                 TRACE_CUR("unusual comparison: "
                                                         "first = %s/%d  first_task = %s/%d  "
                                                         "second = %s/%d  second_task = %s/%d\n",
                                                         first->comm, first->pid,
-                                                        (first->rt_param.inh_task) ? first->rt_param.inh_task->comm : "(nil)",
-                                                        (first->rt_param.inh_task) ? first->rt_param.inh_task->pid : 0,
+                                                        (tsk_rt(first)->inh_task) ? tsk_rt(first)->inh_task->comm : "(nil)",
+                                                        (tsk_rt(first)->inh_task) ? tsk_rt(first)->inh_task->pid : 0,
                                                         second->comm, second->pid,
-                                                        (second->rt_param.inh_task) ? second->rt_param.inh_task->comm : "(nil)",
-                                                        (second->rt_param.inh_task) ? second->rt_param.inh_task->pid : 0);
+                                                        (tsk_rt(second)->inh_task) ? tsk_rt(second)->inh_task->comm : "(nil)",
+                                                        (tsk_rt(second)->inh_task) ? tsk_rt(second)->inh_task->pid : 0);
                                                  */
                                                 return 1;
                                         }
@@ -426,3 +401,4 @@ int edf_preemption_needed(rt_domain_t* rt, struct task_struct *t)
         /* make sure to get non-rt stuff out of the way */
         return !is_realtime(t) || edf_higher_prio(__next_ready(rt), t);
 }
+
diff --git a/litmus/ikglp_lock.c b/litmus/ikglp_lock.c
index aa6b659e437d..170c43fcc78e 100644
--- a/litmus/ikglp_lock.c
+++ b/litmus/ikglp_lock.c
@@ -870,7 +870,8 @@ static void ikglp_enqueue_on_donor(struct ikglp_semaphore *sem,
         }
         else {
                 TRACE_TASK(t, "No change in effective priority (it is %d/%s).  BUG?\n",
-                                   new_max_eff_prio->comm, new_max_eff_prio->pid);
+                                   (new_max_eff_prio) ? new_max_eff_prio->comm : "nil",
+                                   (new_max_eff_prio) ? new_max_eff_prio->pid : -1);
                 raw_spin_unlock(&tsk_rt(donee)->hp_blocked_tasks_lock);
                 unlock_fine_irqrestore(&sem->lock, flags);
         }
diff --git a/litmus/jobs.c b/litmus/jobs.c
index 596b48e699b1..bdfc41004d38 100644
--- a/litmus/jobs.c
+++ b/litmus/jobs.c
@@ -8,6 +8,14 @@
 
 static inline void setup_release(struct task_struct *t, lt_t release)
 {
+        /* Shift all tasks that are actually daemons that inherit
+         * priority to be released immediatly. They need to be ready to run
+         * at _all_ times.
+         */
+        if (unlikely(is_persistent(t))) {
+                release = litmus_clock();
+        }
+
         /* prepare next release */
         t->rt_param.job_params.release = release;
         t->rt_param.job_params.deadline = release + get_rt_relative_deadline(t);
@@ -40,7 +48,7 @@ void release_at(struct task_struct *t, lt_t start)
 {
         BUG_ON(!t);
         setup_release(t, start);
-        tsk_rt(t)->completed = 0;
+        t->rt_param.completed = 0;
 }
 
 
diff --git a/litmus/nvidia_info.c b/litmus/nvidia_info.c
index 7160bb85ac9d..dda863009fee 100644
--- a/litmus/nvidia_info.c
+++ b/litmus/nvidia_info.c
@@ -729,6 +729,7 @@ int gpu_owner_increase_priority(struct task_struct *t)
                 binheap_decrease(&tsk_rt(t)->gpu_owner_node, &reg->owners);
         }
 
+#ifdef CONFIG_LITMUS_SOFTIRQD
         hp = container_of(binheap_top_entry(&reg->owners, struct rt_param, gpu_owner_node),
                                           struct task_struct, rt_param);
 
@@ -738,6 +739,7 @@ int gpu_owner_increase_priority(struct task_struct *t)
 
                 retval = gpu_klmirqd_increase_priority(reg->thread, hp_eff);
         }
+#endif
 
 out:
         return retval;
@@ -774,6 +776,7 @@ int gpu_owner_decrease_priority(struct task_struct *t)
         binheap_add(&tsk_rt(t)->gpu_owner_node, &reg->owners,
                                 struct rt_param, gpu_owner_node);
 
+#ifdef CONFIG_LITMUS_SOFTIRQD
         if (hp == t) { /* t was originally the hp */
                 struct task_struct *new_hp =
                         container_of(binheap_top_entry(&reg->owners, struct rt_param, gpu_owner_node),
@@ -784,6 +787,7 @@ int gpu_owner_decrease_priority(struct task_struct *t)
                         retval = gpu_klmirqd_decrease_priority(reg->thread, hp_eff);
                 }
         }
+#endif
 
 out:
         return retval;
diff --git a/litmus/sched_cedf.c b/litmus/sched_cedf.c
index 2ec919dc850c..1feb2fbe42bc 100644
--- a/litmus/sched_cedf.c
+++ b/litmus/sched_cedf.c
@@ -211,6 +211,7 @@ static cpu_entry_t* lowest_prio_cpu(cedf_domain_t *cluster)
         return binheap_top_entry(&cluster->cpu_heap, cpu_entry_t, hn);
 }
 
+static noinline void unlink(struct task_struct* t);
 
 /* link_task_to_cpu - Update the link of a CPU.
  *                    Handles the case where the to-be-linked task is already
@@ -237,8 +238,28 @@ static noinline void link_task_to_cpu(struct task_struct* linked,
                 on_cpu = linked->rt_param.scheduled_on;
                 if (on_cpu != NO_CPU) {
                         sched = &per_cpu(cedf_cpu_entries, on_cpu);
-                        /* this should only happen if not linked already */
+
                         BUG_ON(sched->linked == linked);
+#if 0
+                        /* this should only happen if not linked already */
+                        if (sched->linked == linked) {
+                                printk(KERN_EMERG "%s/%d hit sched bug. is_aux = %d; has_aux = %d\n",
+                                                linked->comm, linked->pid, tsk_rt(linked)->is_aux_task, tsk_rt(linked)->has_aux_tasks);
+                                printk(KERN_EMERG "eCPU: %d, eLinked: %s/%d, sCPU: %d, sched: %s/%d\n",
+                                                entry->cpu,
+                                                (entry->linked) ?
+                                                        entry->linked->comm : "nil",
+                                                (entry->linked) ?
+                                                        entry->linked->pid : -1,
+                                                sched->cpu,
+                                                (sched->linked) ?
+                                                        sched->linked->comm : "nil",
+                                                (sched->linked) ?
+                                                        sched->linked->pid : -1);
+                                TRACE_TASK(linked, "LINK BUG!\n");
+                                unlink(linked);
+                        }
+#endif
 
                         /* If we are already scheduled on the CPU to which we
                          * wanted to link, we don't need to do the swap --
@@ -957,7 +978,7 @@ static void cedf_task_new(struct task_struct * t, int on_rq, int running)
         cpu_entry_t*            entry;
         cedf_domain_t*          cluster;
 
-        TRACE("c-edf: task new %d\n", t->pid);
+        TRACE("c-edf: task new %d (param running = %d, is_running = %d)\n", t->pid, running, is_running(t));
 
         /* the cluster doesn't change even if t is running */
         cluster = task_cpu_cluster(t);
@@ -986,44 +1007,41 @@ static void cedf_task_new(struct task_struct * t, int on_rq, int running)
         } else {
                 t->rt_param.scheduled_on = NO_CPU;
         }
-        t->rt_param.linked_on          = NO_CPU;
+        t->rt_param.linked_on = NO_CPU;
 
-        cedf_job_arrival(t);
+        if (is_running(t)) {
+                cedf_job_arrival(t);
+        }
+        else {
+                TRACE("Deferred job arrival because %d is blocked.\n", t->pid);
+                unlink(t); /* needed?? */
+        }
         raw_spin_unlock_irqrestore(&(cluster->cluster_lock), flags);
 }
 
 static void cedf_task_wake_up(struct task_struct *task)
 {
         unsigned long flags;
-        //lt_t now;
         cedf_domain_t *cluster;
+        lt_t now = litmus_clock();
 
-        TRACE_TASK(task, "wake_up at %llu\n", litmus_clock());
+        TRACE_TASK(task, "wake_up at %llu\n", now);
 
         cluster = task_cpu_cluster(task);
 
         raw_spin_lock_irqsave(&cluster->cluster_lock, flags);
 
-#if 0
-        /* sporadic task model. will increment job numbers automatically */
-        now = litmus_clock();
-        if (is_tardy(task, now)) {
-                /* new sporadic release */
+        if (unlikely(is_persistent(task) && is_tardy(task, now))) {
+                /* treat tardy perisistent tasks as if they were sporadic
+                   tasks by releasing a new job if they're tardy. */
                 release_at(task, now);
                 sched_trace_task_release(task);
         }
-        else {
-                if (task->rt.time_slice) {
-                        /* came back in time before deadline
-                        */
-                        tsk_rt(task)->completed = 0;
-                }
+        else if (task->rt.time_slice) {
+                /* periodic task model.  don't force job to end.
+                 * rely on user to say when jobs complete or when budget expires. */
+                tsk_rt(task)->completed = 0;
         }
-#else
-        /* periodic task model.  don't force job to end.
-         * rely on user to say when jobs complete or when budget expires. */
-        tsk_rt(task)->completed = 0;
-#endif
 
 #ifdef CONFIG_REALTIME_AUX_TASKS
         if (tsk_rt(task)->has_aux_tasks && !tsk_rt(task)->hide_from_aux_tasks) {
diff --git a/litmus/sched_trace.c b/litmus/sched_trace.c
index f4171fddbbb1..d52e2a75dd96 100644
--- a/litmus/sched_trace.c
+++ b/litmus/sched_trace.c
@@ -75,7 +75,6 @@ void sched_trace_log_message(const char* fmt, ...)
         va_end(args);
 }
 
-
 /*
  * log_read - Read the trace buffer
  *