Fixed three bugs with aux threads and nested locks

Fixes two bugs with nested locks:
1) List of aux threads could become corrupted.
  -- moved modifications to be within scheduler lock.

2) Fixed bad EDF comparison ordering that could lead
to schedule thrashing in an infinite loop.

3) Prevent aux threads from inheriting a priority from
a task that is blocked on a real-time litmus lock.
(since the aux threads can't possibly hold these locks,
we don't have to worry about inheritance.)

10 files changed, 98 insertions, 45 deletions

diff --git a/include/litmus/locking.h b/include/litmus/locking.h
index 296bbf6f7af0..4a5f198a0407 100644
--- a/include/litmus/locking.h
+++ b/include/litmus/locking.h
@@ -156,5 +156,8 @@ struct litmus_lock_ops {
 #endif
 
 
+void suspend_for_lock(void);
+
+
 #endif
 
diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h
index 8b9e14c461dc..44f85a366574 100644
--- a/include/litmus/rt_param.h
+++ b/include/litmus/rt_param.h
@@ -288,6 +288,7 @@ struct rt_param {
 #ifdef CONFIG_REALTIME_AUX_TASKS
         unsigned int            is_aux_task:1;
         unsigned int            has_aux_tasks:1;
+        unsigned int            hide_from_aux_tasks:1;
 
         struct list_head        aux_task_node;
         struct binheap_node     aux_task_owner_node;
diff --git a/litmus/aux_tasks.c b/litmus/aux_tasks.c
index b0617accdf7f..efda7dc0bd76 100644
--- a/litmus/aux_tasks.c
+++ b/litmus/aux_tasks.c
@@ -55,7 +55,7 @@ int exit_aux_task(struct task_struct *t)
         tsk_rt(t)->is_aux_task = 0;
 
         if (tsk_rt(t)->inh_task) {
-                litmus->decrease_prio(t, NULL);
+                litmus->__decrease_prio(t, NULL);
         }
 
         return retval;
@@ -114,31 +114,37 @@ int aux_task_owner_increase_priority(struct task_struct *t)
         int retval = 0;
         struct task_struct *leader;
         struct task_struct *hp = NULL;
+        struct task_struct *hp_eff = NULL;
 
-        BUG_ON(!tsk_rt(t)->has_aux_tasks);
         BUG_ON(!is_realtime(t));
-        BUG_ON(!binheap_is_in_heap(&tsk_rt(t)->aux_task_owner_node));
+        BUG_ON(!tsk_rt(t)->has_aux_tasks);
 
         leader = t->group_leader;
 
+        if (!binheap_is_in_heap(&tsk_rt(t)->aux_task_owner_node)) {
+                WARN_ON(!is_running(t));
+                TRACE_CUR("aux tasks may not inherit from %s/%d in group %s/%d\n",
+                                                t->comm, t->pid, leader->comm, leader->pid);
+                goto out;
+        }
+
         TRACE_CUR("task %s/%d in group %s/%d increasing priority.\n", t->comm, t->pid, leader->comm, leader->pid);
 
         hp = container_of(binheap_top_entry(&tsk_aux(leader)->aux_task_owners, struct rt_param, aux_task_owner_node),
                                           struct task_struct, rt_param);
+        hp_eff = effective_priority(hp);
 
-        if (hp == t) {
-                goto out; // already hp, nothing to do.
+        if (hp != t) { /* our position in the heap may have changed. hp is already at the root. */
+                binheap_decrease(&tsk_rt(t)->aux_task_owner_node, &tsk_aux(leader)->aux_task_owners);
         }
 
-        binheap_decrease(&tsk_rt(t)->aux_task_owner_node, &tsk_aux(leader)->aux_task_owners);
-
         hp = container_of(binheap_top_entry(&tsk_aux(leader)->aux_task_owners, struct rt_param, aux_task_owner_node),
                                           struct task_struct, rt_param);
 
-        if (hp == t) {
+        if (effective_priority(hp) != hp_eff) { /* the eff. prio. of hp has changed */
+                hp_eff = effective_priority(hp);
                 TRACE_CUR("%s/%d is new hp in group %s/%d.\n", t->comm, t->pid, leader->comm, leader->pid);
-                retval = aux_tasks_increase_priority(leader,
-                                           (tsk_rt(hp)->inh_task) ? tsk_rt(hp)->inh_task : hp);
+                retval = aux_tasks_increase_priority(leader, hp_eff);
         }
 
 out:
@@ -150,30 +156,41 @@ int aux_task_owner_decrease_priority(struct task_struct *t)
         int retval = 0;
         struct task_struct *leader;
         struct task_struct *hp = NULL;
-        struct task_struct *new_hp = NULL;
+        struct task_struct *hp_eff = NULL;
 
-        BUG_ON(!tsk_rt(t)->has_aux_tasks);
         BUG_ON(!is_realtime(t));
-        BUG_ON(!binheap_is_in_heap(&tsk_rt(t)->aux_task_owner_node));
+        BUG_ON(!tsk_rt(t)->has_aux_tasks);
 
         leader = t->group_leader;
 
+        if (!binheap_is_in_heap(&tsk_rt(t)->aux_task_owner_node)) {
+                WARN_ON(!is_running(t));
+                TRACE_CUR("aux tasks may not inherit from %s/%d in group %s/%d\n",
+                                                t->comm, t->pid, leader->comm, leader->pid);
+                goto out;
+        }
+
         TRACE_CUR("task %s/%d in group %s/%d decresing priority.\n", t->comm, t->pid, leader->comm, leader->pid);
 
         hp = container_of(binheap_top_entry(&tsk_aux(leader)->aux_task_owners, struct rt_param, aux_task_owner_node),
                                           struct task_struct, rt_param);
+        hp_eff = effective_priority(hp);
         binheap_delete(&tsk_rt(t)->aux_task_owner_node, &tsk_aux(leader)->aux_task_owners);
         binheap_add(&tsk_rt(t)->aux_task_owner_node, &tsk_aux(leader)->aux_task_owners,
                                 struct rt_param, aux_task_owner_node);
-        new_hp = container_of(binheap_top_entry(&tsk_aux(leader)->aux_task_owners, struct rt_param, aux_task_owner_node),
-                                                  struct task_struct, rt_param);
 
-        if (hp == t && new_hp != t) {
-                TRACE_CUR("%s/%d is no longer hp in group %s/%d.\n", t->comm, t->pid, leader->comm, leader->pid);
-                retval = aux_tasks_decrease_priority(leader,
-                                   (tsk_rt(new_hp)->inh_task) ? tsk_rt(new_hp)->inh_task : new_hp);
+        if (hp == t) { /* t was originally the hp */
+                struct task_struct *new_hp =
+                        container_of(binheap_top_entry(&tsk_aux(leader)->aux_task_owners, struct rt_param, aux_task_owner_node),
+                                                struct task_struct, rt_param);
+                if (effective_priority(new_hp) != hp_eff) { /* eff prio. of hp has changed */
+                        hp_eff = effective_priority(new_hp);
+                        TRACE_CUR("%s/%d is no longer hp in group %s/%d.\n", t->comm, t->pid, leader->comm, leader->pid);
+                        retval = aux_tasks_decrease_priority(leader, hp_eff);
+                }
         }
 
+out:
         return retval;
 }
 
@@ -302,8 +319,7 @@ static long __do_enable_aux_tasks(void)
                                   tsk_rt(t)->task_params.period);
 
                 /* inspect heap_node to see if it is an rt task */
-                if (tsk_rt(t)->task_params.period == 0) { //||
-                        //                      tsk_rt(t)->task_params.period == MAGIC_AUX_TASK_PERIOD) {
+                if (tsk_rt(t)->task_params.period == 0) {
                         if (!tsk_rt(t)->is_aux_task) {
                                 TRACE_CUR("AUX task in %s/%d: %s/%d:\n", leader->comm, leader->pid, t->comm, t->pid);
                                 /* hasn't been aux_tasks_increase_priorityted into rt. make it a aux. */
diff --git a/litmus/edf_common.c b/litmus/edf_common.c
index 7e0d3a5d0c4d..f4881452373d 100644
--- a/litmus/edf_common.c
+++ b/litmus/edf_common.c
@@ -237,6 +237,7 @@ int edf_higher_prio(struct task_struct* first, struct task_struct* second)
                                         second_task->rt_param.is_proxy_thread) {
                                         return 1;
                                 }
+                                else if (first_task->rt_param.is_proxy_thread == second_task->rt_param.is_proxy_thread) {
 #endif
 
 #ifdef CONFIG_REALTIME_AUX_TASKS
@@ -244,6 +245,7 @@ int edf_higher_prio(struct task_struct* first, struct task_struct* second)
                                 if (tsk_rt(first)->is_aux_task < tsk_rt(second)->is_aux_task) {
                                         return 1;
                                 }
+                                else if (tsk_rt(first)->is_aux_task == tsk_rt(second)->is_aux_task) {
 #endif
 
                                 /* Something could be wrong if you get this far. */
@@ -281,6 +283,14 @@ int edf_higher_prio(struct task_struct* first, struct task_struct* second)
                                                 return 1;
                                         }
                                 }
+
+#ifdef CONFIG_REALTIME_AUX_TASKS
+                                }
+#endif
+#ifdef CONFIG_LITMUS_SOFTIRQD
+                                }
+#endif
+
                         }
                 }
         }
diff --git a/litmus/ikglp_lock.c b/litmus/ikglp_lock.c
index 83b708ab85cb..bd7bfc0f48ac 100644
--- a/litmus/ikglp_lock.c
+++ b/litmus/ikglp_lock.c
@@ -963,7 +963,7 @@ int ikglp_lock(struct litmus_lock* l)
 
         TS_LOCK_SUSPEND;
 
-        schedule();
+        suspend_for_lock();
 
         TS_LOCK_RESUME;
 
diff --git a/litmus/kfmlp_lock.c b/litmus/kfmlp_lock.c
index bff857ed8d4e..ab472330095d 100644
--- a/litmus/kfmlp_lock.c
+++ b/litmus/kfmlp_lock.c
@@ -267,7 +267,7 @@ int kfmlp_lock(struct litmus_lock* l)
                  * when we wake up; we are guaranteed to have the lock since
                  * there is only one wake up per release (or steal).
                  */
-                schedule();
+                suspend_for_lock();
 
 
                 if(my_queue->owner == t) {
diff --git a/litmus/locking.c b/litmus/locking.c
index 12a23eb715cc..16c936ba8139 100644
--- a/litmus/locking.c
+++ b/litmus/locking.c
@@ -540,6 +540,35 @@ out:
         return passed;
 }
 
+
+void suspend_for_lock(void)
+{
+#ifdef CONFIG_REALTIME_AUX_TASKS
+        unsigned int restore = 0;
+        struct task_struct *t = current;
+        unsigned int hide;
+
+        if (tsk_rt(t)->has_aux_tasks) {
+                /* hide from aux tasks so they can't inherit our priority when we block
+                 * for a litmus lock. inheritance is already going to a litmus lock
+                 * holder. */
+                hide = tsk_rt(t)->hide_from_aux_tasks;
+                restore = 1;
+                tsk_rt(t)->hide_from_aux_tasks = 1;
+        }
+#endif
+
+        schedule();
+
+#ifdef CONFIG_REALTIME_AUX_TASKS
+        if (restore) {
+                /* restore our state */
+                tsk_rt(t)->hide_from_aux_tasks = hide;
+        }
+#endif
+}
+
+
 #else  // CONFIG_LITMUS_LOCKING
 
 struct fdso_ops generic_lock_ops = {};
diff --git a/litmus/rsm_lock.c b/litmus/rsm_lock.c
index 75ed87c5ed48..3dfd8ae9d221 100644
--- a/litmus/rsm_lock.c
+++ b/litmus/rsm_lock.c
@@ -289,7 +289,7 @@ int rsm_mutex_lock(struct litmus_lock* l)
                  * there is only one wake up per release.
                  */
 
-                schedule();
+                suspend_for_lock();
 
                 TS_LOCK_RESUME;
 
diff --git a/litmus/sched_cedf.c b/litmus/sched_cedf.c
index f5c9807090a1..6746d4d6033e 100644
--- a/litmus/sched_cedf.c
+++ b/litmus/sched_cedf.c
@@ -1032,7 +1032,7 @@ static void cedf_task_wake_up(struct task_struct *task)
 #endif
 
 #ifdef CONFIG_REALTIME_AUX_TASKS
-        if (tsk_rt(task)->has_aux_tasks) {
+        if (tsk_rt(task)->has_aux_tasks && !tsk_rt(task)->hide_from_aux_tasks) {
                 TRACE_CUR("%s/%d is ready so aux tasks may not inherit.\n", task->comm, task->pid);
                 disable_aux_task_owner(task);
         }
@@ -1057,7 +1057,7 @@ static void cedf_task_block(struct task_struct *t)
         unlink(t);
 
 #ifdef CONFIG_REALTIME_AUX_TASKS
-        if (tsk_rt(t)->has_aux_tasks) {
+        if (tsk_rt(t)->has_aux_tasks && !tsk_rt(t)->hide_from_aux_tasks) {
 
                 TRACE_CUR("%s/%d is blocked so aux tasks may inherit.\n", t->comm, t->pid);
                 enable_aux_task_owner(t);
@@ -1079,19 +1079,16 @@ static void cedf_task_exit(struct task_struct * t)
         cedf_change_prio_pai_tasklet(t, NULL);
 #endif
 
-#ifdef CONFIG_REALTIME_AUX_TASKS
-        if (tsk_rt(t)->is_aux_task) {
-                exit_aux_task(t); /* cannot be called with gsnedf_lock held */
-        }
-#endif
-
         /* unlink if necessary */
         raw_spin_lock_irqsave(&cluster->cluster_lock, flags);
 
 #ifdef CONFIG_REALTIME_AUX_TASKS
         /* make sure we clean up on our way out */
-        if(tsk_rt(t)->has_aux_tasks) {
-                disable_aux_task_owner(t); /* must be called witl gsnedf_lock held */
+        if (unlikely(tsk_rt(t)->is_aux_task)) {
+                exit_aux_task(t);
+        }
+        else if(tsk_rt(t)->has_aux_tasks) {
+                disable_aux_task_owner(t);
         }
 #endif
 
diff --git a/litmus/sched_gsn_edf.c b/litmus/sched_gsn_edf.c
index ed9b4697a5a2..04b189e54b03 100644
--- a/litmus/sched_gsn_edf.c
+++ b/litmus/sched_gsn_edf.c
@@ -1050,7 +1050,7 @@ static void gsnedf_task_wake_up(struct task_struct *task)
 #endif
 
 #ifdef CONFIG_REALTIME_AUX_TASKS
-        if (tsk_rt(task)->has_aux_tasks) {
+        if (tsk_rt(task)->has_aux_tasks && !tsk_rt(task)->hide_from_aux_tasks) {
                 TRACE_CUR("%s/%d is ready so aux tasks may not inherit.\n", task->comm, task->pid);
                 disable_aux_task_owner(task);
         }
@@ -1072,7 +1072,7 @@ static void gsnedf_task_block(struct task_struct *t)
         unlink(t);
 
 #ifdef CONFIG_REALTIME_AUX_TASKS
-        if (tsk_rt(t)->has_aux_tasks) {
+        if (tsk_rt(t)->has_aux_tasks && !tsk_rt(t)->hide_from_aux_tasks) {
 
                 TRACE_CUR("%s/%d is blocked so aux tasks may inherit.\n", t->comm, t->pid);
                 enable_aux_task_owner(t);
@@ -1093,19 +1093,16 @@ static void gsnedf_task_exit(struct task_struct * t)
         gsnedf_change_prio_pai_tasklet(t, NULL);
 #endif
 
-#ifdef CONFIG_REALTIME_AUX_TASKS
-        if (tsk_rt(t)->is_aux_task) {
-                exit_aux_task(t); /* cannot be called with gsnedf_lock held */
-        }
-#endif
-
         /* unlink if necessary */
         raw_spin_lock_irqsave(&gsnedf_lock, flags);
 
 #ifdef CONFIG_REALTIME_AUX_TASKS
         /* make sure we clean up on our way out */
-        if(tsk_rt(t)->has_aux_tasks) {
-                disable_aux_task_owner(t); /* must be called witl gsnedf_lock held */
+        if (unlikely(tsk_rt(t)->is_aux_task)) {
+                exit_aux_task(t);
+        }
+        else if(tsk_rt(t)->has_aux_tasks) {
+                disable_aux_task_owner(t);
         }
 #endif
 
@@ -1664,7 +1661,7 @@ int gsnedf_fmlp_lock(struct litmus_lock* l)
                  * there is only one wake up per release.
                  */
 
-                schedule();
+                suspend_for_lock();
 
                 TS_LOCK_RESUME;