Cleanup budget tracking in wait_for_release.

Also added logic for real-time tasks to prevent
worker threads (klmirqd and aux tasks) from inheriting
from tasks waiting for release.

6 files changed, 128 insertions, 67 deletions

diff --git a/litmus/budget.c b/litmus/budget.c
index 53fe08d341d0..4b464c4d8507 100644
--- a/litmus/budget.c
+++ b/litmus/budget.c
@@ -181,13 +181,20 @@ void sobliv_on_blocked(struct task_struct* t)
                         /* clear the exhausted flag so handle will re-run. this will not
                          * trigger another exhaustion signal since signals are controled by
                          * BTF_SIG_BUDGET_SENT. */
-                        int exhausted = bt_flag_test_and_clear(t, BTF_BUDGET_EXHAUSTED);
+                        //int exhausted = bt_flag_test_and_clear(t, BTF_BUDGET_EXHAUSTED);
                         //BUG_ON(!exhausted);
-                        WARN_ON(!exhausted);
+                        //WARN_ON(!exhausted);
+                        bt_flag_clear(t, BTF_BUDGET_EXHAUSTED);
 
-                        TRACE_TASK(t, "budget timer not armed. "
-                                           "Raced with exhaustion-resched? Re-arming.\n");
-                        arm_enforcement_timer(t, 1);
+                        if (likely(!bt_flag_is_set(t, BTF_WAITING_FOR_RELEASE))) {
+                                TRACE_TASK(t, "budget timer not armed. "
+                                                   "Raced with exhaustion-resched? Re-arming.\n");
+                                arm_enforcement_timer(t, 1);
+                        }
+                        else {
+                                TRACE_TASK(t, "not arming timer because task is waiting "
+                                                                "for release.\n");
+                        }
                 }
         }
 }
@@ -199,11 +206,14 @@ void sobliv_on_wakeup(struct task_struct* t)
                  * suspended while draining in exec_cost. suspend_timestamp was
                  * either set when we entered top-m while asleep, or when we
                  * blocked. */
-                lt_t suspend_cost;
-                BUG_ON(!tsk_rt(t)->budget.suspend_timestamp);
-                suspend_cost = litmus_clock() - tsk_rt(t)->budget.suspend_timestamp;
-                TRACE_TASK(t, "budget consumed while suspended: %llu\n", suspend_cost);
-                get_exec_time(t) += suspend_cost;
+                if (tsk_rt(t)->budget.suspend_timestamp) {
+                        lt_t suspend_cost = litmus_clock() - tsk_rt(t)->budget.suspend_timestamp;
+                        TRACE_TASK(t, "budget consumed while suspended: %llu\n", suspend_cost);
+                        get_exec_time(t) += suspend_cost;
+                }
+                else {
+                        WARN_ON(!bt_flag_is_set(t, BTF_WAITING_FOR_RELEASE));
+                }
         }
 }
 
diff --git a/litmus/fifo_lock.c b/litmus/fifo_lock.c
index 88f27a016b16..745472c30dfd 100644
--- a/litmus/fifo_lock.c
+++ b/litmus/fifo_lock.c
@@ -739,6 +739,8 @@ void fifo_mutex_budget_exhausted(struct litmus_lock* l, struct task_struct* t)
         /* DGL lock must already be held on this code path if DGLs are enabled. */
         lock_fine_irqsave(&mutex->lock, flags);
 
+        TRACE_TASK(t, "handling budget exhaustion for FIFO lock %d\n", l->ident);
+
         /* unlocks mutex->lock */
         __fifo_mutex_propagate_decrease_inheritance(&mutex->litmus_lock, t, flags, 1);
 }
diff --git a/litmus/jobs.c b/litmus/jobs.c
index 1479cddad9de..018b4982afb5 100644
--- a/litmus/jobs.c
+++ b/litmus/jobs.c
@@ -61,3 +61,55 @@ long complete_job(void)
         schedule();
         return 0;
 }
+
+
+#if defined(CONFIG_REALTIME_AUX_TASKS) || defined(CONFIG_LITMUS_NVIDIA)
+
+void hide_from_workers(struct task_struct *t, worker_visibility_t *wv)
+{
+#ifdef CONFIG_REALTIME_AUX_TASKS
+        if (tsk_rt(t)->has_aux_tasks) {
+                if (wv) {
+                        wv->aux_hide = tsk_rt(t)->hide_from_aux_tasks;
+                        wv->do_aux_restore = 1;
+                }
+                tsk_rt(t)->hide_from_aux_tasks = 1;
+        }
+#endif
+#ifdef CONFIG_LITMUS_NVIDIA
+        if (tsk_rt(t)->held_gpus) {
+                if (wv) {
+                        wv->gpu_hide = tsk_rt(t)->hide_from_gpu;
+                        wv->do_gpu_restore = 1;
+                }
+                tsk_rt(t)->hide_from_gpu = 1;
+        }
+#endif
+}
+
+void show_to_workers(struct task_struct *t, worker_visibility_t *wv)
+{
+        if (wv) {
+#ifdef CONFIG_REALTIME_AUX_TASKS
+                if (wv->do_aux_restore)
+                        tsk_rt(t)->hide_from_aux_tasks = wv->aux_hide;
+#endif
+#ifdef CONFIG_LITMUS_NVIDIA
+                if (wv->do_gpu_restore)
+                        tsk_rt(t)->hide_from_gpu = wv->gpu_hide;
+#endif
+        }
+        else {
+#ifdef CONFIG_REALTIME_AUX_TASKS
+                if (tsk_rt(t)->has_aux_tasks)
+                        tsk_rt(t)->hide_from_aux_tasks = 0;
+#endif
+#ifdef CONFIG_LITMUS_NVIDIA
+                if (tsk_rt(t)->held_gpus)
+                        tsk_rt(t)->hide_from_gpu = 0;
+#endif
+        }
+}
+
+#endif
+
diff --git a/litmus/locking.c b/litmus/locking.c
index 43afa920d3f1..67f5fda590c0 100644
--- a/litmus/locking.c
+++ b/litmus/locking.c
@@ -20,6 +20,10 @@
 #include <litmus/gpu_affinity.h>
 #endif
 
+#if defined(CONFIG_REALTIME_AUX_TASKS) || defined(CONFIG_LITMUS_NVIDIA)
+#include <litmus/jobs.h>
+#endif
+
 static int create_generic_lock(void** obj_ref, obj_type_t type, void* __user arg);
 static int open_generic_lock(struct od_table_entry* entry, void* __user arg);
 static int close_generic_lock(struct od_table_entry* entry);
@@ -767,66 +771,27 @@ void suspend_for_lock(void)
         struct task_struct *t = current;
 #endif
 
-#ifdef CONFIG_REALTIME_AUX_TASKS
-        unsigned int aux_restore = 0;
-        unsigned int aux_hide;
-#endif
-
-#ifdef CONFIG_LITMUS_NVIDIA
-        unsigned int gpu_restore = 0;
-        unsigned int gpu_hide;
-#endif
-
-#ifdef CONFIG_REALTIME_AUX_TASKS
-        if (tsk_rt(t)->has_aux_tasks) {
-                /* hide from aux tasks so they can't inherit our priority when we block
-                 * for a litmus lock. inheritance is already going to a litmus lock
-                 * holder. */
-                aux_hide = tsk_rt(t)->hide_from_aux_tasks;
-                aux_restore = 1;
-                tsk_rt(t)->hide_from_aux_tasks = 1;
-        }
-#endif
-
-#ifdef CONFIG_LITMUS_NVIDIA
-        if (tsk_rt(t)->held_gpus) {
-                gpu_hide = tsk_rt(t)->hide_from_gpu;
-                gpu_restore = 1;
-                tsk_rt(t)->hide_from_gpu = 1;
-        }
+#if defined(CONFIG_REALTIME_AUX_TASKS) || defined(CONFIG_LITMUS_NVIDIA)
+        DECLARE_WORKER_VIS_FLAGS(vis_flags);
+        hide_from_workers(t, &vis_flags);
 #endif
 
 #if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA)
         // disable tracking
-        if(tsk_rt(t)->held_gpus) {
+        if(tsk_rt(t)->held_gpus)
                 tsk_rt(t)->suspend_gpu_tracker_on_block = 1;
-        }
 #endif
 
         schedule();
 
-
-        /* TODO: Move the following to wake_up_for_lock()? */
-
 #if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA)
         // re-enable tracking
-        if(tsk_rt(t)->held_gpus) {
+        if(tsk_rt(t)->held_gpus)
                 tsk_rt(t)->suspend_gpu_tracker_on_block = 0;
-        }
 #endif
 
-#ifdef CONFIG_LITMUS_NVIDIA
-        if (gpu_restore) {
-                /* restore our state */
-                tsk_rt(t)->hide_from_gpu = gpu_hide;
-        }
-#endif
-
-#ifdef CONFIG_REALTIME_AUX_TASKS
-        if (aux_restore) {
-                /* restore our state */
-                tsk_rt(t)->hide_from_aux_tasks = aux_hide;
-        }
+#if defined(CONFIG_REALTIME_AUX_TASKS) || defined(CONFIG_LITMUS_NVIDIA)
+        show_to_workers(t, &vis_flags);
 #endif
 }
 
diff --git a/litmus/sched_cedf.c b/litmus/sched_cedf.c
index 9ec068e9fcf7..aab30a786515 100644
--- a/litmus/sched_cedf.c
+++ b/litmus/sched_cedf.c
@@ -222,6 +222,9 @@ static void cedf_track_in_top_m(struct task_struct *t)
 
                 return;
         }
+
+        BUG_ON(binheap_empty(&cluster->top_m));
+
         bt = binheap_top_entry(&cluster->top_m, struct budget_tracker, top_m_node);
         mth_highest =
                 container_of(
@@ -230,7 +233,6 @@ static void cedf_track_in_top_m(struct task_struct *t)
                                 rt_param);
 
         if (__edf_higher_prio(t, BASE, mth_highest, BASE)) {
-
                 TRACE_TASK(t, "adding to top-m (evicting %s/%d)\n",
                                    mth_highest->comm, mth_highest->pid);
 
@@ -266,7 +268,6 @@ static void cedf_untrack_in_top_m(struct task_struct *t)
         }
 
         if (bt_flag_is_set(t, BTF_IS_TOP_M)) {
-
                 TRACE_TASK(t, "removing task from top-m\n");
 
                 /* delete t's entry */
@@ -1310,8 +1311,15 @@ static struct task_struct* cedf_schedule(struct task_struct * prev)
 #endif
 
         /* Do budget stuff */
-        if (blocks)
-                budget_state_machine(prev,on_blocked);
+        if (blocks) {
+                if (likely(!bt_flag_is_set(prev, BTF_WAITING_FOR_RELEASE)))
+                        budget_state_machine(prev,on_blocked);
+                else {
+                        /* waiting for release. 'exit' the scheduler. */
+                        cedf_untrack_in_top_m(prev);
+                        budget_state_machine(prev,on_exit);
+                }
+        }
         else if (sleep)
                 budget_state_machine(prev,on_sleep);
         else if (preempt)
diff --git a/litmus/sync.c b/litmus/sync.c
index 749ae62c2b87..dfd9e40ab28d 100644
--- a/litmus/sync.c
+++ b/litmus/sync.c
@@ -15,6 +15,7 @@
 #include <litmus/jobs.h>
 
 #include <litmus/sched_trace.h>
+#include <litmus/budget.h>
 
 struct ts_release_wait {
         struct list_head list;
@@ -39,6 +40,13 @@ static long do_wait_for_ts_release(struct timespec *wake)
 
         long ret = -ERESTARTSYS;
 
+        struct task_struct *t = current;
+        int is_rt = is_realtime(t);
+
+#if defined(CONFIG_REALTIME_AUX_TASKS) || defined(CONFIG_LITMUS_NVIDIA)
+        DECLARE_WORKER_VIS_FLAGS(vis_flags);
+#endif
+
         if (mutex_lock_interruptible(&task_release_lock))
                 goto out;
 
@@ -46,29 +54,45 @@ static long do_wait_for_ts_release(struct timespec *wake)
 
         mutex_unlock(&task_release_lock);
 
+        if (is_rt) {
+#if defined(CONFIG_REALTIME_AUX_TASKS) || defined(CONFIG_LITMUS_NVIDIA)
+                hide_from_workers(t, &vis_flags);
+#endif
+                bt_flag_set(t, BTF_WAITING_FOR_RELEASE);
+                mb();
+        }
+
         /* We are enqueued, now we wait for someone to wake us up. */
         ret = wait_for_completion_interruptible(&wait.completion);
 
+        if (is_rt) {
+                bt_flag_clear(t, BTF_WAITING_FOR_RELEASE);
+#if defined(CONFIG_REALTIME_AUX_TASKS) || defined(CONFIG_LITMUS_NVIDIA)
+                show_to_workers(t, &vis_flags);
+#endif
+        }
+
         if (!ret) {
-                if (is_realtime(current)) {
+                if (is_rt) {
                         lt_t phasedRelease = wait.ts_release_time
-                                        + current->rt_param.task_params.phase;
+                                        + t->rt_param.task_params.phase;
                         *wake = ns_to_timespec(phasedRelease);
 
                         /* Setting this flag before releasing ensures that this CPU
                          * will be the next CPU to requeue the task on a ready or
                          * release queue.
                          */
-                        tsk_rt(current)->completed = 1;
-                        tsk_rt(current)->job_params.backlog = 0;
-                        bt_flags_reset(current);
+                        tsk_rt(t)->completed = 1;
+                        tsk_rt(t)->job_params.backlog = 0;
+                        tsk_rt(t)->job_params.is_backlogged_job = 0;
+                        tsk_rt(t)->budget.suspend_timestamp = 0;
+                        bt_flag_clear(t, BTF_BUDGET_EXHAUSTED);
                         mb();
 
                         /* completion succeeded, set up release. subtract off
                          * period because schedule()->job_completion() will
                          * advances us to the correct time */
-                        litmus->release_at(current,
-                                phasedRelease - current->rt_param.task_params.period);
+                        litmus->release_at(t, phasedRelease - t->rt_param.task_params.period);
                         schedule();
                 }
                 else {