Add more robust budget tracking/enforcement.

This patch adds more feature-rich budget tracking/enforcement
features. Budget tracking is now controlled by a state machine.
Each task can elect to use a different budget policy. Hooks
are in place to implement bandwidth inheritance (BWI) and
virtually exclusive resources.

15 files changed, 855 insertions, 128 deletions

diff --git a/include/litmus/budget.h b/include/litmus/budget.h
index bd2d5c964f92..aff48d9218fa 100644
--- a/include/litmus/budget.h
+++ b/include/litmus/budget.h
@@ -1,36 +1,151 @@
 #ifndef _LITMUS_BUDGET_H_
 #define _LITMUS_BUDGET_H_
 
-/* Update the per-processor enforcement timer (arm/reproram/cancel) for
- * the next task. */
-void update_enforcement_timer(struct task_struct* t);
+#include <linux/hrtimer.h>
+#include <linux/semaphore.h>
 
-inline static int budget_exhausted(struct task_struct* t)
+#include <litmus/binheap.h>
+
+struct enforcement_timer
 {
-        return get_exec_time(t) >= get_exec_cost(t);
-}
+        raw_spinlock_t lock;
+        struct hrtimer timer;
+        unsigned int job_when_armed;
+        unsigned int armed:1;
+};
+
+int cancel_enforcement_timer(struct task_struct* t);
+
+typedef void (*scheduled_t)(struct task_struct* t);
+typedef void (*blocked_t)(struct task_struct* t);
+typedef void (*preempt_t)(struct task_struct* t);
+typedef void (*sleep_t)(struct task_struct* t);
+typedef void (*wakeup_t)(struct task_struct* t);
+
+#define IN_SCHEDULE 1
+
+typedef enum hrtimer_restart (*exhausted_t)(struct task_struct* t,
+                                int in_schedule);
+typedef void (*exit_t)(struct task_struct* t);
+typedef void (*inherit_t)(struct task_struct* t, struct task_struct* prio_inh);
+typedef void (*disinherit_t)(struct task_struct* t,
+                                struct task_struct* prio_inh);
+
+typedef void (*enter_top_m_t)(struct task_struct* t);
+typedef void (*exit_top_m_t)(struct task_struct* t);
 
-inline static lt_t budget_remaining(struct task_struct* t)
+struct budget_tracker_ops
 {
-        if (!budget_exhausted(t))
-                return get_exec_cost(t) - get_exec_time(t);
-        else
-                /* avoid overflow */
-                return 0;
-}
+        scheduled_t                     on_scheduled;   /* called from litmus_schedule(). */
+        blocked_t                       on_blocked;             /* called from plugin::schedule() */
+        preempt_t                       on_preempt;             /* called from plugin::schedule() */
+        sleep_t                         on_sleep;               /* called from plugin::schedule() */
+        wakeup_t                        on_wakeup;
 
-#define budget_enforced(t) (tsk_rt(t)->task_params.budget_policy != NO_ENFORCEMENT)
+        exit_t                          on_exit;                /* task exiting rt mode */
 
-#define budget_precisely_enforced(t) (tsk_rt(t)->task_params.budget_policy \
-                                      == PRECISE_ENFORCEMENT)
+        /* called by plugin::tick() or timer interrupt */
+        exhausted_t                     on_exhausted;
 
-static inline int requeue_preempted_job(struct task_struct* t)
+        /* inheritance callbacks for bandwidth inheritance-related
+           budget tracking/enforcement methods */
+        inherit_t                       on_inherit;
+        disinherit_t            on_disinherit;
+
+        enter_top_m_t           on_enter_top_m; /* task enters top-m priority tasks */
+        exit_top_m_t            on_exit_top_m;  /* task exits top-m priority tasks */
+};
+
+struct budget_tracker
+{
+        struct enforcement_timer timer;
+        const struct budget_tracker_ops* ops;
+        unsigned long flags;
+
+        struct binheap_node top_m_node;
+        lt_t suspend_timestamp;
+};
+
+/* budget tracker flags */
+enum BT_FLAGS
 {
-        /* Add task to ready queue only if not subject to budget enforcement or
-         * if the job has budget remaining. t may be NULL.
-         */
-        return t && !is_completed(t) &&
-                (!budget_exhausted(t) || !budget_enforced(t));
-}
+        BTF_BUDGET_EXHAUSTED    = 0,
+        BTF_SIG_BUDGET_SENT             = 1,
+        BTF_IS_TOP_M                    = 2,
+        BTF_WAITING_FOR_RELEASE = 3,
+};
+
+/* Functions for simple DRAIN_SIMPLE policy common
+ * to every scheduler. Scheduler must provide
+ * implementation for simple_on_exhausted().
+ */
+void simple_on_scheduled(struct task_struct* t);
+void simple_on_blocked(struct task_struct* t);
+void simple_on_preempt(struct task_struct* t);
+void simple_on_sleep(struct task_struct* t);
+void simple_on_exit(struct task_struct* t);
+
+
+/* Functions for DRAIN_SIMPLE_IO policy common
+ * to every scheduler. Scheduler must provide
+ * implementation for simple_io_on_exhausted().
+ */
+#define simple_io_on_scheduled  simple_on_scheduled
+void simple_io_on_blocked(struct task_struct* t);
+void simple_io_on_wakeup(struct task_struct* t);
+#define simple_io_on_preempt    simple_on_preempt
+#define simple_io_on_sleep      simple_on_sleep
+#define simple_io_on_exit       simple_on_exit
+
+
+/* Functions for DRAIN_SOBLIV policy common
+ * to every scheduler. Scheduler must provide
+ * implementation for sobliv_on_exhausted().
+ *
+ * Limitation: Quantum budget tracking is unsupported.
+ */
+void sobliv_on_blocked(struct task_struct* t);
+void sobliv_on_wakeup(struct task_struct* t);
+#define sobliv_on_exit  simple_on_exit
+void sobliv_on_inherit(struct task_struct* t, struct task_struct* prio_inh);
+void sobliv_on_disinherit(struct task_struct* t, struct task_struct* prio_inh);
+void sobliv_on_enter_top_m(struct task_struct* t);
+void sobliv_on_exit_top_m(struct task_struct* t);
+
+void reevaluate_inheritance(struct task_struct* t);
+
+#define budget_state_machine(t, evt) \
+        do { \
+                if (get_budget_timer(t).ops && \
+                        get_budget_timer(t).ops->evt != NULL) { \
+                        get_budget_timer(t).ops->evt(t); \
+                } \
+        }while(0)
+
+#define budget_state_machine2(t, evt, param) \
+        do { \
+                if (get_budget_timer(t).ops && \
+                        get_budget_timer(t).ops->evt != NULL) { \
+                        get_budget_timer(t).ops->evt(t, param); \
+                } \
+        }while(0)
+
+#define budget_state_machine_chgprio(a, b, evt) \
+        do { \
+                if (get_budget_timer(a).ops && \
+                        get_budget_timer(b).ops && \
+                        get_budget_timer(a).ops->evt != NULL && \
+                        get_budget_timer(b).ops->evt != NULL) {\
+                        get_budget_timer(a).ops->evt(a, b); \
+                } \
+        }while(0)
+
+
+void init_budget_tracker(struct budget_tracker* bt,
+                                const struct budget_tracker_ops* ops);
+
+
+/* Send SIG_BUDGET to a real-time task. */
+void send_sigbudget(struct task_struct* t);
 
 #endif
diff --git a/include/litmus/litmus.h b/include/litmus/litmus.h
index 74d9426b66d9..861ffd20df9e 100644
--- a/include/litmus/litmus.h
+++ b/include/litmus/litmus.h
@@ -68,16 +68,59 @@ void litmus_do_exit(struct task_struct *tsk);
 #define get_priority(t)         (tsk_rt(t)->task_params.priority)
 #define get_class(t)        (tsk_rt(t)->task_params.cls)
 #define get_release_policy(t) (tsk_rt(t)->task_params.release_policy)
+#define get_drain_policy(t) (tsk_rt(t)->task_params.drain_policy)
 
 /* job_param macros */
 #define get_exec_time(t)    (tsk_rt(t)->job_params.exec_time)
 #define get_deadline(t)         (tsk_rt(t)->job_params.deadline)
 #define get_release(t)          (tsk_rt(t)->job_params.release)
 #define get_lateness(t)         (tsk_rt(t)->job_params.lateness)
+#define get_backlog(t)          (tsk_rt(t)->job_params.backlog)
 
 #define effective_priority(t) ((!(tsk_rt(t)->inh_task)) ? t : tsk_rt(t)->inh_task)
 #define base_priority(t) (t)
 
+/* budget-related functions and macros */
+
+inline static int budget_exhausted(struct task_struct* t) {
+        return get_exec_time(t) >= get_exec_cost(t);
+}
+
+inline static int budget_remaining(struct task_struct* t) {
+        return (!budget_exhausted(t)) ? (get_exec_cost(t) - get_exec_time(t)) : 0;
+}
+
+#define has_backlog(t)          (get_backlog(t) != 0)
+#define get_budget_timer(t)     (tsk_rt(t)->budget)
+#define budget_enforced(t) (\
+        tsk_rt(t)->task_params.budget_policy != NO_ENFORCEMENT)
+#define budget_precisely_tracked(t) (\
+        tsk_rt(t)->task_params.budget_policy == PRECISE_ENFORCEMENT || \
+        tsk_rt(t)->task_params.budget_signal_policy == PRECISE_SIGNALS)
+#define budget_quantum_tracked(t) (\
+        tsk_rt(t)->task_params.budget_policy == QUANTUM_ENFORCEMENT || \
+        tsk_rt(t)->task_params.budget_signal_policy == QUANTUM_SIGNALS)
+#define budget_signalled(t) (\
+        tsk_rt(t)->task_params.budget_signal_policy != NO_SIGNALS)
+#define budget_precisely_signalled(t) (\
+        tsk_rt(t)->task_params.budget_policy == PRECISE_SIGNALS)
+#define bt_flag_is_set(t, flag_nr) (\
+        test_bit(flag_nr, &tsk_rt(t)->budget.flags))
+#define bt_flag_test_and_set(t, flag_nr) (\
+        test_and_set_bit(flag_nr, &tsk_rt(t)->budget.flags))
+#define bt_flag_test_and_clear(t, flag_nr) (\
+        test_and_clear_bit(flag_nr, &tsk_rt(t)->budget.flags))
+#define bt_flag_set(t, flag_nr) (\
+        set_bit(flag_nr, &tsk_rt(t)->budget.flags))
+#define bt_flag_clear(t, flag_nr) (\
+        clear_bit(flag_nr, &tsk_rt(t)->budget.flags))
+#define bt_flags_reset(t) (\
+        tsk_rt(t)->budget.flags = 0)
+//#define should_requeue_preempted_job(t)
+#define requeue_preempted_job(t) \
+        (t && !is_completed(t) && !tsk_rt(t)->dont_requeue && \
+         (!budget_exhausted(t) || !budget_enforced(t)))
+
 /* release policy macros */
 #define is_periodic(t)          (get_release_policy(t) == TASK_PERIODIC)
 #define is_sporadic(t)          (get_release_policy(t) == TASK_SPORADIC)
diff --git a/include/litmus/locking.h b/include/litmus/locking.h
index 04f4e7710a4e..a49099cc84dd 100644
--- a/include/litmus/locking.h
+++ b/include/litmus/locking.h
@@ -280,4 +280,8 @@ void init_wake_queues(void);
          !binheap_empty(&tsk_rt(t)->hp_blocked_tasks))
 #endif
 
+void set_inh_task_linkback(struct task_struct* t, struct task_struct* linkto);
+void clear_inh_task_linkback(struct task_struct* t,
+                                struct task_struct* linkedto);
+
 #endif
diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h
index efae2fbf8371..f28afc43e801 100644
--- a/include/litmus/rt_param.h
+++ b/include/litmus/rt_param.h
@@ -33,6 +33,23 @@ typedef enum {
         PRECISE_ENFORCEMENT  /* budgets are enforced with hrtimers */
 } budget_policy_t;
 
+/* budget draining policy (ignored if neither budget enforcement nor
+   signalling are used). */
+typedef enum {
+        DRAIN_SIMPLE,   /* drains while task is linked */
+        DRAIN_SIMPLE_IO, /* drains while task is linked or blocked
+                                                (not waiting for Litmus lock) */
+        DRAIN_SAWARE,   /* drains according to suspension-aware analysis */
+        DRAIN_SOBLIV    /* drains according to suspension-obliv analysis */
+} budget_drain_policy_t;
+
+/* signal policy for budget exhaustion */
+typedef enum {
+        NO_SIGNALS,             /* job receives no signals when it exhausts its budget */
+        QUANTUM_SIGNALS, /*budget signals are only sent on quantum boundaries */
+        PRECISE_SIGNALS /* budget signals are triggered with hrtimers */
+} budget_signal_policy_t;
+
 /* Release behaviors for jobs. PERIODIC and EARLY jobs
    must end by calling sys_complete_job() (or equivalent)
    to set up their next release and deadline. */
@@ -90,6 +107,8 @@ struct rt_task {
         unsigned int    priority;
         task_class_t    cls;
         budget_policy_t  budget_policy;  /* ignored by pfair */
+        budget_drain_policy_t drain_policy;
+        budget_signal_policy_t budget_signal_policy; /* ignored by pfair */
         release_policy_t release_policy;
 };
 
@@ -171,6 +190,7 @@ struct control_page {
 
 #include <linux/semaphore.h>
 #include <litmus/binheap.h>
+#include <litmus/budget.h>
 
 #ifdef CONFIG_LITMUS_SOFTIRQD
 #include <linux/interrupt.h>
@@ -263,6 +283,19 @@ struct rt_job {
          * Increase this sequence number when a job is released.
          */
         unsigned int    job_no;
+
+        /* Increments each time a job is forced to complete by budget exhaustion.
+         * If a job completes without remaining budget, the next ob will be early-
+         * released __without__ pushing back its deadline. job_backlog is
+         * decremented once per early release. This behavior continues until
+         * backlog == 0.
+         */
+        unsigned int    backlog;
+
+        /* Denotes if the current job is a backlogged job that was early released
+         * due to budget enforcement behaviors.
+         */
+        unsigned int    is_backlogged_job:1;
 };
 
 struct pfair_param;
@@ -344,6 +377,8 @@ struct rt_param {
         unsigned int            sporadic_release:1;
         lt_t                    sporadic_release_time;
 
+        /* budget tracking/enforcement method and data assigned to this task */
+        struct budget_tracker   budget;
 
         /* task representing the current "inherited" task
          * priority, assigned by inherit_priority and
@@ -353,6 +388,17 @@ struct rt_param {
          */
          struct task_struct*    inh_task;
 
+         /* budget enforcement methods may require knowledge of tasks that
+          * inherit this task's priority. There may be more than one such
+          * task w/ priority inheritance chains.
+          */
+         int    inh_task_linkback_idx;  /* idx in inh_task's
+                                                                           inh_task_linkbacks array. */
+
+         struct task_struct** inh_task_linkbacks; /* array w/ BITS_PER_LONG elm */
+         unsigned long  used_linkback_slots;  /* nr used slots
+                                                                                         in inh_task_linkbacks */
+
 #ifdef CONFIG_NP_SECTION
         /* For the FMLP under PSN-EDF, it is required to make the task
          * non-preemptive from kernel space. In order not to interfere with
@@ -426,17 +472,19 @@ struct rt_param {
 #endif /* end LITMUS_SOFTIRQD */
 
 #ifdef CONFIG_REALTIME_AUX_TASKS
+        /* Real-time data for auxiliary tasks */
+        struct list_head        aux_task_node;
+        struct binheap_node     aux_task_owner_node;
+
         unsigned int    is_aux_task:1;
         unsigned int    aux_ready:1;
         unsigned int    has_aux_tasks:1;
         unsigned int    hide_from_aux_tasks:1;
-
-        struct list_head        aux_task_node;
-        struct binheap_node     aux_task_owner_node;
 #endif
 };
 
 #ifdef CONFIG_REALTIME_AUX_TASKS
+/* Auxiliary task data. Appears in task_struct, like rt_param */
 struct aux_data {
         struct list_head        aux_tasks;
         struct binheap  aux_task_owners;
diff --git a/include/litmus/signal.h b/include/litmus/signal.h
new file mode 100644
index 000000000000..38c3207951e0
--- /dev/null
+++ b/include/litmus/signal.h
@@ -0,0 +1,47 @@
+#ifndef LITMUS_SIGNAL_H
+#define LITMUS_SIGNAL_H
+
+#ifdef __KERNEL__
+#include <linux/signal.h>
+#else
+#include <signal.h>
+#endif
+
+/* Signals used by Litmus to asynchronously communicate events
+ * to real-time tasks.
+ *
+ * Signal values overlap with [SIGRTMIN, SIGRTMAX], so beware of
+ * application-level conflicts when dealing with COTS user-level
+ * code.
+ */
+
+/* Sent to a Litmus task when all of the following conditions are true:
+ * (1) The task has exhausted its budget.
+ * (2) budget_signal_policy is QUANTUM_SIGNALS or PRECISE_SIGNALS.
+ *
+ * Note: If a task does not have a registered handler for SIG_BUDGET,
+ * the signal will cause the task to terminate (default action).
+ */
+
+/* Assigned values start at SIGRTMAX and decrease, hopefully reducing
+ * likelihood of user-level conflicts.
+ */
+#define SIG_BUDGET                                      (SIGRTMAX - 0)
+
+/*
+Future signals could include:
+
+#define SIG_DEADLINE_MISS                       (SIGRTMAX - 1)
+#define SIG_CRIT_LEVEL_CHANGE           (SIGRTMAX - 2)
+*/
+
+#define SIGLITMUSMIN                            SIG_BUDGET
+
+#ifdef __KERNEL__
+#if (SIGLITMUSMIN < SIGRTMIN)
+/* no compile-time check in user-space since SIGRTMIN may be a variable. */
+#error "Too many LITMUS^RT signals!"
+#endif
+#endif
+
+#endif
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 8fbdf5b639fa..c62ed018d994 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -92,6 +92,7 @@
 #include <litmus/sched_plugin.h>
 
 void litmus_tick(struct rq*, struct task_struct*);
+void litmus_handle_budget_exhaustion(struct task_struct*);
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/sched.h>
@@ -3107,6 +3108,11 @@ litmus_need_resched_nonpreemptible:
 
         post_schedule(rq);
 
+        if (is_realtime(current) &&
+                unlikely(budget_enforced(current) && budget_exhausted(current))) {
+                litmus_handle_budget_exhaustion(current);
+        }
+
         if (sched_state_validate_switch()) {
                 TS_SCHED2_END(prev);
                 goto litmus_need_resched_nonpreemptible;
diff --git a/kernel/sched/litmus.c b/kernel/sched/litmus.c
index 857a8c8443bd..23ce659dc62c 100644
--- a/kernel/sched/litmus.c
+++ b/kernel/sched/litmus.c
@@ -93,7 +93,7 @@ litmus_schedule(struct rq *rq, struct task_struct *prev)
                                 TRACE_TASK(next,"descheduled. Proceeding.\n");
 
                         if (lt_before(_maybe_deadlock + 1000000000L,
-                                      litmus_clock())) {
+                                          litmus_clock())) {
                                 /* We've been spinning for 1s.
                                  * Something can't be right!
                                  * Let's abandon the task and bail out; at least
@@ -138,8 +138,6 @@ litmus_schedule(struct rq *rq, struct task_struct *prev)
                         }
                 }
 
-                set_task_cpu(next, smp_processor_id());
-
                 /* DEBUG: now that we have the lock we need to make sure a
                  *  couple of things still hold:
                  *  - it is still a real-time task
@@ -147,7 +145,7 @@ litmus_schedule(struct rq *rq, struct task_struct *prev)
                  * If either is violated, then the active plugin is
                  * doing something wrong.
                  */
-                if (!is_realtime(next) || !is_running(next)) {
+                if (unlikely(!is_realtime(next) || !is_running(next))) {
                         /* BAD BAD BAD */
                         TRACE_TASK(next,"BAD: migration invariant FAILED: "
                                    "rt=%d running=%d\n",
@@ -156,6 +154,10 @@ litmus_schedule(struct rq *rq, struct task_struct *prev)
                         /* drop the task */
                         next = NULL;
                 }
+                else {
+                        set_task_cpu(next, smp_processor_id());
+                }
+
                 /* release the other CPU's runqueue, but keep ours */
                 raw_spin_unlock(&other_rq->lock);
         }
@@ -168,12 +170,71 @@ litmus_schedule(struct rq *rq, struct task_struct *prev)
                 next->rt_param.stack_in_use = 0;
 #endif
                 next->se.exec_start = rq->clock;
+
+                if (is_realtime(next)) {
+                        budget_state_machine(next, on_scheduled);
+
+#ifdef CONFIG_LITMUS_AFFINITY_AWARE_GPU_ASSINGMENT
+                        /* turn GPU tracking back on if needed */
+                        if(tsk_rt(next)->held_gpus) {
+                                if(0 == tsk_rt(next)->gpu_time_stamp) {
+                                        start_gpu_tracker(next);
+                                }
+                        }
+#endif
+                }
         }
 
-        update_enforcement_timer(next);
         return next;
 }
 
+void litmus_handle_budget_exhaustion(struct task_struct *t)
+{
+        /* We're unlikely to pick a task that has an exhausted budget, so this
+         * provides a failsafe. */
+
+        /* BUG: Virtual unlock of OMLP-family locking protocols is not triggered.
+         *
+         * TODO-FIX: Add a new virtual-unlock call to budget state machine and do
+         * the virtual unlock in plugin::schedule(), instead of in budget
+         * timer handler. This bug should only be raised EXTREMELY infrequently.
+         */
+
+        int handle_exhausion = 1;
+
+        BUG_ON(current != t);
+
+        if (is_np(t) && bt_flag_is_set(t, BTF_BUDGET_EXHAUSTED)) {
+                /* ignore. will handle exhausion in the future. */
+                TRACE_TASK(t, "Task is np and already flagged as exhausted. "
+                                        "Allow scheduling.\n");
+                return;
+        }
+
+        if (unlikely(bt_flag_is_set(t, BTF_WAITING_FOR_RELEASE))) {
+                TRACE_TASK(t, "Waiting for release. Skipping exhaustion.\n");
+                return;
+        }
+
+        if (budget_precisely_tracked(t)) {
+                if (cancel_enforcement_timer(t) < 0) {
+                        TRACE_TASK(t, "schedule() raced with timer. Deferring to timer.\n");
+                        handle_exhausion = 0;
+                }
+        }
+
+        if (handle_exhausion) {
+                if (likely(!is_np(t))) {
+                        TRACE_TASK(t, "picked task without budget => FORCE_RESCHED.\n");
+                        litmus_reschedule_local();
+                }
+                else if (is_user_np(t)) {
+                        TRACE_TASK(t, "is non-preemptable, preemption delayed.\n");
+                        request_exit_np(t);
+                }
+        }
+}
+
 static void enqueue_task_litmus(struct rq *rq, struct task_struct *p,
                                 int flags)
 {
@@ -187,7 +248,7 @@ static void enqueue_task_litmus(struct rq *rq, struct task_struct *p,
                  * state already here.
                  *
                  * WARNING: this needs to be re-evaluated when porting
-                 *          to newer kernel versions.
+                 *                to newer kernel versions.
                  */
                 p->state = TASK_RUNNING;
                 litmus->task_wake_up(p);
@@ -340,7 +401,7 @@ const struct sched_class litmus_sched_class = {
         .pre_schedule           = pre_schedule_litmus,
 #endif
 
-        .set_curr_task          = set_curr_task_litmus,
+        .set_curr_task           = set_curr_task_litmus,
         .task_tick              = task_tick_litmus,
 
         .get_rr_interval        = get_rr_interval_litmus,
diff --git a/litmus/Makefile b/litmus/Makefile
index a312ef4dd52a..bb28045a5221 100644
--- a/litmus/Makefile
+++ b/litmus/Makefile
@@ -6,11 +6,12 @@ obj-y     = sched_plugin.o litmus.o \
             preempt.o \
             litmus_proc.o \
             budget.o \
-                aux_tasks.o \
+            aux_tasks.o \
             clustered.o \
             jobs.o \
             sync.o \
             rt_domain.o \
+            budget.o \
             edf_common.o \
             fp_common.o \
             fdso.o \
diff --git a/litmus/aux_tasks.c b/litmus/aux_tasks.c
index 5b34b9b99941..a5d44775b41d 100644
--- a/litmus/aux_tasks.c
+++ b/litmus/aux_tasks.c
@@ -35,10 +35,8 @@ static int admit_aux_task(struct task_struct *t)
                 .priority = LITMUS_LOWEST_PRIORITY,
                 .cls = RT_CLASS_BEST_EFFORT,
                 .budget_policy = QUANTUM_ENFORCEMENT,
-#if 0 /* PORT RECHECK */
                 .drain_policy = DRAIN_SIMPLE,
                 .budget_signal_policy = NO_SIGNALS,
-#endif
                 /* use SPORADIC instead of EARLY since util = 1.0 */
                 .release_policy = TASK_SPORADIC,
         };
diff --git a/litmus/budget.c b/litmus/budget.c
index f7712be29adb..9e36ce34e417 100644
--- a/litmus/budget.c
+++ b/litmus/budget.c
@@ -1,113 +1,459 @@
 #include <linux/sched.h>
 #include <linux/percpu.h>
 #include <linux/hrtimer.h>
+#include <linux/signal.h>
 
 #include <litmus/litmus.h>
 #include <litmus/preempt.h>
-
+#include <litmus/sched_plugin.h>
 #include <litmus/budget.h>
+#include <litmus/signal.h>
+
+int cancel_enforcement_timer(struct task_struct* t)
+{
+        struct enforcement_timer* et;
+        int ret = 0;
+        unsigned long flags;
+
+        BUG_ON(!t);
+        BUG_ON(!is_realtime(t));
+
+        et = &tsk_rt(t)->budget.timer;
 
-struct enforcement_timer {
-        /* The enforcement timer is used to accurately police
-         * slice budgets. */
-        struct hrtimer          timer;
-        int                     armed;
-};
+        TRACE_TASK(t, "canceling enforcement timer.\n");
 
-DEFINE_PER_CPU(struct enforcement_timer, budget_timer);
+        if (et->armed) {
+                raw_spin_lock_irqsave(&et->lock, flags);
+                if (et->armed) {
+                        ret = hrtimer_try_to_cancel(&et->timer);
+                        if (ret < 0)
+                                TRACE_TASK(t, "timer already running. failed to cancel.\n");
+                        else {
+                                TRACE_TASK(t, "canceled timer with %lld ns remaining.\n",
+                                        ktime_to_ns(hrtimer_expires_remaining(&et->timer)));
+                                et->armed = 0;
+                        }
+                }
+                else
+                        TRACE_TASK(t, "timer was not armed (race).\n");
+                raw_spin_unlock_irqrestore(&et->lock, flags);
+        }
+        else
+                TRACE_TASK(t, "timer was not armed.\n");
+
+        return ret;
+}
 
-static enum hrtimer_restart on_enforcement_timeout(struct hrtimer *timer)
+inline static void arm_enforcement_timer(struct task_struct* t, int force)
 {
-        struct enforcement_timer* et = container_of(timer,
-                                                    struct enforcement_timer,
-                                                    timer);
+        struct enforcement_timer* et;
+        lt_t when_to_fire, remaining_budget;
+        lt_t now;
         unsigned long flags;
 
-        local_irq_save(flags);
-        TRACE("enforcement timer fired.\n");
-        et->armed = 0;
-        /* activate scheduler */
-        litmus_reschedule_local();
-        local_irq_restore(flags);
+        BUG_ON(!t);
+        BUG_ON(!is_realtime(t));
+
+        et = &tsk_rt(t)->budget.timer;
+        if (et->armed) {
+                TRACE_TASK(t, "timer already armed!\n");
+                return;
+        }
+
+        if (!force) {
+                if ( (!budget_enforced(t) ||
+                                (budget_enforced(t) &&
+                                        bt_flag_is_set(t, BTF_BUDGET_EXHAUSTED)))
+                                &&
+                        (!budget_signalled(t) ||
+                                (budget_signalled(t) &&
+                                        bt_flag_is_set(t, BTF_SIG_BUDGET_SENT)))) {
+                        TRACE_TASK(t,
+                                        "trying to arm timer when budget "
+                                        "has already been exhausted.\n");
+                        return;
+                }
+        }
+
+        TRACE_TASK(t, "arming enforcement timer.\n");
+
+        /* __hrtimer_start_range_ns() cancels the timer
+         * anyway, so we don't have to check whether it is still armed */
+        raw_spin_lock_irqsave(&et->lock, flags);
+
+        if (et->armed) {
+                TRACE_TASK(t, "timer already armed (race)!\n");
+                goto out;
+        }
+
+        now = litmus_clock();
+        remaining_budget = budget_remaining(t);
+        when_to_fire = now + remaining_budget;
+
+        TRACE_TASK(t, "budget remaining: %ld, when_to_fire: %ld\n",
+                                        remaining_budget, when_to_fire);
 
-        return  HRTIMER_NORESTART;
+        __hrtimer_start_range_ns(&et->timer,
+                                 ns_to_ktime(when_to_fire),
+                                 0 /* delta */,
+                                 HRTIMER_MODE_ABS_PINNED,  /* TODO: need to use non-pinned? */
+                                 0 /* no wakeup */);
+        et->armed = 1;
+
+out:
+        raw_spin_unlock_irqrestore(&et->lock, flags);
 }
 
-/* assumes called with IRQs off */
-static void cancel_enforcement_timer(struct enforcement_timer* et)
+void send_sigbudget(struct task_struct* t)
 {
-        int ret;
+        if (!bt_flag_test_and_set(t, BTF_SIG_BUDGET_SENT)) {
+                /* signal has not yet been sent and we are responsible for sending
+                 * since we just set the sent-bit when it was previously 0. */
 
-        TRACE("cancelling enforcement timer.\n");
+                TRACE_TASK(t, "SIG_BUDGET being sent!\n");
+                send_sig(SIG_BUDGET, t, 1); /* '1' denotes signal sent from kernel */
+        }
+}
 
-        /* Since interrupts are disabled and et->armed is only
-         * modified locally, we do not need any locks.
-         */
+/*
+ * DRAIN_SIMPLE
+ */
 
-        if (et->armed) {
-                ret = hrtimer_try_to_cancel(&et->timer);
-                /* Should never be inactive. */
-                BUG_ON(ret == 0);
-                /* Should never be running concurrently. */
-                BUG_ON(ret == -1);
+void simple_on_scheduled(struct task_struct* t)
+{
+        BUG_ON(!t);
+
+        if(budget_precisely_tracked(t) && !bt_flag_is_set(t, BTF_SIG_BUDGET_SENT))
+                if (!tsk_rt(t)->budget.timer.armed)
+                        arm_enforcement_timer(t, 0);
+}
+
+inline static void __simple_on_unscheduled(struct task_struct* t)
+{
+        BUG_ON(!t);
+
+        if (budget_precisely_tracked(t))
+                cancel_enforcement_timer(t);
+}
+
+void simple_on_blocked(struct task_struct* t)
+{
+        __simple_on_unscheduled(t);
+}
+
+void simple_on_preempt(struct task_struct* t)
+{
+        __simple_on_unscheduled(t);
+}
+
+void simple_on_sleep(struct task_struct* t)
+{
+        __simple_on_unscheduled(t);
+}
+
+void simple_on_exit(struct task_struct* t)
+{
+        __simple_on_unscheduled(t);
+}
+
+/*
+ * DRAIN_SIMPLE_IO
+ */
+
+void simple_io_on_blocked(struct task_struct* t)
+{
+        /* hiding is turned on by locking protocols, so if there isn't any
+           hiding, then we're blocking for some other reason.  assume it's I/O. */
+
+        int for_io = !tsk_rt(t)->blocked_lock || (0
+#ifdef CONFIG_REALTIME_AUX_TASKS
+                || (tsk_rt(t)->has_aux_tasks && !tsk_rt(t)->hide_from_aux_tasks)
+#endif
+#ifdef CONFIG_LITMUS_NVIDIA
+                || (tsk_rt(t)->held_gpus && !tsk_rt(t)->hide_from_gpu)
+#endif
+                );
 
-                et->armed = 0;
+        /* we drain budget for io-based suspensions */
+        if (for_io) {
+                /* there is a fraction of time where we're double-counting the
+                 * time tracked by the rq and suspension time.
+                 * TODO: Do this recording closer to suspension time. */
+                tsk_rt(t)->budget.suspend_timestamp = litmus_clock();
+
+                TRACE_TASK(t, "blocking for I/O.\n");
+
+                if (!tsk_rt(t)->budget.timer.armed) {
+                        bt_flag_clear(t, BTF_BUDGET_EXHAUSTED);
+
+                        if (likely(!bt_flag_is_set(t, BTF_WAITING_FOR_RELEASE))) {
+                                TRACE_TASK(t, "budget timer not armed. "
+                                                   "Raced with exhaustion-resched? Re-arming.\n");
+                                arm_enforcement_timer(t, 1);
+                        }
+                        else {
+                                TRACE_TASK(t, "not arming timer because task is waiting "
+                                                   "for release.\n");
+                        }
+                }
+        }
+        else {
+                TRACE_TASK(t, "blocking for litmus lock. stop draining.\n");
+                simple_on_blocked(t);
         }
 }
 
-/* assumes called with IRQs off */
-static void arm_enforcement_timer(struct enforcement_timer* et,
-                                  struct task_struct* t)
+void simple_io_on_wakeup(struct task_struct* t)
 {
-        lt_t when_to_fire;
-        TRACE_TASK(t, "arming enforcement timer.\n");
+        /* we're waking up from an io-based suspension */
+        if (tsk_rt(t)->budget.suspend_timestamp) {
+                lt_t suspend_cost = litmus_clock() -
+                                tsk_rt(t)->budget.suspend_timestamp;
+                tsk_rt(t)->budget.suspend_timestamp = 0;
+                TRACE_TASK(t, "budget consumed while io-suspended: %llu\n",
+                                                suspend_cost);
+                get_exec_time(t) += suspend_cost;
+        }
+        else {
+                TRACE_TASK(t, "waking from non-io blocking\n");
+        }
+}
 
-        /* Calling this when there is no budget left for the task
-         * makes no sense, unless the task is non-preemptive. */
-        BUG_ON(budget_exhausted(t) && (!is_np(t)));
 
-        /* __hrtimer_start_range_ns() cancels the timer
-         * anyway, so we don't have to check whether it is still armed */
+/*
+ * DRAIN_SOBLIV
+ */
+
+void sobliv_on_blocked(struct task_struct* t)
+{
+        if (bt_flag_is_set(t, BTF_IS_TOP_M)) {
+                /* there is a fraction of time where we're double-counting the
+                 * time tracked by the rq and suspension time.
+                 * TODO: Do this recording closer to suspension time. */
+                tsk_rt(t)->budget.suspend_timestamp = litmus_clock();
+
+                if (!tsk_rt(t)->budget.timer.armed) {
+                        /* budget exhaustion timer fired as t was waking up, so budget
+                         * routine thought t was running. We need to re-trigger the budget
+                         * exhastion routine via timer. Schedulers do not call
+                         * job_completion() when a task blocks, even if t's budget has been
+                         * exhausted. Unfortunately, we cannot rerun the exhaustion routine
+                         * here due to spinlock ordering issues. Just re-arm the timer with
+                         * the exhausted time, re-running the timer routine immediately once
+                         * interrupts have been re-enabled. */
+
+                        /* clear the exhausted flag so handle will re-run. this will not
+                         * trigger another exhaustion signal since signals are controled by
+                         * BTF_SIG_BUDGET_SENT. */
+                        bt_flag_clear(t, BTF_BUDGET_EXHAUSTED);
 
-        if (likely(!is_np(t))) {
-                when_to_fire = litmus_clock() + budget_remaining(t);
-                __hrtimer_start_range_ns(&et->timer,
-                                         ns_to_ktime(when_to_fire),
-                                         0 /* delta */,
-                                         HRTIMER_MODE_ABS_PINNED,
-                                         0 /* no wakeup */);
-                et->armed = 1;
+                        if (likely(!bt_flag_is_set(t, BTF_WAITING_FOR_RELEASE))) {
+                                TRACE_TASK(t, "budget timer not armed. "
+                                                   "Raced with exhaustion-resched? Re-arming.\n");
+                                arm_enforcement_timer(t, 1);
+                        }
+                        else {
+                                TRACE_TASK(t, "not arming timer because task is waiting "
+                                                                "for release.\n");
+                        }
+                }
         }
 }
 
+void sobliv_on_wakeup(struct task_struct* t)
+{
+        if (bt_flag_is_set(t, BTF_IS_TOP_M)) {
+                /* we're waking up while in top-m.  record the time spent
+                 * suspended while draining in exec_cost. suspend_timestamp was
+                 * either set when we entered top-m while asleep, or when we
+                 * blocked. */
+                if (tsk_rt(t)->budget.suspend_timestamp) {
+                        lt_t suspend_cost = litmus_clock() -
+                                        tsk_rt(t)->budget.suspend_timestamp;
+                        tsk_rt(t)->budget.suspend_timestamp = 0;
+                        TRACE_TASK(t, "budget consumed while suspended: %llu\n",
+                                        suspend_cost);
+                        get_exec_time(t) += suspend_cost;
+                }
+                else {
+                        WARN_ON(!bt_flag_is_set(t, BTF_WAITING_FOR_RELEASE));
+                }
+        }
+}
 
-/* expects to be called with IRQs off */
-void update_enforcement_timer(struct task_struct* t)
+void sobliv_on_inherit(struct task_struct* t, struct task_struct* prio_inh)
 {
-        struct enforcement_timer* et = &__get_cpu_var(budget_timer);
+        /* TODO: Budget credit accounting. */
+        BUG_ON(!prio_inh);
+}
+
+void sobliv_on_disinherit(struct task_struct* t, struct task_struct* prio_inh)
+{
+        /* TODO: Budget credit accounting. */
+}
 
-        if (t && budget_precisely_enforced(t)) {
-                /* Make sure we call into the scheduler when this budget
-                 * expires. */
-                arm_enforcement_timer(et, t);
-        } else if (et->armed) {
-                /* Make sure we don't cause unnecessary interrupts. */
-                cancel_enforcement_timer(et);
+void sobliv_on_enter_top_m(struct task_struct* t)
+{
+        if (!bt_flag_is_set(t, BTF_SIG_BUDGET_SENT)) {
+                if (tsk_rt(t)->budget.timer.armed)
+                        TRACE_TASK(t, "budget timer already armed.\n");
+                else {
+                        /* if we're blocked, then record the time at which we
+                           started measuring */
+                        if (!is_running(t))
+                                tsk_rt(t)->budget.suspend_timestamp = litmus_clock();
+
+                        /* the callback will handle it if it is executing */
+                        if (!hrtimer_callback_running(&tsk_rt(t)->budget.timer.timer)) {
+                                arm_enforcement_timer(t, 0);
+                        }
+                        else {
+                                TRACE_TASK(t,
+                                        "within callback context. deferring timer arm.\n");
+                        }
+                }
         }
 }
 
+void sobliv_on_exit_top_m(struct task_struct* t)
+{
+        if (budget_precisely_tracked(t)) {
+                if (tsk_rt(t)->budget.timer.armed) {
+
+                        if (!is_running(t)) {
+                                /* the time at which we started draining budget while
+                                 * suspended is recorded in evt_timestamp.  evt_timestamp
+                                 * was set either when 't' exited the top-m while suspended
+                                 * or when 't' blocked. */
+                                lt_t suspend_cost;
+                                BUG_ON(!tsk_rt(t)->budget.suspend_timestamp);
+                                suspend_cost = litmus_clock() -
+                                                tsk_rt(t)->budget.suspend_timestamp;
+                                TRACE_TASK(t, "budget consumed while suspended: %llu\n",
+                                                                suspend_cost);
+                                get_exec_time(t) += suspend_cost;
+
+                                /* timer should have fired before now */
+                                if (get_exec_time(t) + 1000000/10 > get_exec_cost(t)) {
+                                        TRACE_TASK(t,
+                                                "budget overrun while suspended by over 1/10 "
+                                                "millisecond! timer should have already fired!\n");
+                                        WARN_ON(1);
+                                }
+                        }
 
-static int __init init_budget_enforcement(void)
+                        TRACE_TASK(t, "stops draining budget\n");
+                        /* the callback will handle it if it is executing */
+                        if (!hrtimer_callback_running(&tsk_rt(t)->budget.timer.timer)) {
+                                /* TODO: record a timestamp if the task isn't running */
+                                cancel_enforcement_timer(t);
+                        }
+                        else {
+                                TRACE_TASK(t,
+                                        "within callback context. skipping operation.\n");
+                        }
+                }
+                else {
+                        TRACE_TASK(t, "was not draining budget\n");
+                }
+        }
+}
+
+void reevaluate_inheritance(struct task_struct* t)
 {
-        int cpu;
-        struct enforcement_timer* et;
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+        struct litmus_lock *blocked_lock = NULL;
+
+        TRACE_TASK(t, "reevaluating locks in light of budget exhaustion.\n");
+
+        /* do we need to inherit from any tasks now that our own
+         * priority has decreased? */
+        raw_spin_lock(&tsk_rt(t)->hp_blocked_tasks_lock);
+        if (holds_locks(t)) {
+                struct task_struct* hp_blocked =
+                                top_priority(&tsk_rt(t)->hp_blocked_tasks);
+
+                if (litmus->compare(hp_blocked, t))
+                        litmus->increase_prio(t, effective_priority(hp_blocked));
+        }
+        raw_spin_unlock(&tsk_rt(t)->hp_blocked_tasks_lock);
+
+        /* do we need to tell the lock we're blocked on about our
+         * changed priority? */
+        blocked_lock = tsk_rt(t)->blocked_lock;
+        if(blocked_lock) {
+                if(blocked_lock->ops->supports_budget_exhaustion) {
+                        TRACE_TASK(t, "Lock %d supports budget exhaustion.\n",
+                                           blocked_lock->ident);
+                        blocked_lock->ops->budget_exhausted(blocked_lock, t);
+                }
+        }
+        else {
+                TRACE_TASK(t,
+                        "Budget exhausted while task not blocked on Litmus lock.\n");
+        }
+#else
+        /* prio-reeval currently relies upon nested locking infrastructure */
+        TRACE_TASK(t,
+                "Unable to check if sleeping task is blocked "
+                "on Litmus lock without "
+                "CONFIG_LITMUS_NESTED_LOCKING enabled.\n");
+#endif
+}
+
+
 
-        for (cpu = 0; cpu < NR_CPUS; cpu++)  {
-                et = &per_cpu(budget_timer, cpu);
-                hrtimer_init(&et->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
-                et->timer.function = on_enforcement_timeout;
+static enum hrtimer_restart __on_timeout(struct hrtimer *timer)
+{
+        enum hrtimer_restart restart = HRTIMER_NORESTART;
+        unsigned long flags;
+
+        struct budget_tracker* bt =
+                container_of(
+                        container_of(timer,
+                                struct enforcement_timer,
+                                timer),
+                        struct budget_tracker,
+                        timer);
+
+        struct task_struct* t =
+                container_of(
+                        container_of(bt, struct rt_param, budget),
+                        struct task_struct,
+                        rt_param);
+
+        TRACE_TASK(t, "budget timer interrupt fired at time %lu\n",
+                                        litmus_clock());
+
+        raw_spin_lock_irqsave(&bt->timer.lock, flags);
+        tsk_rt(t)->budget.timer.armed = 0;
+        raw_spin_unlock_irqrestore(&bt->timer.lock, flags);
+
+        if (unlikely(bt_flag_is_set(t, BTF_WAITING_FOR_RELEASE))) {
+                TRACE_TASK(t,
+                        "spurious exhastion while waiting for release. dropping.\n");
+                goto out;
         }
-        return 0;
+
+        restart = bt->ops->on_exhausted(t,!IN_SCHEDULE);
+
+        raw_spin_lock_irqsave(&bt->timer.lock, flags);
+        tsk_rt(t)->budget.timer.armed = (restart == HRTIMER_RESTART);
+        raw_spin_unlock_irqrestore(&bt->timer.lock, flags);
+
+out:
+        return restart;
 }
 
-module_init(init_budget_enforcement);
+void init_budget_tracker(struct budget_tracker* bt,
+                                const struct budget_tracker_ops* ops)
+{
+        BUG_ON(!bt);
+
+        memset(bt, 0, sizeof(*bt));
+        raw_spin_lock_init(&bt->timer.lock);
+        hrtimer_init(&bt->timer.timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
+        bt->timer.timer.function = __on_timeout;
+        bt->ops = ops;
+        INIT_BINHEAP_NODE(&bt->top_m_node);
+}
diff --git a/litmus/jobs.c b/litmus/jobs.c
index ce7d591b82c5..d955bbde8d57 100644
--- a/litmus/jobs.c
+++ b/litmus/jobs.c
@@ -13,11 +13,9 @@ void setup_release(struct task_struct *t, lt_t release)
         t->rt_param.job_params.deadline = release + get_rt_relative_deadline(t);
         t->rt_param.job_params.exec_time = 0;
 
-#if 0 /* PORT CHECK */
-        /* kludge - TODO: Move this to budget.h/.c */
+        /* TODO: Move this to budget.h/.c */
         if (t->rt_param.budget.ops)
                 bt_flags_reset(t);
-#endif
 
         /* update job sequence number */
         t->rt_param.job_params.job_no++;
diff --git a/litmus/klmirqd.c b/litmus/klmirqd.c
index c1f4aef97063..b16e9c271118 100644
--- a/litmus/klmirqd.c
+++ b/litmus/klmirqd.c
@@ -286,10 +286,8 @@ static int become_litmus_daemon(struct task_struct* tsk)
                 .priority = LITMUS_LOWEST_PRIORITY,
                 .cls = RT_CLASS_BEST_EFFORT,
                 .budget_policy = NO_ENFORCEMENT,
-#if 0 /* PORT RECHECK */
                 .drain_policy = DRAIN_SIMPLE,
                 .budget_signal_policy = NO_SIGNALS,
-#endif
                 /* use SPORADIC instead of EARLY since util = 1.0 */
                 .release_policy = TASK_SPORADIC,
         };
diff --git a/litmus/locking.c b/litmus/locking.c
index a2f38900e02f..0de007cc9732 100644
--- a/litmus/locking.c
+++ b/litmus/locking.c
@@ -956,6 +956,77 @@ int flush_pending_wakes()
         return count;
 }
 
+void set_inh_task_linkback(struct task_struct* t, struct task_struct* linkto)
+{
+    const int MAX_IDX = BITS_PER_LONG - 1;
+
+    int success = 0;
+    int old_idx = tsk_rt(t)->inh_task_linkback_idx;
+
+    /* is the linkback already set? */
+    if (old_idx >= 0 && old_idx <= MAX_IDX) {
+        if ((BIT_MASK(old_idx) & tsk_rt(linkto)->used_linkback_slots) &&
+            (tsk_rt(linkto)->inh_task_linkbacks[old_idx] == t)) {
+            TRACE_TASK(t, "linkback is current.\n");
+            return;
+        }
+        BUG();
+    }
+
+    /* kludge: upper limit on num linkbacks */
+    BUG_ON(tsk_rt(linkto)->used_linkback_slots == ~0ul);
+
+    while(!success) {
+        int b = find_first_zero_bit(&tsk_rt(linkto)->used_linkback_slots,
+                    BITS_PER_BYTE*sizeof(tsk_rt(linkto)->used_linkback_slots));
+
+        BUG_ON(b > MAX_IDX);
+
+        /* set bit... */
+        if (!test_and_set_bit(b, &tsk_rt(linkto)->used_linkback_slots)) {
+            TRACE_TASK(t, "linking back to %s/%d in slot %d\n",
+                                                        linkto->comm, linkto->pid, b);
+            if (tsk_rt(linkto)->inh_task_linkbacks[b])
+                TRACE_TASK(t, "%s/%d already has %s/%d in slot %d\n",
+                           linkto->comm, linkto->pid,
+                           tsk_rt(linkto)->inh_task_linkbacks[b]->comm,
+                           tsk_rt(linkto)->inh_task_linkbacks[b]->pid,
+                           b);
+
+            /* TODO: allow dirty data to remain in [b] after code is tested */
+            BUG_ON(tsk_rt(linkto)->inh_task_linkbacks[b] != NULL);
+            /* ...before setting slot */
+            tsk_rt(linkto)->inh_task_linkbacks[b] = t;
+            tsk_rt(t)->inh_task_linkback_idx = b;
+            success = 1;
+        }
+    }
+}
+
+void clear_inh_task_linkback(struct task_struct* t,
+                                struct task_struct* linkedto)
+{
+    const int MAX_IDX = BITS_PER_LONG - 1;
+
+    int success = 0;
+    int slot = tsk_rt(t)->inh_task_linkback_idx;
+
+    if (slot < 0) {
+        TRACE_TASK(t, "assuming linkback already cleared.\n");
+        return;
+    }
+
+    BUG_ON(slot > MAX_IDX);
+    BUG_ON(tsk_rt(linkedto)->inh_task_linkbacks[slot] != t);
+
+    /* be safe - clear slot before clearing the bit */
+    tsk_rt(t)->inh_task_linkback_idx = -1;
+    tsk_rt(linkedto)->inh_task_linkbacks[slot] = NULL;
+
+    success = test_and_clear_bit(slot, &tsk_rt(linkedto)->used_linkback_slots);
+
+    BUG_ON(!success);
+}
 
 #else  /* CONFIG_LITMUS_LOCKING */
 
diff --git a/litmus/sched_task_trace.c b/litmus/sched_task_trace.c
index 574957fdc1c0..fc2f696b26ef 100644
--- a/litmus/sched_task_trace.c
+++ b/litmus/sched_task_trace.c
@@ -191,10 +191,10 @@ feather_callback void do_sched_trace_task_completion(unsigned long id,
         struct st_event_record* rec = get_record(ST_COMPLETION, t);
         if (rec) {
                 rec->data.completion.when   = now();
-#if 0
-                rec->data.completion.backlog_remaining = tsk_rt(t)->job_params.backlog;
-                rec->data.completion.was_backlog_job = tsk_rt(t)->job_params.is_backlogged_job;
-#endif
+                rec->data.completion.backlog_remaining =
+                                tsk_rt(t)->job_params.backlog;
+                rec->data.completion.was_backlog_job =
+                                tsk_rt(t)->job_params.is_backlogged_job;
                 rec->data.completion.forced = forced;
                 put_record(rec);
         }
@@ -208,8 +208,9 @@ feather_callback void do_sched_trace_task_block(unsigned long id,
         if (rec) {
                 rec->data.block.when      = now();
 
-                // hiding is turned on by locking protocols, so if there isn't any
-                // hiding, then we're blocking for some other reason.  assume it's I/O.
+                /* Hiding is turned on by locking protocols, so if there isn't any
+                   hiding, then we're blocking for some other reason: assume
+                   it's I/O. */
                 rec->data.block.for_io  = !tsk_rt(t)->blocked_lock || (0
 #ifdef CONFIG_REALTIME_AUX_TASKS
                         || (tsk_rt(t)->has_aux_tasks && !tsk_rt(t)->hide_from_aux_tasks)
diff --git a/litmus/sync.c b/litmus/sync.c
index 681a65d836c6..08a71361a995 100644
--- a/litmus/sync.c
+++ b/litmus/sync.c
@@ -13,9 +13,7 @@
 #include <litmus/jobs.h>
 
 #include <litmus/sched_trace.h>
-#if 0 /* PORT RECHECK */
 #include <litmus/budget.h>
-#endif
 
 struct ts_release_wait {
         struct list_head list;
@@ -58,19 +56,15 @@ static long do_wait_for_ts_release(struct timespec *wake)
 #if defined(CONFIG_REALTIME_AUX_TASKS) || defined(CONFIG_LITMUS_NVIDIA)
                 hide_from_workers(t, &vis_flags);
 #endif
-#if 0 /* PORT RECHECK */
                 bt_flag_set(t, BTF_WAITING_FOR_RELEASE);
                 mb();
-                budget_state_machine(t,on_exit); // do this here and not in schedule()?
-#endif
+                budget_state_machine(t, on_exit); /* TODO: maybe call in schedule() */
         }
 
         TRACE_TASK(t, "waiting for ts release.\n");
 
-#if 0 /* PORT RECHECK */
         if (is_rt)
                 BUG_ON(!bt_flag_is_set(t, BTF_WAITING_FOR_RELEASE));
-#endif
 
         /* We are enqueued, now we wait for someone to wake us up. */
         ret = wait_for_completion_interruptible(&wait.completion);
@@ -78,9 +72,7 @@ static long do_wait_for_ts_release(struct timespec *wake)
         TRACE_TASK(t, "released by ts release!\n");
 
         if (is_rt) {
-#if 0 /* PORT RECHECK */
                 bt_flag_clear(t, BTF_WAITING_FOR_RELEASE);
-#endif
 #if defined(CONFIG_REALTIME_AUX_TASKS) || defined(CONFIG_LITMUS_NVIDIA)
                 show_to_workers(t, &vis_flags);
 #endif
@@ -102,13 +94,11 @@ static long do_wait_for_ts_release(struct timespec *wake)
                          */
                         tsk_rt(current)->dont_requeue = 1;
                         tsk_rt(t)->completed = 1;
-#if 0 /* PORT RECHECK */
                         tsk_rt(t)->job_params.backlog = 0;
                         tsk_rt(t)->job_params.is_backlogged_job = 0;
                         tsk_rt(t)->budget.suspend_timestamp = 0;
                         bt_flag_clear(t, BTF_BUDGET_EXHAUSTED);
                         mb();
-#endif
 
                         /* completion succeeded, set up release. subtract off
                          * period because schedule()->job_completion() will