Moved user-space state of tasks into new rt_param.user_job.

Both sched_color and sched_mc assumed seperate kernel and userspace views of job
states, where the kernel view is used for scheduling while the userspace view
is used for statistics (tardiness etc). This commit merges both approaches.

21 files changed, 164 insertions, 193 deletions

diff --git a/include/litmus/budget.h b/include/litmus/budget.h
index d1c73f5cf73e..79fec2257ad2 100644
--- a/include/litmus/budget.h
+++ b/include/litmus/budget.h
@@ -26,8 +26,7 @@ void cancel_enforcement_timer(struct enforcement_timer* et);
  * itself. This happens when budget enforcement has caused a task to be
  * booted off until the next period.
  */
-#define behind_server(t)\
-        (lt_before((t)->rt_param.job_params.real_release, get_release(t)))
+
 
 /**
  * server_release() - Prepare the task server parameters for the next period.
diff --git a/include/litmus/ce_domain.h b/include/litmus/ce_domain.h
index 5d5fdf7d1efc..a088547e95d3 100644
--- a/include/litmus/ce_domain.h
+++ b/include/litmus/ce_domain.h
@@ -19,7 +19,6 @@ void ce_domain_init(domain_t*,
                 requeue_t,
                 peek_ready_t,
                 take_ready_t,
-                preempt_needed_t,
                 task_prio_t,
                 struct ce_dom_data*,
                 const int,
diff --git a/include/litmus/debug_trace.h b/include/litmus/debug_trace.h
index 48d086d5a44c..5ca88dec31a5 100644
--- a/include/litmus/debug_trace.h
+++ b/include/litmus/debug_trace.h
@@ -29,7 +29,7 @@ extern atomic_t __log_seq_no;
 
 #define TRACE_TASK(t, fmt, args...)                     \
         TRACE("(%s/%d:%d) " fmt, (t)->comm, (t)->pid,   \
-              (t)->rt_param.job_params.job_no,  ##args)
+              get_user_job(t),  ##args)
 
 #define TRACE_CUR(fmt, args...) \
         TRACE_TASK(current, fmt, ## args)
diff --git a/include/litmus/domain.h b/include/litmus/domain.h
index d16ed1872a52..eed7c485004e 100644
--- a/include/litmus/domain.h
+++ b/include/litmus/domain.h
@@ -13,7 +13,6 @@ typedef void (*requeue_t)(struct domain*, struct task_struct*);
 typedef void (*remove_t)(struct domain*, struct task_struct*);
 typedef struct task_struct* (*peek_ready_t)(struct domain*);
 typedef struct task_struct* (*take_ready_t)(struct domain*);
-typedef int (*preempt_needed_t)(struct domain*, struct task_struct*);
 typedef int (*task_prio_t)(struct task_struct*, struct task_struct*);
 
 typedef struct domain {
@@ -30,10 +29,6 @@ typedef struct domain {
         peek_ready_t            peek_ready;
         /* remove and return next ready task */
         take_ready_t            take_ready;
-        /* return true if the domain has a task which should preempt the
-         * task given
-         */
-        preempt_needed_t        preempt_needed;
         /* for tasks within this domain, returns true if the first has
          * has a higher priority than the second
          */
@@ -45,6 +40,5 @@ void domain_init(domain_t *dom,
                  requeue_t requeue,
                  peek_ready_t peek_ready,
                  take_ready_t take_ready,
-                 preempt_needed_t preempt_needed,
                  task_prio_t priority);
 #endif
diff --git a/include/litmus/jobs.h b/include/litmus/jobs.h
index 9bd361ef3943..b63d3e695b69 100644
--- a/include/litmus/jobs.h
+++ b/include/litmus/jobs.h
@@ -4,6 +4,7 @@
 void prepare_for_next_period(struct task_struct *t);
 void release_at(struct task_struct *t, lt_t start);
 long complete_job(void);
+void setup_user_release(struct task_struct *t, lt_t release);
 
 #endif
 
diff --git a/include/litmus/litmus.h b/include/litmus/litmus.h
index c3b91fe8115c..2cd68ba3b752 100644
--- a/include/litmus/litmus.h
+++ b/include/litmus/litmus.h
@@ -44,7 +44,9 @@ void litmus_exit_task(struct task_struct *tsk);
 
 #define tsk_rt(t)               (&(t)->rt_param)
 
-#define get_server_job(t) (tsk_rt(t)->job_params.fake_job_no)
+#define get_user_job(t)         (tsk_rt(t)->user_job.job_no)
+#define get_user_release(t)     (tsk_rt(t)->user_job.release)
+#define get_user_deadline(t)    (tsk_rt(t)->user_job.deadline)
 
 /*      Realtime utility macros */
 #define get_rt_flags(t)         (tsk_rt(t)->flags)
@@ -73,6 +75,9 @@ void litmus_exit_task(struct task_struct *tsk);
 #define is_server(t)            (tsk_rt(t)->is_server)
 #define get_task_server(task)      (tsk_rt(task)->server)
 
+#define userspace_behind(t)\
+        (lt_before(get_user_release(t), get_release(t)))
+
 #define is_priority_boosted(t)  (tsk_rt(t)->priority_boosted)
 #define get_boost_start(t)      (tsk_rt(t)->boost_start_time)
 #define get_lateness(t)         (tsk_rt(t)->job_params.lateness)
diff --git a/include/litmus/rt_domain.h b/include/litmus/rt_domain.h
index 03826228dbd9..cba96e25f39d 100644
--- a/include/litmus/rt_domain.h
+++ b/include/litmus/rt_domain.h
@@ -97,7 +97,6 @@ void pd_domain_init(domain_t *dom,
                     bheap_prio_t order,
                     check_resched_needed_t check,
                     release_jobs_t release,
-                    preempt_needed_t preempt_needed,
                     task_prio_t priority);
 
 void __add_ready(rt_domain_t* rt, struct task_struct *new);
diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h
index a8c82eed5562..d63821d9467a 100644
--- a/include/litmus/rt_param.h
+++ b/include/litmus/rt_param.h
@@ -199,8 +199,10 @@ struct rt_param {
         /* user controlled parameters */
         struct rt_task          task_params;
 
-        /* timing parameters */
-        struct rt_job           job_params;
+        /* kernelspace view of the job */
+        struct rt_job           job_params;
+        /* userspace view of the job */
+        struct rt_job           user_job;
 
         /* task representing the current "inherited" task
          * priority, assigned by inherit_priority and
diff --git a/include/litmus/sched_mc.h b/include/litmus/sched_mc.h
index 1d491ce6a31a..f0526e0d5589 100644
--- a/include/litmus/sched_mc.h
+++ b/include/litmus/sched_mc.h
@@ -16,11 +16,6 @@ struct mc_task {
         int lvl_a_eligible;
 };
 
-struct mc_job {
-        int is_ghost:1;
-        lt_t ghost_budget;
-};
-
 #ifdef __KERNEL__
 /*
  * These are used only in the kernel. Userspace programs like RTSpin won't see
@@ -28,19 +23,19 @@ struct mc_job {
  */
 struct mc_data {
         struct mc_task mc_task;
-        struct mc_job mc_job;
 };
 
 #define tsk_mc_data(t)   (tsk_rt(t)->mc_data)
 #define tsk_mc_crit(t)   (tsk_mc_data(t) ? tsk_mc_data(t)->mc_task.crit : CRIT_LEVEL_C)
-#define is_ghost(t)      (tsk_mc_data(t)->mc_job.is_ghost)
+#define is_ghost(t)      (get_rt_job(t) < get_user_job(t))
 
-#define TS "(%s/%d:%d:%s)"
+#define TS "(%s/%d:%d:%d:%s)"
 #define TA(t) (t) ? tsk_mc_data(t) ? is_ghost(t) ? "ghost" : t->comm \
-                                                 : t->comm : "NULL", \
-              (t) ? t->pid : 1,                                 \
-              (t) ? t->rt_param.job_params.job_no : 1,          \
-              (t && get_task_domain(t)) ? get_task_domain(t)->name : ""
+                                                 : t->comm : "NULL",    \
+                (t) ? t->pid : 1,                                       \
+                (t) ? get_rt_job(t) : 1,                                \
+                (t) ? get_user_job(t) : 1,                              \
+                (t && get_task_domain(t)) ? get_task_domain(t)->name : ""
 #define STRACE(fmt, args...) \
         sched_trace_log_message("%d P%d      [%s@%s:%d]: " fmt, \
                                 TRACE_ARGS,  ## args)
@@ -128,6 +123,7 @@ lt_t mc_ce_timer_callback_common(domain_t*);
 void mc_ce_release_at_common(struct task_struct*, lt_t);
 long mc_ce_activate_plugin_common(void);
 long mc_ce_deactivate_plugin_common(void);
+void mc_ce_job_completion(struct domain *dom, struct task_struct *ts);
 
 #endif /* __KERNEL__ */
 
diff --git a/include/litmus/sched_trace.h b/include/litmus/sched_trace.h
index 0e050ac3748c..6fb923bc5241 100644
--- a/include/litmus/sched_trace.h
+++ b/include/litmus/sched_trace.h
@@ -337,9 +337,10 @@ feather_callback void do_sched_trace_task_tardy(unsigned long id,
                 trace_litmus_server_switch_away(sid, job, tid, tjob);   \
         } while (0)
 
-#define sched_trace_server_release(sid, job, rel, dead)                 \
+#define sched_trace_server_release(sid, job, job_params)                \
         do {                                                            \
-                trace_litmus_server_release(sid, job, rel, dead);       \
+                trace_litmus_server_release(sid, job, (job_params).release, \
+                                            (job_params).deadline);     \
         } while (0)
 
 #define sched_trace_server_completion(sid, job)                         \
diff --git a/include/trace/events/litmus.h b/include/trace/events/litmus.h
index 474aa129c233..58ffdcd4c546 100644
--- a/include/trace/events/litmus.h
+++ b/include/trace/events/litmus.h
@@ -29,7 +29,7 @@ TRACE_EVENT(litmus_task_param,
 
         TP_fast_assign(
                 __entry->pid    = t ? t->pid : 0;
-                __entry->job    = t ? t->rt_param.job_params.job_no : 0;
+                __entry->job    = t ? get_user_job(t) : 0,
                 __entry->wcet   = get_exec_cost(t);
                 __entry->period = get_rt_period(t);
                 __entry->phase  = get_rt_phase(t);
@@ -59,9 +59,9 @@ TRACE_EVENT(litmus_task_release,
 
         TP_fast_assign(
                 __entry->pid    = t ? t->pid : 0;
-                __entry->job    = t ? t->rt_param.job_params.job_no : 0;
-                __entry->release        = get_release(t);
-                __entry->deadline       = get_deadline(t);
+                __entry->job    = t ? get_user_job(t) : 0;
+                __entry->release        = get_user_release(t);
+                __entry->deadline       = get_user_deadline(t);
         ),
 
         TP_printk("release(job(%u, %u)): %Lu\ndeadline(job(%u, %u)): %Lu\n",
@@ -86,7 +86,7 @@ TRACE_EVENT(litmus_switch_to,
 
         TP_fast_assign(
                 __entry->pid    = is_realtime(t) ? t->pid : 0;
-                __entry->job    = is_realtime(t) ? t->rt_param.job_params.job_no : 0;
+                __entry->job    = is_realtime(t) ? get_user_job(t) : 0;
                 __entry->exec_time      = get_exec_time(t);
         ),
 
@@ -111,7 +111,7 @@ TRACE_EVENT(litmus_switch_away,
 
         TP_fast_assign(
                 __entry->pid    = is_realtime(t) ? t->pid : 0;
-                __entry->job    = is_realtime(t) ? t->rt_param.job_params.job_no : 0;
+                __entry->job    = is_realtime(t) ? get_user_job(t) : 0;
                 __entry->exec_time      = get_exec_time(t);
         ),
 
@@ -136,7 +136,7 @@ TRACE_EVENT(litmus_task_completion,
 
         TP_fast_assign(
                 __entry->pid    = t ? t->pid : 0;
-                __entry->job    = t ? t->rt_param.job_params.job_no : 0;
+                __entry->job    = t ? get_user_job(t) : 0;
                 __entry->forced = forced;
         ),
 
@@ -249,7 +249,7 @@ TRACE_EVENT(litmus_task_resume,
 
         TP_fast_assign(
                 __entry->pid    = t ? t->pid : 0;
-                __entry->job    = t ? t->rt_param.job_params.job_no : 0;
+                __entry->job    = t ? get_user_job(t) : 0;
                 __entry->lid    = lid;
         ),
 
diff --git a/litmus/budget.c b/litmus/budget.c
index f7505b0f86e5..b1c0a4b84c02 100644
--- a/litmus/budget.c
+++ b/litmus/budget.c
@@ -109,28 +109,4 @@ static int __init init_budget_enforcement(void)
         return 0;
 }
 
-void task_release(struct task_struct *t)
-{
-        /* Also wrong */
-        t->rt_param.job_params.real_release = t->rt_param.job_params.real_deadline;
-        t->rt_param.job_params.real_deadline += get_rt_period(t);
-        t->rt_param.job_params.job_no++;
-        sched_trace_task_release(t);
-}
-
-void server_release(struct task_struct *t)
-{
-        t->rt_param.job_params.exec_time = 0;
-        t->rt_param.job_params.release = t->rt_param.job_params.deadline;
-        t->rt_param.job_params.deadline +=  get_rt_period(t);
-        t->rt_param.job_params.fake_job_no++;
-
-        /* don't confuse linux */
-        t->rt.time_slice = 1;
-
-        sched_trace_server_release(-t->pid, get_server_job(t),
-                                    t->rt_param.job_params.release,
-                                    t->rt_param.job_params.deadline);
-}
-
 module_init(init_budget_enforcement);
diff --git a/litmus/ce_domain.c b/litmus/ce_domain.c
index b2c5d4e935a5..c750b95581b9 100644
--- a/litmus/ce_domain.c
+++ b/litmus/ce_domain.c
@@ -80,14 +80,12 @@ void ce_domain_init(domain_t *dom,
                 requeue_t requeue,
                 peek_ready_t peek_ready,
                 take_ready_t take_ready,
-                preempt_needed_t preempt_needed,
                 task_prio_t task_prio,
                 struct ce_dom_data *dom_data,
                 const int cpu,
                 ce_timer_callback_t ce_timer_callback)
 {
-        domain_init(dom, lock, requeue, peek_ready, take_ready, preempt_needed,
-                        task_prio);
+        domain_init(dom, lock, requeue, peek_ready, take_ready, task_prio);
         dom->data = dom_data;
         dom->remove = ce_remove;
         dom_data->cpu = cpu;
diff --git a/litmus/dgl.c b/litmus/dgl.c
index dd2a42cc9ca6..e2286b3e9239 100644
--- a/litmus/dgl.c
+++ b/litmus/dgl.c
@@ -173,7 +173,6 @@ static unsigned long try_acquire(struct dgl *dgl, struct dgl_resource *resource,
 
         mask_idx(rid, &word, &bit);
 
-
         TRACE("0x%p, %lu, 0x%p\n", greq->waiting, greq->waiting[word],
               &greq->waiting[word]);
 
@@ -290,7 +289,6 @@ void remove_group_req(struct dgl *dgl, struct dgl_group_req *greq)
                                 if (try_acquire(dgl, resource, next)) {
                                         list_del_init(&next->list);
                                         print_waiting(dgl, resource);
-
                                 }
                         }
                 }
diff --git a/litmus/domain.c b/litmus/domain.c
index 4dc3649a0389..54c060d4c8af 100644
--- a/litmus/domain.c
+++ b/litmus/domain.c
@@ -8,7 +8,6 @@ void domain_init(domain_t *dom,
                  requeue_t requeue,
                  peek_ready_t peek_ready,
                  take_ready_t take_ready,
-                 preempt_needed_t preempt_needed,
                  task_prio_t priority)
 {
         INIT_LIST_HEAD(&dom->list);
@@ -16,6 +15,5 @@ void domain_init(domain_t *dom,
         dom->requeue = requeue;
         dom->peek_ready = peek_ready;
         dom->take_ready = take_ready;
-        dom->preempt_needed = preempt_needed;
         dom->higher_prio = priority;
 }
diff --git a/litmus/jobs.c b/litmus/jobs.c
index bd3175baefae..983b084cf32b 100644
--- a/litmus/jobs.c
+++ b/litmus/jobs.c
@@ -1,4 +1,5 @@
 /* litmus/jobs.c - common job control code
+ * TODO: modified heavily for sched_mc
  */
 
 #include <linux/sched.h>
@@ -6,25 +7,23 @@
 #include <litmus/litmus.h>
 #include <litmus/jobs.h>
 #include <litmus/trace.h>
+#include <litmus/sched_trace.h>
+#include <litmus/domain.h>
+#include <litmus/sched_mc.h>
 
-static inline void setup_release(struct task_struct *t, lt_t release)
+static inline void setup_release(struct task_struct *t, struct rt_job *job,
+                                 lt_t release)
 {
         /* prepare next release */
-        tsk_rt(t)->job_params.release   = release;
-        tsk_rt(t)->job_params.deadline  = release + get_rt_relative_deadline(t);
-        tsk_rt(t)->job_params.exec_time = 0;
+        job->release   = release;
+        job->deadline  = release + get_rt_relative_deadline(t);
+        job->exec_time = 0;
 
         /* update job sequence number */
-        tsk_rt(t)->job_params.job_no++;
-
-        /* don't confuse Linux */
-        t->rt.time_slice = 1;
-
-        TRACE_TASK(t, "Releasing at %llu, deadline: %llu, period: %llu, now: %llu\n",
-                   release, get_deadline(t), get_rt_period(t), litmus_clock());
+        ++job->job_no;
 }
 
-void prepare_for_next_period(struct task_struct *t)
+static inline void setup_kernel_release(struct task_struct *t, lt_t release)
 {
         BUG_ON(!t);
 
@@ -32,21 +31,49 @@ void prepare_for_next_period(struct task_struct *t)
          * release and deadline. Lateness may be negative.
          */
         t->rt_param.job_params.lateness =
-                (long long)litmus_clock() - 
+                (long long)litmus_clock() -
                 (long long)t->rt_param.job_params.deadline;
 
-        setup_release(t, get_release(t) + get_rt_period(t));
+        t->rt.time_slice = 1;
+
+        setup_release(t, &tsk_rt(t)->job_params, release);
+
+        TRACE_MC_TASK(t, "kernel rel=%llu, dead=%llu\n", get_release(t), get_deadline(t));
+
+        sched_trace_server_release(-t->pid, get_rt_job(t),
+                                   tsk_rt(t)->job_params);
+}
+
+void setup_user_release(struct task_struct *t, lt_t release)
+{
+        setup_release(t, &tsk_rt(t)->user_job, release);
+        TRACE_MC_TASK(t, "user rel=%llu, dead=%llu\n", get_user_release(t), get_user_deadline(t));
+        if (CRIT_LEVEL_A != tsk_mc_crit(t))
+                sched_trace_task_release(t);
+
+}
+
+void prepare_for_next_period(struct task_struct *t)
+{
+        setup_kernel_release(t, get_release(t) + get_rt_period(t));
 }
 
 void release_at(struct task_struct *t, lt_t start)
 {
         BUG_ON(!t);
-        setup_release(t, start);
+
+        TRACE_MC_TASK(t, "Releasing at %llu\n", start);
+
+        setup_kernel_release(t, start);
+        setup_user_release(t, start);
+
+        BUG_ON(!is_released(t, start));
+
         set_rt_flags(t, RT_F_RUNNING);
 }
 
 /*
- * Deactivate current task until the beginning of the next period.
+ * User-space job has completed execution
  */
 long complete_job(void)
 {
diff --git a/litmus/rt_domain.c b/litmus/rt_domain.c
index c63bd0303916..98a30b88baab 100644
--- a/litmus/rt_domain.c
+++ b/litmus/rt_domain.c
@@ -469,13 +469,11 @@ void pd_domain_init(domain_t *dom,
                     bheap_prio_t order,
                     check_resched_needed_t check,
                     release_jobs_t release,
-                    preempt_needed_t preempt_needed,
                     task_prio_t priority)
 {
         rt_domain_init(domain, order, check, release);
         domain_init(dom, &domain->ready_lock,
-                    pd_requeue, pd_peek_ready, pd_take_ready,
-                    preempt_needed, priority);
+                    pd_requeue, pd_peek_ready, pd_take_ready, priority);
         dom->remove = pd_remove;
         dom->data = domain;
 }
diff --git a/litmus/sched_color.c b/litmus/sched_color.c
index 66ce40fd1b57..28a7cfa401b4 100644
--- a/litmus/sched_color.c
+++ b/litmus/sched_color.c
@@ -320,8 +320,7 @@ static void job_completion(struct rt_server *server)
         prepare_for_next_period(t);
 
         if (is_server(t))
-                sched_trace_server_release(t->pid, get_rt_job(t),
-                                           get_release(t), get_deadline(t));
+                sched_trace_server_release(t->pid, get_rt_job(t), tsk_rt(t)->job_params);
         else
                 sched_trace_task_release(t);
 
diff --git a/litmus/sched_mc.c b/litmus/sched_mc.c
index f24c51653fe2..8c2238c6bf43 100644
--- a/litmus/sched_mc.c
+++ b/litmus/sched_mc.c
@@ -1,4 +1,4 @@
-/**
+/*
  * litmus/sched_mc.c
  *
  * Implementation of the Mixed Criticality scheduling algorithm.
@@ -270,13 +270,13 @@ static void update_ghost_time(struct task_struct *p)
                 delta = 0;
                 TRACE_MC_TASK(p, "WARNING: negative time delta\n");
         }
-        if (tsk_mc_data(p)->mc_job.ghost_budget <= delta) {
+        if (budget_remaining(p) <= delta) {
                 TRACE_MC_TASK(p, "Ghost job could have ended\n");
-                tsk_mc_data(p)->mc_job.ghost_budget = 0;
+                tsk_rt(p)->job_params.exec_time = get_exec_cost(p);
                 p->se.exec_start = clock;
         } else {
                 TRACE_MC_TASK(p, "Ghost job updated, but didn't finish\n");
-                tsk_mc_data(p)->mc_job.ghost_budget -= delta;
+                tsk_rt(p)->job_params.exec_time += delta;
                 p->se.exec_start = clock;
         }
 }
@@ -302,13 +302,13 @@ static void link_task_to_crit(struct crit_entry *ce,
                 ce->linked->rt_param.linked_on = NO_CPU;
                 if (is_ghost(ce->linked)) {
                         cancel_ghost(ce);
-                        if (tsk_mc_data(ce->linked)->mc_job.ghost_budget > 0) {
+                        if (!budget_exhausted(ce->linked)) {
                                 /* Job isn't finished, so do accounting */
                                 update_ghost_time(ce->linked);
                         }
                 }
-                sched_trace_server_switch_away(sid(ce), 0, ce->linked->pid,
-                                               tsk_rt(ce->linked)->job_params.job_no);
+                sched_trace_server_switch_away(sid(ce), 0, -ce->linked->pid,
+                                               get_rt_job(ce->linked));
         }
 
         /* Actually link task */
@@ -321,15 +321,11 @@ static void link_task_to_crit(struct crit_entry *ce,
                          * tasks. Otherwise reset the budget timer.
                          */
                         task->se.exec_start = litmus_clock();
-                        when_to_fire = task->se.exec_start +
-                                tsk_mc_data(task)->mc_job.ghost_budget;
+                        when_to_fire = task->se.exec_start + budget_remaining(task);
                         arm_ghost(ce, when_to_fire);
-
-                        sched_trace_server_switch_to(sid(ce), 0, 0, 0);
-                } else {
-                        sched_trace_server_switch_to(sid(ce), 0, task->pid,
-                                                     get_rt_job(ce->linked));
                 }
+                sched_trace_server_switch_to(sid(ce), 0, -task->pid,
+                                             get_rt_job(ce->linked));
         }
 }
 
@@ -434,18 +430,19 @@ static void link_task_to_cpu(struct cpu_entry *entry, struct task_struct *task)
 
         if (entry->linked) {
                 sched_trace_server_switch_away(-entry->linked->pid,
-                                               get_server_job(entry->linked),
+                                               get_rt_job(entry->linked),
                                                entry->linked->pid,
-                                               get_rt_job(entry->linked));
+                                               get_user_job(entry->linked));
         }
 
-        if (task){
+        if (task) {
                 set_rt_flags(task, RT_F_RUNNING);
                 sched_trace_server_switch_to(-task->pid,
-                                             get_server_job(task),
+                                             get_rt_job(task),
                                              task->pid,
-                                             get_rt_job(task));
+                                             get_user_job(task));
         }
+
         entry->linked = task;
 
         /* Higher criticality crit entries are now usable */
@@ -480,7 +477,11 @@ static void preempt(struct domain *dom, struct crit_entry *ce)
         }
         update_crit_position(ce);
 
-        /* Preempt actual execution if this is a running task */
+        /* Preempt actual execution if this is a running task.
+         * We know that our task is higher priority than what is currently
+         * running on this CPU as otherwise the crit_entry would have
+         * been disabled and a preemption could not have occurred
+         */
         if (!is_ghost(task)) {
                 link_task_to_cpu(entry, task);
                 preempt_if_preemptable(entry->scheduled, entry->cpu);
@@ -564,7 +565,7 @@ static void check_for_preempt(struct domain *dom)
                         if (!can_use(ce))
                                 /* CPU disabled while locking! */
                                 fix_crit_position(ce);
-                        else if (dom->preempt_needed(dom, ce->linked))
+                        else if (mc_preempt_needed(dom, ce->linked))
                                 /* Success! Check for more preemptions */
                                 preempt(dom, ce);
                         else {
@@ -582,7 +583,7 @@ static void check_for_preempt(struct domain *dom)
                 dom->peek_ready(dom);
 
                 raw_spin_lock(&entry->lock);
-                if (can_use(ce) && dom->preempt_needed(dom, ce->linked)) {
+                if (can_use(ce) && mc_preempt_needed(dom, ce->linked)) {
                         preempt(dom, ce);
                         update_crit_levels(entry);
                 } else {
@@ -609,6 +610,7 @@ static void remove_from_all(struct task_struct* task)
 
         /* Remove the task from any CPU state */
         if (task->rt_param.linked_on != NO_CPU) {
+                TRACE_MC_TASK(task, "Linked to something\n");
                 entry = &per_cpu(cpus, task->rt_param.linked_on);
                 raw_spin_lock(&entry->lock);
 
@@ -618,7 +620,7 @@ static void remove_from_all(struct task_struct* task)
                         BUG_ON(ce->linked != task);
                         link_task_to_crit(ce, NULL);
                         update_crit_position(ce);
-                        if (!is_ghost(task) && entry->linked == task) {
+                        if (entry->linked == task) {
                                 update = 1;
                                 link_task_to_cpu(entry, NULL);
                         }
@@ -646,41 +648,25 @@ static void remove_from_all(struct task_struct* task)
  */
 static void job_completion(struct task_struct *task, int forced)
 {
-        int behind;
         TRACE_MC_TASK(task, "Completed\n");
 
         /* Logically stop the task execution */
         set_rt_flags(task, RT_F_SLEEP);
         remove_from_all(task);
 
-        /* Level-A tasks cannot ever get behind */
-        behind = tsk_mc_crit(task) != CRIT_LEVEL_A && behind_server(task);
-
-        if (!forced && !is_ghost(task)) {
-                /* Task voluntarily ceased execution. Move on to next period */
-                task_release(task);
-                sched_trace_task_completion(task, forced);
-
-                /* Convert to ghost job */
-                tsk_mc_data(task)->mc_job.ghost_budget = budget_remaining(task);
-                tsk_mc_data(task)->mc_job.is_ghost = 1;
+        if (!forced) {
+                /* Userspace releases */
+                sched_trace_task_completion(current, 0);
+                setup_user_release(current, get_user_deadline(current));
         }
 
-        /* If the task has no ghost budget, convert back from ghost.
-         * If the task is behind, undo ghost conversion so that it
-         * can catch up.
-         */
-        if (behind || tsk_mc_data(task)->mc_job.ghost_budget == 0) {
-                TRACE_MC_TASK(task, "Not a ghost task\n");
-                tsk_mc_data(task)->mc_job.is_ghost = 0;
-                tsk_mc_data(task)->mc_job.ghost_budget = 0;
-        }
 
-        /* If server has run out of budget, wait until next release */
-        if (budget_exhausted(task)) {
-                sched_trace_server_completion(-task->pid,
-                                              get_server_job(task));
-                server_release(task);
+        /* If server has run out of budget, wait until next release
+         * TODO: Level A does this independently and should not.
+         */
+        if (budget_exhausted(task) && CRIT_LEVEL_A != tsk_mc_crit(task)) {
+                sched_trace_server_completion(-task->pid, get_rt_job(task));
+                prepare_for_next_period(task);
         }
 
         /* Requeue non-blocking tasks */
@@ -706,7 +692,7 @@ static enum hrtimer_restart mc_ghost_exhausted(struct hrtimer *timer)
 
         local_irq_save(flags);
         TRACE("Ghost exhausted\n");
-        TRACE_CRIT_ENTRY(ce, "Firing here\n");
+        TRACE_CRIT_ENTRY(ce, "Firing here at %llu\n", litmus_clock());
 
         /* Due to race conditions, we cannot just set the linked
          * task's budget to 0 as it may no longer be the task
@@ -716,14 +702,14 @@ static enum hrtimer_restart mc_ghost_exhausted(struct hrtimer *timer)
         raw_spin_lock(&crit_cpu(ce)->lock);
         if (ce->linked && is_ghost(ce->linked)) {
                 update_ghost_time(ce->linked);
-                if (tsk_mc_data(ce->linked)->mc_job.ghost_budget == 0) {
+                if (budget_exhausted(ce->linked)) {
                         tmp = ce->linked;
                 }
         }
         raw_spin_unlock(&crit_cpu(ce)->lock);
 
         if (tmp)
-                job_completion(tmp, 0);
+                job_completion(tmp, 1);
 
         local_irq_restore(flags);
 #ifndef CONFIG_MERGE_TIMERS
@@ -748,12 +734,11 @@ static lt_t __ce_timer_function(struct ce_dom_data *ce_data)
 
         raw_spin_lock(&crit_cpu(ce)->lock);
         if (ce->linked &&
-            ce->linked == ce_data->should_schedule &&
-            is_ghost(ce->linked))
+            ce->linked == ce_data->should_schedule)
         {
                 old_link = ce->linked;
-                tsk_mc_data(ce->linked)->mc_job.ghost_budget = 0;
                 link_task_to_crit(ce, NULL);
+                mc_ce_job_completion(dom, old_link);
         }
         raw_spin_unlock(&crit_cpu(ce)->lock);
 
@@ -764,7 +749,7 @@ static lt_t __ce_timer_function(struct ce_dom_data *ce_data)
         if (NULL != old_link) {
                 STRACE("old_link " TS " so will call job completion\n", TA(old_link));
                 raw_spin_unlock(dom->lock);
-                job_completion(old_link, 0);
+                job_completion(old_link, 1);
         } else {
                 STRACE("old_link was null, so will call check for preempt\n");
                 raw_spin_unlock(dom->lock);
@@ -837,12 +822,13 @@ static void mc_task_new(struct task_struct *t, int on_rq, int running)
         unsigned long flags;
         struct cpu_entry* entry;
         enum crit_level level = tsk_mc_crit(t);
-        char name[TASK_COMM_LEN];
-        strcpy(name, "rtspin");
 
         local_irq_save(flags);
         TRACE("New mixed criticality task %d\n", t->pid);
 
+        if (level == CRIT_LEVEL_A)
+                get_rt_relative_deadline(t) = get_exec_cost(t);
+
         /* Assign domain */
         if (level < CRIT_LEVEL_C)
                 entry = &per_cpu(cpus, get_partition(t));
@@ -850,19 +836,15 @@ static void mc_task_new(struct task_struct *t, int on_rq, int running)
                 entry = &per_cpu(cpus, task_cpu(t));
         t->rt_param._domain = entry->crit_entries[level].domain;
 
-        if (budget_enforced(t)) {
-                /* Userspace and kernelspace view of task state may differ.
-                 * Model kernel state as an additional container
-                 */
-                sched_trace_container_param(t->pid, name);
-                sched_trace_server_param(-t->pid, t->pid,
-                                         get_exec_cost(t), get_rt_period(t));
-        }
+        /* Userspace and kernelspace view of task state may differ.
+         * Model kernel state as an additional container
+         */
+        sched_trace_container_param(t->pid, t->comm);
+        sched_trace_server_param(-t->pid, t->pid,
+                                 get_exec_cost(t), get_rt_period(t));
 
         /* Setup job params */
         release_at(t, litmus_clock());
-        tsk_mc_data(t)->mc_job.ghost_budget = 0;
-        tsk_mc_data(t)->mc_job.is_ghost = 0;
         if (running) {
                 BUG_ON(entry->scheduled);
                 entry->scheduled = t;
@@ -872,7 +854,6 @@ static void mc_task_new(struct task_struct *t, int on_rq, int running)
         }
         t->rt_param.linked_on = NO_CPU;
 
-
         job_arrival(t);
 
         local_irq_restore(flags);
@@ -1057,12 +1038,14 @@ static struct task_struct* mc_schedule(struct task_struct* prev)
                 raw_spin_lock(&entry->lock);
 
                 if (!entry->linked && !ce->linked && dtask && can_use(ce)) {
+                        /* Pop dtask */
                         dom->take_ready(dom);
+
                         link_task_to_crit(ce, dtask);
                         update_crit_position(ce);
-                        ready_task = (is_ghost(dtask)) ? NULL : dtask;
 
-                        /* Task found! */
+                        /* Actual running task found */
+                        ready_task = (is_ghost(dtask)) ? NULL : dtask;
                         if (ready_task) {
                                 link_task_to_cpu(entry, ready_task);
                                 raw_spin_unlock(dom->lock);
@@ -1084,6 +1067,7 @@ static struct task_struct* mc_schedule(struct task_struct* prev)
         raw_spin_unlock(&entry->lock);
         local_irq_restore(flags);
         if (next) {
+                BUG_ON(!get_rt_job(next));
                 TRACE_MC_TASK(next, "Picked this task\n");
         } else if (exists && !next)
                 TRACE_ENTRY(entry, "Becomes idle at %llu\n", litmus_clock());
@@ -1105,11 +1089,7 @@ void mc_finish_switch(struct task_struct *prev)
  */
 void mc_release_at(struct task_struct *ts, lt_t start)
 {
-        /* hack so that we can have CE timers start at the right time */
-        if (CRIT_LEVEL_A == tsk_mc_crit(ts))
-                mc_ce_release_at_common(ts, start);
-        else
-                release_at(ts, start);
+        release_at(ts, start);
 }
 
 long mc_deactivate_plugin(void)
@@ -1191,7 +1171,7 @@ static void mc_release_ts(lt_t time)
         strcpy(name, "LVL-A");
         for_each_online_cpu(cpu) {
                 entry = &per_cpu(cpus, cpu);
-                trace_litmus_container_param(++cont_id, (const char*)&name);
+                sched_trace_container_param(++cont_id, (const char*)&name);
                 ce = &entry->crit_entries[level];
                 sched_trace_server_param(sid(ce), cont_id, 0, 0);
         }
@@ -1200,22 +1180,21 @@ static void mc_release_ts(lt_t time)
         strcpy(name, "LVL-B");
         for_each_online_cpu(cpu) {
                 entry = &per_cpu(cpus, cpu);
-                trace_litmus_container_param(++cont_id, (const char*)&name);
+                sched_trace_container_param(++cont_id, (const char*)&name);
                 ce = &entry->crit_entries[level];
                 sched_trace_server_param(sid(ce), cont_id, 0, 0);
         }
 
         level = CRIT_LEVEL_C;
         strcpy(name, "LVL-C");
-        trace_litmus_container_param(++cont_id, (const char*)&name);
+        sched_trace_container_param(++cont_id, (const char*)&name);
         for_each_online_cpu(cpu) {
                 entry = &per_cpu(cpus, cpu);
                 ce = &entry->crit_entries[level];
                 sched_trace_server_param(sid(ce), cont_id, 0, 0);
         }
 
-
-
+        mc_ce_release_at_common(NULL, time);
 }
 
 static struct sched_plugin mc_plugin __cacheline_aligned_in_smp = {
@@ -1287,8 +1266,7 @@ static inline void init_edf_domain(struct domain *dom, rt_domain_t *rt,
                                    enum crit_level prio, int is_partitioned, int cpu)
 {
         pd_domain_init(dom, rt, edf_ready_order, NULL,
-                       mc_release_jobs, mc_preempt_needed,
-                       edf_higher_prio);
+                       mc_release_jobs, edf_higher_prio);
         rt->level = prio;
 #if defined(CONFIG_PLUGIN_MC_RELEASE_MASTER) && defined(CONFIG_MERGE_TIMERS)
         /* All timers are on one CPU and release-master is using the event
@@ -1345,8 +1323,7 @@ static int __init init_mc(void)
                 raw_spin_lock_init(a_dom_lock);
                 ce_domain_init(&dom_data->domain,
                                 a_dom_lock, ce_requeue, ce_peek_and_take_ready,
-                                ce_peek_and_take_ready, mc_preempt_needed,
-                                ce_higher_prio, ce_data, cpu,
+                                ce_peek_and_take_ready, ce_higher_prio, ce_data, cpu,
                                 ce_timer_function);
                 init_local_domain(entry, dom_data, CRIT_LEVEL_A);
                 dom_data->domain.name = "LVL-A";
diff --git a/litmus/sched_mc_ce.c b/litmus/sched_mc_ce.c
index 702b46da93d5..e4d66bad2138 100644
--- a/litmus/sched_mc_ce.c
+++ b/litmus/sched_mc_ce.c
@@ -130,7 +130,7 @@ static inline lt_t get_cycle_offset(const lt_t when, const lt_t cycle_time)
  *
  * Do not call prepare_for_next_period on Level-A tasks!
  */
-static void mc_ce_job_completion(struct domain *dom, struct task_struct *ts)
+void mc_ce_job_completion(struct domain *dom, struct task_struct *ts)
 {
         const int cpu = task_cpu(ts);
         const int idx = tsk_mc_data(ts)->mc_task.lvl_a_id;
@@ -141,6 +141,7 @@ static void mc_ce_job_completion(struct domain *dom, struct task_struct *ts)
 
         /* sched_trace_task_completion(ts, 0); */
         /* post-increment is important here */
+        sched_trace_server_completion(-ts->pid, get_rt_job(ts));
         just_finished = (tsk_rt(ts)->job_params.job_no)++;
 
         /* Job completes in expected window: everything is normal.
@@ -157,7 +158,6 @@ static void mc_ce_job_completion(struct domain *dom, struct task_struct *ts)
                 printk(KERN_CRIT "job %u completed in expected job %u which "
                                 "seems too early\n", just_finished,
                                 pid_entry->expected_job);
-                BUG();
         }
 }
 
@@ -451,7 +451,7 @@ lt_t mc_ce_timer_callback_common(struct domain *dom)
         struct ce_pid_table *pid_table;
         struct ce_pid_entry *pid_entry;
         struct ce_dom_data *ce_data;
-        int idx, budget_overrun;
+        int idx, budget_overrun, expected;
 
         ce_data = dom->data;
         pid_table = get_pid_table(ce_data->cpu);
@@ -481,26 +481,31 @@ lt_t mc_ce_timer_callback_common(struct domain *dom)
                  * If jobs are not overrunning their budgets, then this
                  * should not happen.
                  */
-                pid_entry->expected_job++;
-                budget_overrun = pid_entry->expected_job !=
+                expected = ++pid_entry->expected_job;
+                TRACE_MC_TASK(should_schedule, "Expected now: %d\n", expected);
+                budget_overrun = expected !=
                         tsk_rt(should_schedule)->job_params.job_no;
-                if (budget_overrun)
+                if (budget_overrun) {
                         TRACE_MC_TASK(should_schedule,
                                       "timer expected job number: %u "
                                       "but current job: %u\n",
-                                      pid_entry->expected_job,
+                                      expected,
                                       tsk_rt(should_schedule)->job_params.job_no);
+                }
         }
 
         if (ce_data->should_schedule) {
-                tsk_rt(should_schedule)->job_params.deadline =
+                get_deadline(should_schedule) =
                         cycle_start_abs + pid_entry->acc_time;
-                tsk_rt(should_schedule)->job_params.release =
-                        tsk_rt(should_schedule)->job_params.deadline -
+                get_release(should_schedule) = tsk_rt(should_schedule)->job_params.deadline -
                         pid_entry->budget;
                 tsk_rt(should_schedule)->job_params.exec_time = 0;
-                /* sched_trace_task_release(should_schedule); */
+
+                TRACE_MC_TASK(should_schedule, "Released!\n");
                 set_rt_flags(ce_data->should_schedule, RT_F_RUNNING);
+                sched_trace_task_release(should_schedule);
+                sched_trace_server_release(-should_schedule->pid, get_rt_job(should_schedule),
+                                           tsk_rt(should_schedule)->job_params);
         }
         return next_timer_abs;
 }
@@ -603,7 +608,7 @@ static void arm_all_timers(void)
                 if (0 == pid_table->num_pid_entries)
                         continue;
                 for (idx = 0; idx < pid_table->num_pid_entries; idx++) {
-                        pid_table->entries[idx].expected_job = 0;
+                        pid_table->entries[idx].expected_job = 1;
                 }
 #ifdef CONFIG_PLUGIN_MC_RELEASE_MASTER
                 cpu_for_timer = interrupt_cpu;
@@ -629,7 +634,7 @@ static void arm_all_timers(void)
  */
 void mc_ce_release_at_common(struct task_struct *ts, lt_t start)
 {
-        TRACE_TASK(ts, "release at\n");
+        TRACE("release CE at %llu\n", start);
         if (atomic_inc_and_test(&start_time_set)) {
                 /* in this case, we won the race */
                 cancel_all_timers();
@@ -664,8 +669,7 @@ long mc_ce_activate_plugin_common(void)
 
         atomic_set(&start_time_set, -1);
         atomic64_set(&start_time, litmus_clock());
-        /* may not want to arm timers on activation, just after release */
-        arm_all_timers();
+
         ret = 0;
 out:
         return ret;
@@ -707,7 +711,7 @@ static void clear_pid_entries(void)
                         }
                         pid_table->entries[entry].budget = 0;
                         pid_table->entries[entry].acc_time = 0;
-                        pid_table->entries[entry].expected_job = 0;
+                        pid_table->entries[entry].expected_job = 1;
                 }
         }
 }
@@ -752,7 +756,7 @@ static int __init init_sched_mc_ce(void)
                 raw_spin_lock_init(ce_lock);
                 dom_data = &per_cpu(_mc_ce_doms, cpu);
                 dom = &dom_data->domain;
-                ce_domain_init(dom, ce_lock, NULL, NULL, NULL, NULL, NULL,
+                ce_domain_init(dom, ce_lock, NULL, NULL, NULL, NULL,
                                 &per_cpu(_mc_ce_dom_data, cpu), cpu,
                                 mc_ce_timer_callback);
         }
diff --git a/litmus/sched_task_trace.c b/litmus/sched_task_trace.c
index 67b01c1dd51b..a5e7f799c1a9 100644
--- a/litmus/sched_task_trace.c
+++ b/litmus/sched_task_trace.c
@@ -98,7 +98,7 @@ static inline struct st_event_record* get_record(u8 type, struct task_struct* t)
                 rec->hdr.type = type;
                 rec->hdr.cpu  = smp_processor_id();
                 rec->hdr.pid  = t ? t->pid : 0;
-                rec->hdr.job  = t ? t->rt_param.job_params.job_no : 0;
+                rec->hdr.job  = t ? get_user_job(t) : 0;
         } else {
                 put_cpu_var(st_event_buffer);
         }
@@ -146,8 +146,8 @@ feather_callback void do_sched_trace_task_release(unsigned long id, unsigned lon
         struct task_struct *t = (struct task_struct*) _task;
         struct st_event_record* rec = get_record(ST_RELEASE, t);
         if (rec) {
-                rec->data.release.release  = tsk_rt(t)->job_params.real_release;
-                rec->data.release.deadline = tsk_rt(t)->job_params.real_deadline;
+                rec->data.release.release  = get_user_release(t);
+                rec->data.release.deadline = get_user_deadline(t);
                 put_record(rec);
         }
 }