15 files changed, 481 insertions, 57 deletions

diff --git a/include/litmus/affinity.h b/include/litmus/affinity.h
new file mode 100644
index 000000000000..ca2e442eb547
--- /dev/null
+++ b/include/litmus/affinity.h
@@ -0,0 +1,80 @@
+#ifndef __LITMUS_AFFINITY_H
+#define __LITMUS_AFFINITY_H
+
+#include <linux/cpumask.h>
+
+/*
+  L1 (instr) = depth 0
+  L1 (data)  = depth 1
+  L2 = depth 2
+  L3 = depth 3
+ */
+#define NUM_CACHE_LEVELS 4
+
+struct neighborhood
+{
+        unsigned int size[NUM_CACHE_LEVELS];
+        cpumask_var_t neighbors[NUM_CACHE_LEVELS];
+};
+
+/* topology info is stored redundently in a big array for fast lookups */
+extern struct neighborhood neigh_info[NR_CPUS];
+
+void init_topology(void); /* called by Litmus module's _init_litmus() */
+
+/* Works like:
+void get_nearest_available_cpu(
+        cpu_entry_t **nearest,
+        cpu_entry_t *start,
+        cpu_entry_t *entries,
+        int release_master)
+
+Set release_master = NO_CPU for no Release Master.
+
+We use a macro here to exploit the fact that C-EDF and G-EDF
+have similar structures for their cpu_entry_t structs, even though
+they do not share a common base-struct.  The macro allows us to
+avoid code duplication.
+
+TODO: Factor out the job-to-processor linking from C/G-EDF into
+a reusable "processor mapping".  (See B.B.'s RTSS'09 paper &
+dissertation.)
+ */
+#define get_nearest_available_cpu(nearest, start, entries, release_master) \
+{ \
+        (nearest) = NULL; \
+        if (!(start)->linked) { \
+                (nearest) = (start); \
+        } else { \
+                int __level; \
+                int __cpu; \
+                int __release_master = ((release_master) == NO_CPU) ? -1 : (release_master); \
+                struct neighborhood *__neighbors = &neigh_info[(start)->cpu]; \
+                \
+                for (__level = 0; (__level < NUM_CACHE_LEVELS) && !(nearest); ++__level) { \
+                        if (__neighbors->size[__level] > 1) { \
+                                for_each_cpu(__cpu, __neighbors->neighbors[__level]) { \
+                                        if (__cpu != __release_master) { \
+                                                cpu_entry_t *__entry = &per_cpu((entries), __cpu); \
+                                                if (!__entry->linked) { \
+                                                        (nearest) = __entry; \
+                                                        break; \
+                                                } \
+                                        } \
+                                } \
+                        } else if (__neighbors->size[__level] == 0) { \
+                                break; \
+                        } \
+                } \
+        } \
+        \
+        if ((nearest)) { \
+                TRACE("P%d is closest available CPU to P%d\n", \
+                                (nearest)->cpu, (start)->cpu); \
+        } else { \
+                TRACE("Could not find an available CPU close to P%d\n", \
+                                (start)->cpu); \
+        } \
+}
+
+#endif
diff --git a/include/litmus/budget.h b/include/litmus/budget.h
index ff18d89e8630..265f2b1e62b8 100644
--- a/include/litmus/budget.h
+++ b/include/litmus/budget.h
@@ -1,6 +1,11 @@
 #ifndef _LITMUS_BUDGET_H_
 #define _LITMUS_BUDGET_H_
 
+struct enforcement_timer {
+        struct hrtimer          timer;
+        int                     armed;
+};
+
 /**
  * update_enforcement_timer() - Update per-processor enforcement timer for
  * the next scheduled task.
@@ -11,6 +16,12 @@
 */
 void update_enforcement_timer(struct task_struct* t);
 
+void init_enforcement_timer(struct enforcement_timer *et);
+
+void arm_enforcement_timer(struct enforcement_timer* et, struct task_struct* t);
+
+void cancel_enforcement_timer(struct enforcement_timer* et);
+
 /* True if a task's server has progressed farther than the task
  * itself. This happens when budget enforcement has caused a task to be
  * booted off until the next period.
@@ -32,4 +43,5 @@ void server_release(struct task_struct *t);
  * so that we can write them to feather trace.
  */
 void task_release(struct task_struct *t);
+
 #endif
diff --git a/include/litmus/color.h b/include/litmus/color.h
new file mode 100644
index 000000000000..eefb6c6dddf5
--- /dev/null
+++ b/include/litmus/color.h
@@ -0,0 +1,51 @@
+#ifndef LITMUS_COLOR_H
+#define LITMUS_COLOR_H
+
+#ifdef __KERNEL__
+
+#define ONE_COLOR_LEN 11
+#define ONE_COLOR_FMT "%4lu: %4d\n"
+
+struct color_cache_info {
+        unsigned long size;
+        unsigned long line_size;
+        unsigned long ways;
+        unsigned long sets;
+        unsigned long nr_colors;
+};
+
+/* defined in litmus/color.c */
+extern struct color_cache_info color_cache_info;
+extern unsigned long color_chunk;
+
+struct page* get_colored_page(unsigned long);
+void add_page_to_color_list(struct page*);
+void add_page_to_alloced_list(struct page*, struct vm_area_struct*);
+void reclaim_pages(struct vm_area_struct*);
+
+int color_server_params(int cpu, unsigned long *wcet, unsigned long *period);
+
+int color_add_pages_handler(struct ctl_table *, int, void __user *,
+                size_t *, loff_t *);
+int color_nr_pages_handler(struct ctl_table *, int, void __user *,
+                size_t *, loff_t *);
+int color_reclaim_pages_handler(struct ctl_table *, int, void __user *,
+                size_t *, loff_t *);
+
+#ifdef CONFIG_LOCKDEP
+#define LITMUS_LOCKDEP_NAME_MAX_LEN 50
+#define LOCKDEP_DYNAMIC_ALLOC(lock, key, name_buf, fmt, args...)        \
+        do {                                                            \
+                snprintf(name_buf, LITMUS_LOCKDEP_NAME_MAX_LEN,         \
+                                fmt, ## args);                          \
+                lockdep_set_class_and_name(lock, key, name_buf);        \
+        } while (0)
+#else
+#define LITMUS_LOCKDEP_NAME_MAX_LEN 0
+#define LOCKDEP_DYNAMIC_ALLOC(lock, key, name_buf, fmt, args)           \
+        do { (void)(key); } while (0)
+#endif
+
+#endif
+
+#endif
diff --git a/include/litmus/dgl.h b/include/litmus/dgl.h
new file mode 100644
index 000000000000..acd58f80b58b
--- /dev/null
+++ b/include/litmus/dgl.h
@@ -0,0 +1,65 @@
+#ifndef __DGL_H_
+#define __DGL_H_
+
+#include <litmus/color.h>
+#include <linux/list.h>
+
+/*
+ * A request for @replica amount of a single resource.
+ */
+struct dgl_req {
+        unsigned short          replicas;
+        struct list_head        list;
+        struct dgl_group_req    *greq;
+};
+
+/*
+ * Simultaneous @requests for multiple resources.
+ */
+struct dgl_group_req {
+        int                     cpu;
+        unsigned long           *requested;
+        unsigned long           *waiting;
+
+        struct dgl_req          *requests;
+
+        unsigned long long      ts;
+};
+
+/*
+ * A single resource.
+ */
+struct dgl_resource {
+        unsigned long           free_replicas;
+        struct list_head        waiting;
+};
+
+/*
+ * A group of resources.
+ */
+struct dgl {
+        struct dgl_resource     *resources;
+        struct dgl_group_req*   *acquired;
+
+        char                    requests;
+        char                    running;
+        unsigned long long      ts;
+
+        unsigned long           num_resources;
+        unsigned long           num_replicas;
+};
+
+void dgl_init(struct dgl *dgl, unsigned long num_resources,
+              unsigned long num_replicas);
+void dgl_free(struct dgl *dgl);
+
+void dgl_group_req_init(struct dgl *dgl, struct dgl_group_req *greq);
+void dgl_group_req_free(struct dgl_group_req *greq);
+
+void set_req(struct dgl *dgl, struct dgl_group_req *greq,
+             int resource, int replicas);
+
+void add_group_req(struct dgl *dgl, struct dgl_group_req *greq, int cpu);
+void remove_group_req(struct dgl *dgl, struct dgl_group_req *greq);
+
+#endif
diff --git a/include/litmus/fifo_common.h b/include/litmus/fifo_common.h
new file mode 100644
index 000000000000..4756f77bd511
--- /dev/null
+++ b/include/litmus/fifo_common.h
@@ -0,0 +1,25 @@
+/*
+ * EDF common data structures and utility functions shared by all EDF
+ * based scheduler plugins
+ */
+
+/* CLEANUP: Add comments and make it less messy.
+ *
+ */
+
+#ifndef __FIFO_COMMON_H__
+#define __FIFO_COMMON_H__
+
+#include <litmus/rt_domain.h>
+
+void fifo_domain_init(rt_domain_t* rt, check_resched_needed_t resched,
+                      release_jobs_t release);
+
+int fifo_higher_prio(struct task_struct* first,
+                    struct task_struct* second);
+
+int fifo_ready_order(struct bheap_node* a, struct bheap_node* b);
+
+int fifo_preemption_needed(rt_domain_t* rt, struct task_struct *t);
+
+#endif
diff --git a/include/litmus/ftdev.h b/include/litmus/ftdev.h
index 348387e9adf9..0b959874dd70 100644
--- a/include/litmus/ftdev.h
+++ b/include/litmus/ftdev.h
@@ -16,7 +16,8 @@ typedef int  (*ftdev_can_open_t)(struct ftdev* dev, unsigned int buf_no);
 /* return 0 on success, otherwise -$REASON */
 typedef int  (*ftdev_alloc_t)(struct ftdev* dev, unsigned int buf_no);
 typedef void (*ftdev_free_t)(struct ftdev* dev, unsigned int buf_no);
-
+/* Let devices handle writes from userspace. No synchronization provided. */
+typedef ssize_t (*ftdev_write_t)(struct ft_buffer* buf, size_t len, const char __user *from);
 
 struct ftdev_event;
 
@@ -27,6 +28,7 @@ struct ftdev_minor {
         /* FIXME: filter for authorized events */
         struct ftdev_event*     events;
         struct device*          device;
+        struct ftdev*           ftdev;
 };
 
 struct ftdev {
@@ -39,6 +41,7 @@ struct ftdev {
         ftdev_alloc_t           alloc;
         ftdev_free_t            free;
         ftdev_can_open_t        can_open;
+        ftdev_write_t           write;
 };
 
 struct ft_buffer* alloc_ft_buffer(unsigned int count, size_t size);
diff --git a/include/litmus/litmus.h b/include/litmus/litmus.h
index d5dbc82f0dfc..1455f249c1fb 100644
--- a/include/litmus/litmus.h
+++ b/include/litmus/litmus.h
@@ -26,7 +26,7 @@ static inline int in_list(struct list_head* list)
                 );
 }
 
-struct task_struct* waitqueue_first(wait_queue_head_t *wq);
+struct task_struct* __waitqueue_remove_first(wait_queue_head_t *wq);
 
 #define NO_CPU                  0xffffffff
 
@@ -53,11 +53,15 @@ void litmus_exit_task(struct task_struct *tsk);
 #define get_exec_time(t)        (tsk_rt(t)->job_params.exec_time)
 #define get_rt_period(t)        (tsk_rt(t)->task_params.period)
 #define get_rt_phase(t)         (tsk_rt(t)->task_params.phase)
+#define get_rt_job(t)           (tsk_rt(t)->job_params.job_no)
 #define get_partition(t)        (tsk_rt(t)->task_params.cpu)
 #define get_deadline(t)         (tsk_rt(t)->job_params.deadline)
 #define get_release(t)          (tsk_rt(t)->job_params.release)
 #define get_class(t)            (tsk_rt(t)->task_params.cls)
+
 #define get_task_domain(t)      (tsk_rt(t)->_domain)
+#define is_server(t)            (tsk_rt(t)->is_server)
+#define get_task_server(task)      (tsk_rt(task)->server)
 
 #define is_priority_boosted(t)  (tsk_rt(t)->priority_boosted)
 #define get_boost_start(t)      (tsk_rt(t)->boost_start_time)
@@ -150,7 +154,7 @@ static inline int is_kernel_np(struct task_struct *t)
 
 static inline int is_user_np(struct task_struct *t)
 {
-        return tsk_rt(t)->ctrl_page ? tsk_rt(t)->ctrl_page->np_flag : 0;
+        return tsk_rt(t)->ctrl_page ? tsk_rt(t)->ctrl_page->sched.np.flag : 0;
 }
 
 static inline void request_exit_np(struct task_struct *t)
@@ -160,20 +164,58 @@ static inline void request_exit_np(struct task_struct *t)
                  * into the kernel at the end of a critical section. */
                 if (likely(tsk_rt(t)->ctrl_page)) {
                         TRACE_TASK(t, "setting delayed_preemption flag\n");
-                        tsk_rt(t)->ctrl_page->delayed_preemption = 1;
+                        tsk_rt(t)->ctrl_page->sched.np.preempt = 1;
                 }
         }
 }
 
-static inline void clear_exit_np(struct task_struct *t)
+static inline void make_np(struct task_struct *t)
 {
-        if (likely(tsk_rt(t)->ctrl_page))
-                tsk_rt(t)->ctrl_page->delayed_preemption = 0;
+        tsk_rt(t)->kernel_np = 1;
+}
+
+/* Caller should check if preemption is necessary when
+ * the function return 0.
+ */
+static inline int take_np(struct task_struct *t)
+{
+        return tsk_rt(t)->kernel_np = 0;
+}
+
+/* returns 0 if remote CPU needs an IPI to preempt, 1 if no IPI is required */
+static inline int request_exit_np_atomic(struct task_struct *t)
+{
+        union np_flag old, new;
+
+        if (tsk_rt(t)->ctrl_page) {
+                old.raw = tsk_rt(t)->ctrl_page->sched.raw;
+                if (old.np.flag == 0) {
+                        /* no longer non-preemptive */
+                        return 0;
+                } else if (old.np.preempt) {
+                        /* already set, nothing for us to do */
+                        return 1;
+                } else {
+                        /* non preemptive and flag not set */
+                        new.raw = old.raw;
+                        new.np.preempt = 1;
+                        /* if we get old back, then we atomically set the flag */
+                        return cmpxchg(&tsk_rt(t)->ctrl_page->sched.raw, old.raw, new.raw) == old.raw;
+                        /* If we raced with a concurrent change, then so be
+                         * it. Deliver it by IPI.  We don't want an unbounded
+                         * retry loop here since tasks might exploit that to
+                         * keep the kernel busy indefinitely. */
+                }
+        } else
+                return 0;
 }
 
+#else
+
+
 static inline void make_np(struct task_struct *t)
 {
-        tsk_rt(t)->kernel_np++;
+
 }
 
 /* Caller should check if preemption is necessary when
@@ -181,11 +223,9 @@ static inline void make_np(struct task_struct *t)
  */
 static inline int take_np(struct task_struct *t)
 {
-        return --tsk_rt(t)->kernel_np;
+        return 0;
 }
 
-#else
-
 static inline int is_kernel_np(struct task_struct* t)
 {
         return 0;
@@ -202,22 +242,25 @@ static inline void request_exit_np(struct task_struct *t)
         BUG();
 }
 
-static inline void clear_exit_np(struct task_struct* t)
+static inline int request_exit_np_atomic(struct task_struct *t)
 {
+        return 0;
 }
 
 #endif
 
+static inline void clear_exit_np(struct task_struct *t)
+{
+        if (likely(tsk_rt(t)->ctrl_page))
+                tsk_rt(t)->ctrl_page->sched.np.preempt = 0;
+}
+
 static inline int is_np(struct task_struct *t)
 {
 #ifdef CONFIG_SCHED_DEBUG_TRACE
         int kernel, user;
         kernel = is_kernel_np(t);
         user   = is_user_np(t);
-        if (kernel || user)
-                TRACE_TASK(t, " is non-preemptive: kernel=%d user=%d\n",
-
-                           kernel, user);
         return kernel || user;
 #else
         return unlikely(is_kernel_np(t) || is_user_np(t));
diff --git a/include/litmus/preempt.h b/include/litmus/preempt.h
index 29bf9bc47daf..5b69e26fc57d 100644
--- a/include/litmus/preempt.h
+++ b/include/litmus/preempt.h
@@ -11,7 +11,7 @@
 
 extern DEFINE_PER_CPU_SHARED_ALIGNED(atomic_t, resched_state);
 
-#ifdef CONFIG_DEBUG_KERNEL
+#ifdef CONFIG_PREEMPT_STATE_TRACE
 const char* sched_state_name(int s);
 //#define TRACE_STATE(fmt, args...) TRACE("SCHED_STATE " fmt, args)
 #define TRACE_STATE(fmt, args...) /* ignore */
diff --git a/include/litmus/rm_common.h b/include/litmus/rm_common.h
new file mode 100644
index 000000000000..3e03d9b5d140
--- /dev/null
+++ b/include/litmus/rm_common.h
@@ -0,0 +1,25 @@
+/*
+ * RM common data structures and utility functions shared by all RM
+ * based scheduler plugins
+ */
+
+/* CLEANUP: Add comments and make it less messy.
+ *
+ */
+
+#ifndef __UNC_RM_COMMON_H__
+#define __UNC_RM_COMMON_H__
+
+#include <litmus/rt_domain.h>
+
+void rm_domain_init(rt_domain_t* rt, check_resched_needed_t resched,
+                     release_jobs_t release);
+
+int rm_higher_prio(struct task_struct* first,
+                    struct task_struct* second);
+
+int rm_ready_order(struct bheap_node* a, struct bheap_node* b);
+
+int rm_preemption_needed(rt_domain_t* rt, struct task_struct *t);
+
+#endif
diff --git a/include/litmus/rt_domain.h b/include/litmus/rt_domain.h
index 3a0861c1d700..03826228dbd9 100644
--- a/include/litmus/rt_domain.h
+++ b/include/litmus/rt_domain.h
@@ -104,6 +104,8 @@ void __add_ready(rt_domain_t* rt, struct task_struct *new);
 void __merge_ready(rt_domain_t* rt, struct bheap *tasks);
 void __add_release(rt_domain_t* rt, struct task_struct *task);
 
+struct release_heap* release_heap_alloc(int gfp_flags);
+
 static inline struct task_struct* __take_ready(rt_domain_t* rt)
 {
         struct bheap_node* hn = bheap_take(rt->order, &rt->ready_queue);
diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h
index 4293575d3472..2ced0dba067d 100644
--- a/include/litmus/rt_param.h
+++ b/include/litmus/rt_param.h
@@ -30,7 +30,7 @@ typedef enum {
 typedef enum {
         NO_ENFORCEMENT,      /* job may overrun unhindered */
         QUANTUM_ENFORCEMENT, /* budgets are only checked on quantum boundaries */
-        PRECISE_ENFORCEMENT  /* NOT IMPLEMENTED - enforced with hrtimers */
+        PRECISE_ENFORCEMENT  /* budgets are enforced with hrtimers */
 } budget_policy_t;
 
 struct rt_task {
@@ -42,6 +42,16 @@ struct rt_task {
         budget_policy_t budget_policy; /* ignored by pfair */
 };
 
+union np_flag {
+        uint32_t raw;
+        struct {
+                /* Is the task currently in a non-preemptive section? */
+                uint32_t flag:31;
+                /* Should the task call into the scheduler? */
+                uint32_t preempt:1;
+        } np;
+};
+
 /* The definition of the data that is shared between the kernel and real-time
  * tasks via a shared page (see litmus/ctrldev.c).
  *
@@ -57,16 +67,33 @@ struct rt_task {
  * determining preemption/migration overheads).
  */
 struct control_page {
-        /* Is the task currently in a non-preemptive section? */
-        int np_flag;
-        /* Should the task call into the kernel when it leaves
-         * its non-preemptive section? */
-        int delayed_preemption;
+        volatile union np_flag sched;
 
         /* locking overhead tracing: time stamp prior to system call */
         uint64_t ts_syscall_start; /* Feather-Trace cycles */
 
-        /* to be extended */
+        int colors_updated:8;
+};
+
+#ifndef __KERNEL__
+/*
+ * XXX This is a terrible hack so liblitmus can use the PAGE_SIZE macro.
+ * We should fix liblitmus to do setup the page size at runtime.
+ */
+#define CACHE_LINE_SIZE 64
+#if (ARCH == x86_64)
+#define PAGE_SIZE 4096
+#elif (ARCH == sparc64)
+#define PAGE_SIZE 8192
+#endif
+#endif /* ifndef __KERNEL__ */
+
+typedef uint8_t color_t;
+#define COLORS_PER_CONTROL_PAGE (PAGE_SIZE / (2 * sizeof(color_t)))
+struct color_ctrl_page {
+        color_t colors[COLORS_PER_CONTROL_PAGE];
+        /* must be same type to guarantee equal array sizes */
+        color_t pages[COLORS_PER_CONTROL_PAGE];
 };
 
 /* don't export internal data structures to user space (liblitmus) */
@@ -76,6 +103,8 @@ struct _rt_domain;
 struct bheap_node;
 struct release_heap;
 struct domain;
+struct rt_server;
+struct dgl_group_req;
 
 struct rt_job {
         /* Time instant the the job was or will be released.  */
@@ -119,6 +148,8 @@ struct rt_param {
         /* is the task present? (true if it can be scheduled) */
         unsigned int            present:1;
 
+        unsigned int            is_server:1;
+
 #ifdef CONFIG_LITMUS_LOCKING
         /* Is the task being priority-boosted by a locking protocol? */
         unsigned int            priority_boosted:1;
@@ -134,6 +165,9 @@ struct rt_param {
         struct rt_event *event;
 #endif
 
+        struct rt_server *server;
+
+
         /* user controlled parameters */
         struct rt_task          task_params;
 
@@ -213,6 +247,16 @@ struct rt_param {
 
         /* Pointer to the page shared between userspace and kernel. */
         struct control_page * ctrl_page;
+
+        lt_t            total_tardy;
+        lt_t            max_tardy;
+        unsigned int    missed;
+
+        lt_t max_exec_time;
+        lt_t tot_exec_time;
+        lt_t last_exec_time;
+        struct color_ctrl_page *color_ctrl_page;
+        struct dgl_group_req *req;
 };
 
 /*      Possible RT flags       */
diff --git a/include/litmus/rt_server.h b/include/litmus/rt_server.h
new file mode 100644
index 000000000000..0e2feb6c6b0e
--- /dev/null
+++ b/include/litmus/rt_server.h
@@ -0,0 +1,31 @@
+#ifndef __RT_SERVER_H
+#define __RT_SERVER_H
+
+#include <linux/sched.h>
+#include <litmus/litmus.h>
+#include <litmus/rt_domain.h>
+
+struct rt_server;
+
+typedef int (*need_preempt_t)(rt_domain_t *rt, struct task_struct *t);
+typedef void (*server_update_t)(struct rt_server *srv);
+
+struct rt_server {
+        int sid;
+        int cpu;
+        struct task_struct*     linked;
+        rt_domain_t*            domain;
+        int running;
+
+        /* Does this server have a higher-priority task? */
+        need_preempt_t          need_preempt;
+        /* System state has changed, so should server */
+        server_update_t         update;
+};
+
+void init_rt_server(struct rt_server *server,
+                    int sid, int cpu, rt_domain_t *domain,
+                    need_preempt_t need_preempt,
+                    server_update_t update);
+
+#endif
diff --git a/include/litmus/sched_trace.h b/include/litmus/sched_trace.h
index e193267a321f..0370cdc091aa 100644
--- a/include/litmus/sched_trace.h
+++ b/include/litmus/sched_trace.h
@@ -81,6 +81,17 @@ struct st_sys_release_data {
         u64     release;
 };
 
+struct st_task_exit_data {
+        u64     avg_exec_time;
+        u64     max_exec_time;
+};
+
+struct st_task_tardy_data {
+        u64     total_tardy;
+        u32     max_tardy;
+        u32     missed;
+};
+
 #define DATA(x) struct st_ ## x ## _data x;
 
 typedef enum {
@@ -95,7 +106,9 @@ typedef enum {
         ST_BLOCK,
         ST_RESUME,
         ST_ACTION,
-        ST_SYS_RELEASE
+        ST_SYS_RELEASE,
+        ST_TASK_EXIT,
+        ST_TASK_TARDY,
 } st_event_record_type_t;
 
 struct st_event_record {
@@ -114,6 +127,8 @@ struct st_event_record {
                 DATA(resume);
                 DATA(action);
                 DATA(sys_release);
+                DATA(task_exit);
+                DATA(task_tardy);
         } data;
 };
 
@@ -155,6 +170,10 @@ feather_callback void do_sched_trace_action(unsigned long id,
                                             unsigned long action);
 feather_callback void do_sched_trace_sys_release(unsigned long id,
                                                  lt_t* start);
+feather_callback void do_sched_trace_task_exit(unsigned long id,
+                                                     struct task_struct* task);
+feather_callback void do_sched_trace_task_tardy(unsigned long id,
+                                                     struct task_struct* task);
 
 #endif
 
@@ -181,14 +200,17 @@ feather_callback void do_sched_trace_sys_release(unsigned long id,
 #define trace_litmus_task_resume(t, i)
 #define trace_litmus_sys_release(start)
 
+#define trace_litmus_task_exit(t)
+#define trace_litmus_task_tardy(t)
+
 #define trace_litmus_resource_acquire(t, i);
 #define trace_litmus_resource_release(t, i);
 #define trace_litmus_priority_donate(t, d, i)
 
 #define trace_litmus_container_param(cid, name)
 #define trace_litmus_server_param(sid, cid, wcet, time)
-#define trace_litmus_server_switch_to(sid, job, tid)
-#define trace_litmus_server_switch_away(sid, job, tid)
+#define trace_litmus_server_switch_to(sid, job, tid, tjob)
+#define trace_litmus_server_switch_away(sid, job, tid, tjob)
 #define trace_litmus_server_release(sid, job, release, deadline)
 #define trace_litmus_server_completion(sid, job)
 
@@ -264,6 +286,21 @@ feather_callback void do_sched_trace_sys_release(unsigned long id,
                 trace_litmus_sys_release(when);                         \
         } while (0)
 
+#define sched_trace_task_exit(t)                                        \
+        do {                                                            \
+                SCHED_TRACE(SCHED_TRACE_BASE_ID + 11,                   \
+                        do_sched_trace_task_exit, t);                   \
+                trace_litmus_task_exit(t);                              \
+        } while (0)
+
+
+#define sched_trace_task_tardy(t)                                       \
+        do {                                                            \
+                SCHED_TRACE(SCHED_TRACE_BASE_ID + 12,                   \
+                        do_sched_trace_task_tardy, t);                  \
+                trace_litmus_task_tardy(t);                             \
+        } while (0)
+
 #define QT_START lt_t _qt_start = litmus_clock()
 #define QT_END \
         sched_trace_log_message("%d P%d      [%s@%s:%d]: Took %llu\n\n", \
@@ -294,14 +331,14 @@ feather_callback void do_sched_trace_sys_release(unsigned long id,
                 trace_litmus_server_param(sid, cid, wcet, period);      \
         } while(0)
 
-#define sched_trace_server_switch_to(sid, job, tid)                     \
+#define sched_trace_server_switch_to(sid, job, tid, tjob)               \
         do {                                                            \
-                trace_litmus_server_switch_to(sid, job, tid);           \
+                trace_litmus_server_switch_to(sid, job, tid, tjob);     \
         } while(0)
 
-#define sched_trace_server_switch_away(sid, job, tid)                   \
+#define sched_trace_server_switch_away(sid, job, tid, tjob)             \
         do {                                                            \
-                trace_litmus_server_switch_away(sid, job, tid);         \
+                trace_litmus_server_switch_away(sid, job, tid, tjob);   \
         } while (0)
 
 #define sched_trace_server_release(sid, job, rel, dead)                 \
diff --git a/include/litmus/trace.h b/include/litmus/trace.h
index c1bbf168e6d4..d868144f6928 100644
--- a/include/litmus/trace.h
+++ b/include/litmus/trace.h
@@ -23,7 +23,9 @@ struct timestamp {
         uint32_t                seq_no;
         uint8_t                 cpu;
         uint8_t                 event;
-        uint8_t                 task_type;
+        uint8_t                 task_type:2;
+        uint8_t                 irq_flag:1;
+        uint8_t                 irq_count:5;
 };
 
 /* tracing callbacks */
@@ -32,7 +34,6 @@ feather_callback void save_timestamp_def(unsigned long event, unsigned long type
 feather_callback void save_timestamp_task(unsigned long event, unsigned long t_ptr);
 feather_callback void save_timestamp_cpu(unsigned long event, unsigned long cpu);
 feather_callback void save_task_latency(unsigned long event, unsigned long when_ptr);
-feather_callback void save_timestamp_time(unsigned long event, unsigned long time_ptr);
 
 #define TIMESTAMP(id) ft_event0(id, save_timestamp)
 
@@ -47,11 +48,6 @@ feather_callback void save_timestamp_time(unsigned long event, unsigned long tim
 #define LTIMESTAMP(id, task) \
         ft_event1(id, save_task_latency, (unsigned long) task)
 
-#define TIMESTAMP_TIME(id, time_ptr) \
-        ft_event1(id, save_timestamp_time, (unsigned long) time_ptr)
-
-#define TIMESTAMP_PID(id) ft_event0(id, save_timestamp_pid)
-
 #else /* !CONFIG_SCHED_OVERHEAD_TRACE */
 
 #define TIMESTAMP(id)        /* no tracing */
@@ -64,10 +60,6 @@ feather_callback void save_timestamp_time(unsigned long event, unsigned long tim
 
 #define LTIMESTAMP(id, when_ptr) /* no tracing */
 
-#define TIMESTAMP_TIME(id, time_ptr) /* no tracing */
-
-#define TIMESTAMP_PID(id) /* no tracing */
-
 #endif
 
 
@@ -79,21 +71,6 @@ feather_callback void save_timestamp_time(unsigned long event, unsigned long tim
  * always the next number after the start time event id.
  */
 
-#define __TS_SYSCALL_IN_START(p)        TIMESTAMP_TIME(10, p)
-#define TS_SYSCALL_IN_END               TIMESTAMP_PID(11)
-
-#define TS_SYSCALL_OUT_START            TIMESTAMP_PID(20)
-#define TS_SYSCALL_OUT_END              TIMESTAMP_PID(21)
-
-#define TS_LOCK_START                   TIMESTAMP_PID(30)
-#define TS_LOCK_END                     TIMESTAMP_PID(31)
-
-#define TS_LOCK_SUSPEND                 TIMESTAMP_PID(38)
-#define TS_LOCK_RESUME                  TIMESTAMP_PID(39)
-
-#define TS_UNLOCK_START                 TIMESTAMP_PID(40)
-#define TS_UNLOCK_END                   TIMESTAMP_PID(41)
-
 #define TS_SCHED_START                  DTIMESTAMP(100, TSK_UNKNOWN) /* we only
                                                                       * care
                                                                       * about
@@ -144,6 +121,14 @@ feather_callback void save_timestamp_time(unsigned long event, unsigned long tim
 #define TS_EXIT_NP_START                TIMESTAMP(150)
 #define TS_EXIT_NP_END                  TIMESTAMP(151)
 
+#define TS_LOCK_START                   TIMESTAMP(170)
+#define TS_LOCK_SUSPEND                 TIMESTAMP(171)
+#define TS_LOCK_RESUME                  TIMESTAMP(172)
+#define TS_LOCK_END                     TIMESTAMP(173)
+
+#define TS_UNLOCK_START                 TIMESTAMP(180)
+#define TS_UNLOCK_END                   TIMESTAMP(181)
+
 #define TS_SEND_RESCHED_START(c)        CTIMESTAMP(190, c)
 #define TS_SEND_RESCHED_END             DTIMESTAMP(191, TSK_UNKNOWN)
 
diff --git a/include/litmus/trace_irq.h b/include/litmus/trace_irq.h
new file mode 100644
index 000000000000..f18b127a089d
--- /dev/null
+++ b/include/litmus/trace_irq.h
@@ -0,0 +1,21 @@
+#ifndef _LITMUS_TRACE_IRQ_H_
+#define _LITMUS_TRACE_IRQ_H_
+
+#ifdef CONFIG_SCHED_OVERHEAD_TRACE
+
+extern DEFINE_PER_CPU(atomic_t, irq_fired_count);
+
+static inline void ft_irq_fired(void)
+{
+        /* Only called with preemptions disabled.  */
+        atomic_inc(&__get_cpu_var(irq_fired_count));
+}
+
+
+#else
+
+#define ft_irq_fired() /* nothing to do */
+
+#endif
+
+#endif