Full patch for klitirqd with Nvidia GPU support.

17 files changed, 633 insertions, 26 deletions

diff --git a/include/linux/completion.h b/include/linux/completion.h
index c63950e8a863..3ce20dd3086e 100644
--- a/include/linux/completion.h
+++ b/include/linux/completion.h
@@ -76,6 +76,7 @@ static inline void init_completion(struct completion *x)
         init_waitqueue_head(&x->wait);
 }
 
+extern void __wait_for_completion_locked(struct completion *);
 extern void wait_for_completion(struct completion *);
 extern int wait_for_completion_interruptible(struct completion *x);
 extern int wait_for_completion_killable(struct completion *x);
diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index a0384a4d1e6f..5d22f5342376 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -459,6 +459,10 @@ struct tasklet_struct
         atomic_t count;
         void (*func)(unsigned long);
         unsigned long data;
+
+#ifdef CONFIG_LITMUS_SOFTIRQD
+        struct task_struct *owner;
+#endif
 };
 
 #define DECLARE_TASKLET(name, func, data) \
@@ -496,6 +500,7 @@ static inline void tasklet_unlock_wait(struct tasklet_struct *t)
 #define tasklet_unlock(t) do { } while (0)
 #endif
 
+extern void ___tasklet_schedule(struct tasklet_struct *t);
 extern void __tasklet_schedule(struct tasklet_struct *t);
 
 static inline void tasklet_schedule(struct tasklet_struct *t)
@@ -504,6 +509,7 @@ static inline void tasklet_schedule(struct tasklet_struct *t)
                 __tasklet_schedule(t);
 }
 
+extern void ___tasklet_hi_schedule(struct tasklet_struct *t);
 extern void __tasklet_hi_schedule(struct tasklet_struct *t);
 
 static inline void tasklet_hi_schedule(struct tasklet_struct *t)
@@ -512,6 +518,7 @@ static inline void tasklet_hi_schedule(struct tasklet_struct *t)
                 __tasklet_hi_schedule(t);
 }
 
+extern void ___tasklet_hi_schedule_first(struct tasklet_struct *t);
 extern void __tasklet_hi_schedule_first(struct tasklet_struct *t);
 
 /*
@@ -541,7 +548,7 @@ static inline void tasklet_disable(struct tasklet_struct *t)
 }
 
 static inline void tasklet_enable(struct tasklet_struct *t)
-{
+{       
         smp_mb__before_atomic_dec();
         atomic_dec(&t->count);
 }
diff --git a/include/linux/mutex.h b/include/linux/mutex.h
index f363bc8fdc74..9f3199571994 100644
--- a/include/linux/mutex.h
+++ b/include/linux/mutex.h
@@ -126,6 +126,15 @@ static inline int mutex_is_locked(struct mutex *lock)
         return atomic_read(&lock->count) != 1;
 }
 
+/* return non-zero to abort.  only pre-side-effects may abort */
+typedef int (*side_effect_t)(unsigned long);
+extern void mutex_lock_sfx(struct mutex *lock,
+                                                   side_effect_t pre, unsigned long pre_arg,
+                                                   side_effect_t post, unsigned long post_arg);
+extern void mutex_unlock_sfx(struct mutex *lock,
+                                                         side_effect_t pre, unsigned long pre_arg,
+                                                         side_effect_t post, unsigned long post_arg);
+
 /*
  * See kernel/mutex.c for detailed documentation of these APIs.
  * Also see Documentation/mutex-design.txt.
@@ -145,6 +154,7 @@ extern void mutex_lock(struct mutex *lock);
 extern int __must_check mutex_lock_interruptible(struct mutex *lock);
 extern int __must_check mutex_lock_killable(struct mutex *lock);
 
+
 # define mutex_lock_nested(lock, subclass) mutex_lock(lock)
 # define mutex_lock_interruptible_nested(lock, subclass) mutex_lock_interruptible(lock)
 # define mutex_lock_killable_nested(lock, subclass) mutex_lock_killable(lock)
diff --git a/include/linux/semaphore.h b/include/linux/semaphore.h
index 5310d27abd2a..69e3f57661ec 100644
--- a/include/linux/semaphore.h
+++ b/include/linux/semaphore.h
@@ -49,4 +49,13 @@ extern int __must_check down_trylock(struct semaphore *sem);
 extern int __must_check down_timeout(struct semaphore *sem, long jiffies);
 extern void up(struct semaphore *sem);
 
+extern void __down(struct semaphore *sem);
+extern void __up(struct semaphore *sem);
+
+struct semaphore_waiter {
+        struct list_head list;
+        struct task_struct *task;
+        int up;
+};
+
 #endif /* __LINUX_SEMAPHORE_H */
diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h
index 25e02c941bac..5fecfb375eeb 100644
--- a/include/linux/workqueue.h
+++ b/include/linux/workqueue.h
@@ -83,6 +83,9 @@ struct work_struct {
 #ifdef CONFIG_LOCKDEP
         struct lockdep_map lockdep_map;
 #endif
+#ifdef CONFIG_LITMUS_SOFTIRQD
+        struct task_struct *owner;
+#endif
 };
 
 #define WORK_DATA_INIT()        ATOMIC_LONG_INIT(WORK_STRUCT_NO_CPU)
@@ -115,11 +118,25 @@ struct execute_work {
 #define __WORK_INIT_LOCKDEP_MAP(n, k)
 #endif
 
+#ifdef CONFIG_LITMUS_SOFTIRQD
+#define __WORK_INIT_OWNER() \
+        .owner = NULL,
+
+#define PREPARE_OWNER(_work, _owner) \
+        do { \
+                (_work)->owner = (_owner); \
+        } while(0)
+#else
+#define __WORK_INIT_OWNER()
+#define PREPARE_OWNER(_work, _owner)
+#endif
+
 #define __WORK_INITIALIZER(n, f) {                              \
         .data = WORK_DATA_STATIC_INIT(),                        \
         .entry  = { &(n).entry, &(n).entry },                   \
         .func = (f),                                            \
         __WORK_INIT_LOCKDEP_MAP(#n, &(n))                       \
+        __WORK_INIT_OWNER() \
         }
 
 #define __DELAYED_WORK_INITIALIZER(n, f) {                      \
@@ -327,6 +344,7 @@ extern void flush_workqueue(struct workqueue_struct *wq);
 extern void flush_scheduled_work(void);
 extern void flush_delayed_work(struct delayed_work *work);
 
+extern int __schedule_work(struct work_struct *work);
 extern int schedule_work(struct work_struct *work);
 extern int schedule_work_on(int cpu, struct work_struct *work);
 extern int schedule_delayed_work(struct delayed_work *work, unsigned long delay);
diff --git a/include/litmus/affinity.h b/include/litmus/affinity.h
new file mode 100644
index 000000000000..877b4099c6e2
--- /dev/null
+++ b/include/litmus/affinity.h
@@ -0,0 +1,78 @@
+#ifndef __LITMUS_AFFINITY_H
+#define __LITMUS_AFFINITY_H
+
+#include <linux/cpumask.h>
+
+/*
+  L1 (instr) = depth 0
+  L1 (data)  = depth 1
+  L2 = depth 2
+  L3 = depth 3
+ */
+#define NUM_CACHE_LEVELS 4
+
+struct neighborhood
+{
+        unsigned int size[NUM_CACHE_LEVELS];
+        cpumask_var_t neighbors[NUM_CACHE_LEVELS];
+};
+
+/* topology info is stored redundently in a big array for fast lookups */
+extern struct neighborhood neigh_info[NR_CPUS];
+
+void init_topology(void); /* called by Litmus module's _init_litmus() */
+
+/* Works like:
+void get_nearest_available_cpu(cpu_entry_t* nearest, cpu_entry_t* start, cpu_entry_t* entries, int release_master)
+
+Set release_master = -1 for no RM.
+ */
+#define get_nearest_available_cpu(nearest, start, entries, release_master) \
+{ \
+        (nearest) = NULL; \
+        if(!(start)->linked) \
+        { \
+                (nearest) = (start); \
+        } \
+        else \
+        { \
+                int __level; \
+                int __cpu; \
+                struct neighborhood* __neighbors = &neigh_info[(start)->cpu]; \
+                \
+                for(__level = 0; (__level < NUM_CACHE_LEVELS) && !(nearest); ++__level) \
+                { \
+                        if(__neighbors->size[__level] > 1) \
+                        { \
+                                for_each_cpu(__cpu, __neighbors->neighbors[__level]) \
+                                { \
+                                        if(__cpu != (release_master)) \
+                                        { \
+                                                cpu_entry_t* __entry = &per_cpu((entries), __cpu); \
+                                                if(!__entry->linked) \
+                                                { \
+                                                        (nearest) = __entry; \
+                                                        break; \
+                                                } \
+                                        } \
+                                } \
+                        } \
+                        else if(__neighbors->size[__level] == 0) \
+                        { \
+                                break; \
+                        } \
+                } \
+        } \
+        \
+        if((nearest)) \
+        { \
+                TRACE("P%d is closest available CPU to P%d\n", (nearest)->cpu, (start)->cpu); \
+        } \
+        else \
+        { \
+                TRACE("Could not find an available CPU close to P%d\n", \
+                                                (start)->cpu); \
+        } \
+}
+
+#endif
diff --git a/include/litmus/fdso.h b/include/litmus/fdso.h
index caf2a1e6918c..c740e8fc3e88 100644
--- a/include/litmus/fdso.h
+++ b/include/litmus/fdso.h
@@ -18,9 +18,10 @@ typedef enum  {
         MIN_OBJ_TYPE    = 0,
 
         FMLP_SEM        = 0,
-        SRP_SEM         = 1,
+        KFMLP_SEM       = 1,
+        SRP_SEM         = 2,
 
-        MAX_OBJ_TYPE    = 1
+        MAX_OBJ_TYPE    = SRP_SEM
 } obj_type_t;
 
 struct inode_obj_id {
@@ -64,6 +65,7 @@ static inline void* od_lookup(int od, obj_type_t type)
 }
 
 #define lookup_fmlp_sem(od)((struct pi_semaphore*)  od_lookup(od, FMLP_SEM))
+#define lookup_kfmlp_sem(od)((struct pi_semaphore*)  od_lookup(od, KFMLP_SEM))
 #define lookup_srp_sem(od) ((struct srp_semaphore*) od_lookup(od, SRP_SEM))
 #define lookup_ics(od)     ((struct ics*)           od_lookup(od, ICS_ID))
 
diff --git a/include/litmus/litmus.h b/include/litmus/litmus.h
index e7769ca36ec0..3df242bf272f 100644
--- a/include/litmus/litmus.h
+++ b/include/litmus/litmus.h
@@ -26,6 +26,7 @@ static inline int in_list(struct list_head* list)
                 );
 }
 
+
 struct task_struct* __waitqueue_remove_first(wait_queue_head_t *wq);
 
 #define NO_CPU                  0xffffffff
diff --git a/include/litmus/litmus_softirq.h b/include/litmus/litmus_softirq.h
new file mode 100644
index 000000000000..34287f3cbb8d
--- /dev/null
+++ b/include/litmus/litmus_softirq.h
@@ -0,0 +1,199 @@
+#ifndef __LITMUS_SOFTIRQ_H
+#define __LITMUS_SOFTIRQ_H
+
+#include <linux/interrupt.h>
+#include <linux/workqueue.h>
+
+/*
+   Threaded tasklet handling for Litmus.  Tasklets
+   are scheduled with the priority of the tasklet's
+   owner---that is, the RT task on behalf the tasklet
+   runs.
+ 
+   Tasklets are current scheduled in FIFO order with
+   NO priority inheritance for "blocked" tasklets.
+ 
+   klitirqd assumes the priority of the owner of the
+   tasklet when the tasklet is next to execute.
+ 
+   Currently, hi-tasklets are scheduled before
+   low-tasklets, regardless of priority of low-tasklets.
+   And likewise, low-tasklets are scheduled before work
+   queue objects.  This priority inversion probably needs
+   to be fixed, though it is not an issue if our work with
+   GPUs as GPUs are owned (and associated klitirqds) for
+   exclusive time periods, thus no inversions can
+   occur.
+ */
+
+
+
+#define NR_LITMUS_SOFTIRQD CONFIG_NR_LITMUS_SOFTIRQD
+
+/* Spawns NR_LITMUS_SOFTIRQD klitirqd daemons.
+   Actual launch of threads is deffered to kworker's
+   workqueue, so daemons will likely not be immediately
+   running when this function returns, though the required
+   data will be initialized.
+ 
+   @affinity_set: an array expressing the processor affinity
+    for each of the NR_LITMUS_SOFTIRQD daemons.  May be set
+    to NULL for global scheduling.
+ 
+        - Examples -
+        8-CPU system with two CPU clusters:
+                affinity[] = {0, 0, 0, 0, 3, 3, 3, 3}
+                NOTE: Daemons not actually bound to specified CPU, but rather
+                cluster in which the CPU resides.
+ 
+        8-CPU system, partitioned:
+                affinity[] = {0, 1, 2, 3, 4, 5, 6, 7}
+ 
+        FIXME: change array to a CPU topology or array of cpumasks
+ 
+ */
+void spawn_klitirqd(int* affinity);
+
+
+/* Raises a flag to tell klitirqds to terminate.
+   Termination is async, so some threads may be running
+   after function return. */
+void kill_klitirqd(void);
+
+
+/* Returns 1 if all NR_LITMUS_SOFTIRQD klitirqs are ready
+   to handle tasklets. 0, otherwise.*/
+int klitirqd_is_ready(void);
+
+/* Returns 1 if no NR_LITMUS_SOFTIRQD klitirqs are ready
+   to handle tasklets. 0, otherwise.*/
+int klitirqd_is_dead(void);
+
+/* Flushes all pending work out to the OS for regular
+ * tasklet/work processing of the specified 'owner'
+ *
+ * PRECOND: klitirqd_thread must have a clear entry
+ * in the GPU registry, otherwise this call will become
+ * a no-op as work will loop back to the klitirqd_thread.
+ *
+ * Pass NULL for owner to flush ALL pending items.
+ */
+void flush_pending(struct task_struct* klitirqd_thread,
+                                   struct task_struct* owner);
+
+struct task_struct* get_klitirqd(unsigned int k_id);
+
+
+extern int __litmus_tasklet_schedule(
+        struct tasklet_struct *t,
+        unsigned int k_id);
+
+/* schedule a tasklet on klitirqd #k_id */
+static inline int litmus_tasklet_schedule(
+    struct tasklet_struct *t,
+    unsigned int k_id)
+{
+        int ret = 0;
+        if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))
+                ret = __litmus_tasklet_schedule(t, k_id);
+        return(ret);
+}
+
+/* for use by __tasklet_schedule() */
+static inline int _litmus_tasklet_schedule(
+    struct tasklet_struct *t,
+    unsigned int k_id)
+{
+    return(__litmus_tasklet_schedule(t, k_id));
+}
+
+
+
+
+extern int __litmus_tasklet_hi_schedule(struct tasklet_struct *t,
+                                         unsigned int k_id);
+
+/* schedule a hi tasklet on klitirqd #k_id */
+static inline int litmus_tasklet_hi_schedule(struct tasklet_struct *t,
+                                              unsigned int k_id)
+{
+        int ret = 0;
+        if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))
+                ret = __litmus_tasklet_hi_schedule(t, k_id);
+        return(ret);
+}
+
+/* for use by __tasklet_hi_schedule() */
+static inline int _litmus_tasklet_hi_schedule(struct tasklet_struct *t,
+                                               unsigned int k_id)
+{
+    return(__litmus_tasklet_hi_schedule(t, k_id));
+}
+
+
+
+
+
+extern int __litmus_tasklet_hi_schedule_first(
+    struct tasklet_struct *t,
+    unsigned int k_id);
+
+/* schedule a hi tasklet on klitirqd #k_id on next go-around */
+/* PRECONDITION: Interrupts must be disabled. */
+static inline int litmus_tasklet_hi_schedule_first(
+    struct tasklet_struct *t,
+    unsigned int k_id)
+{
+        int ret = 0;
+        if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))
+                ret = __litmus_tasklet_hi_schedule_first(t, k_id);
+        return(ret);
+}
+
+/* for use by __tasklet_hi_schedule_first() */
+static inline int _litmus_tasklet_hi_schedule_first(
+    struct tasklet_struct *t,
+    unsigned int k_id)
+{
+    return(__litmus_tasklet_hi_schedule_first(t, k_id));
+}
+
+
+
+//////////////
+
+extern int __litmus_schedule_work(
+        struct work_struct* w,
+        unsigned int k_id);
+
+static inline int litmus_schedule_work(
+        struct work_struct* w,
+        unsigned int k_id)
+{
+        return(__litmus_schedule_work(w, k_id));
+}
+
+
+
+///////////// mutex operations for client threads.
+ 
+void down_and_set_stat(struct task_struct* t,
+                                         enum klitirqd_sem_status to_set,
+                                         struct mutex* sem);
+
+void __down_and_reset_and_set_stat(struct task_struct* t,
+                                enum klitirqd_sem_status to_reset,
+                                enum klitirqd_sem_status to_set,
+                                struct mutex* sem);
+
+void up_and_set_stat(struct task_struct* t,
+                                        enum klitirqd_sem_status to_set,
+                                        struct mutex* sem);
+
+
+
+void release_klitirqd_lock(struct task_struct* t);
+
+int reacquire_klitirqd_lock(struct task_struct* t);
+
+#endif
diff --git a/include/litmus/nvidia_info.h b/include/litmus/nvidia_info.h
new file mode 100644
index 000000000000..579301d77cf5
--- /dev/null
+++ b/include/litmus/nvidia_info.h
@@ -0,0 +1,37 @@
+#ifndef __LITMUS_NVIDIA_H
+#define __LITMUS_NVIDIA_H
+
+#include <linux/interrupt.h>
+
+
+#include <litmus/litmus_softirq.h>
+
+
+#define NV_DEVICE_NUM NR_LITMUS_SOFTIRQD
+
+int init_nvidia_info(void);
+
+int is_nvidia_func(void* func_addr);
+
+void dump_nvidia_info(const struct tasklet_struct *t);
+
+
+// Returns the Nvidia device # associated with provided tasklet and work_struct.
+u32 get_tasklet_nv_device_num(const struct tasklet_struct *t);
+u32 get_work_nv_device_num(const struct work_struct *t);
+
+
+int init_nv_device_reg(void);
+//int get_nv_device_id(struct task_struct* owner);
+
+
+int reg_nv_device(int reg_device_id, int register_device);
+
+struct task_struct* get_nv_device_owner(u32 target_device_id);
+
+void lock_nv_registry(u32 reg_device_id, unsigned long* flags);
+void unlock_nv_registry(u32 reg_device_id, unsigned long* flags);
+
+void increment_nv_int_count(u32 device);
+
+#endif
diff --git a/include/litmus/preempt.h b/include/litmus/preempt.h
index 260c6fe17986..244924f93c48 100644
--- a/include/litmus/preempt.h
+++ b/include/litmus/preempt.h
@@ -26,6 +26,7 @@ const char* sched_state_name(int s);
                                     (x), #x, __FUNCTION__);             \
         } while (0);
 
+//#define TRACE_SCHED_STATE_CHANGE(x, y, cpu) /* ignore */
 #define TRACE_SCHED_STATE_CHANGE(x, y, cpu)                             \
         TRACE_STATE("[P%d] 0x%x (%s) -> 0x%x (%s)\n",                   \
                     cpu,  (x), sched_state_name(x),                     \
diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h
index 5de422c742f6..53af3ce1d955 100644
--- a/include/litmus/rt_param.h
+++ b/include/litmus/rt_param.h
@@ -69,6 +69,8 @@ struct control_page {
 /* don't export internal data structures to user space (liblitmus) */
 #ifdef __KERNEL__
 
+#include <linux/semaphore.h>
+
 struct _rt_domain;
 struct bheap_node;
 struct release_heap;
@@ -94,6 +96,14 @@ struct rt_job {
 
 struct pfair_param;
 
+enum klitirqd_sem_status
+{
+        NEED_TO_REACQUIRE,
+        REACQUIRING,
+        NOT_HELD,
+        HELD
+};
+
 /*      RT task parameters for scheduling extensions
  *      These parameters are inherited during clone and therefore must
  *      be explicitly set up before the task set is launched.
@@ -108,6 +118,38 @@ struct rt_param {
         /* is the task present? (true if it can be scheduled) */
         unsigned int            present:1;
 
+#ifdef CONFIG_LITMUS_SOFTIRQD
+    /* proxy threads have minimum priority by default */
+    unsigned int        is_proxy_thread:1;
+    
+        /* pointer to klitirqd currently working on this
+           task_struct's behalf.  only set by the task pointed
+           to by klitirqd.
+         
+           ptr only valid if is_proxy_thread == 0
+         */
+        struct task_struct* cur_klitirqd;
+
+        /* Used to implement mutual execution exclusion between
+         * job and klitirqd execution.  Job must always hold
+         * it's klitirqd_sem to execute.  klitirqd instance
+         * must hold the semaphore before executing on behalf
+         * of a job.
+         */
+        //struct semaphore                      klitirqd_sem;
+        struct mutex                            klitirqd_sem;
+
+        /* status of held klitirqd_sem, even if the held klitirqd_sem is from
+           another task (only proxy threads do this though).
+         */
+        atomic_t                                        klitirqd_sem_stat;
+#endif
+
+#ifdef CONFIG_LITMUS_NVIDIA
+        /* number of top-half interrupts handled on behalf of current job */
+        atomic_t                                        nv_int_count;
+#endif
+
 #ifdef CONFIG_LITMUS_LOCKING
         /* Is the task being priority-boosted by a locking protocol? */
         unsigned int            priority_boosted:1;
@@ -128,7 +170,7 @@ struct rt_param {
          * an increased task priority.
          */
          struct task_struct*    inh_task;
-
+    
 #ifdef CONFIG_NP_SECTION
         /* For the FMLP under PSN-EDF, it is required to make the task
          * non-preemptive from kernel space. In order not to interfere with
diff --git a/include/litmus/sched_plugin.h b/include/litmus/sched_plugin.h
index 6e7cabdddae8..df50930d14a0 100644
--- a/include/litmus/sched_plugin.h
+++ b/include/litmus/sched_plugin.h
@@ -29,7 +29,6 @@ typedef struct task_struct* (*schedule_t)(struct task_struct * prev);
  */
 typedef void (*finish_switch_t)(struct task_struct *prev);
 
-
 /********************* task state changes ********************/
 
 /* Called to setup a new real-time task.
@@ -58,6 +57,17 @@ typedef void (*task_exit_t)    (struct task_struct *);
 typedef long (*allocate_lock_t) (struct litmus_lock **lock, int type,
                                  void* __user config);
 
+/* Called to change inheritance levels of given task */
+typedef void (*set_prio_inh_t)(struct task_struct* t,
+                               struct task_struct* prio_inh);
+typedef void (*clear_prio_inh_t)(struct task_struct* t);
+
+
+typedef void (*set_prio_inh_klitirq_t)(struct task_struct* klitirqd,
+                                       struct task_struct* old_owner,
+                                       struct task_struct* new_owner);
+typedef void (*clear_prio_inh_klitirqd_t)(struct task_struct* klitirqd,
+                                          struct task_struct* old_owner);
 
 /********************* sys call backends  ********************/
 /* This function causes the caller to sleep until the next release */
@@ -88,7 +98,7 @@ struct sched_plugin {
         /*      task state changes      */
         admit_task_t            admit_task;
 
-        task_new_t              task_new;
+    task_new_t                  task_new;
         task_wake_up_t          task_wake_up;
         task_block_t            task_block;
         task_exit_t             task_exit;
@@ -96,6 +106,14 @@ struct sched_plugin {
 #ifdef CONFIG_LITMUS_LOCKING
         /*      locking protocols       */
         allocate_lock_t         allocate_lock;
+    
+    set_prio_inh_t      set_prio_inh;
+    clear_prio_inh_t    clear_prio_inh;
+#endif
+    
+#ifdef CONFIG_LITMUS_SOFTIRQD
+    set_prio_inh_klitirq_t              set_prio_inh_klitirqd;
+    clear_prio_inh_klitirqd_t   clear_prio_inh_klitirqd;
 #endif
 } __attribute__ ((__aligned__(SMP_CACHE_BYTES)));
 
diff --git a/include/litmus/sched_trace.h b/include/litmus/sched_trace.h
index 7ca34cb13881..1486c778aff8 100644
--- a/include/litmus/sched_trace.h
+++ b/include/litmus/sched_trace.h
@@ -11,12 +11,12 @@ struct st_trace_header {
         u8      cpu;            /* On which CPU was it recorded? */
         u16     pid;            /* PID of the task.              */
         u32     job;            /* The job sequence number.      */
-};
+} __attribute__((packed));
 
 #define ST_NAME_LEN 16
 struct st_name_data {
         char    cmd[ST_NAME_LEN];/* The name of the executable of this process. */
-};
+} __attribute__((packed));
 
 struct st_param_data {          /* regular params */
         u32     wcet;
@@ -25,30 +25,29 @@ struct st_param_data {		/* regular params */
         u8      partition;
         u8      class;
         u8      __unused[2];
-};
+} __attribute__((packed));
 
 struct st_release_data {        /* A job is was/is going to be released. */
         u64     release;        /* What's the release time?              */
         u64     deadline;       /* By when must it finish?               */
-};
+} __attribute__((packed));
 
 struct st_assigned_data {       /* A job was asigned to a CPU.           */
         u64     when;
         u8      target;         /* Where should it execute?              */
         u8      __unused[7];
-};
+} __attribute__((packed));
 
 struct st_switch_to_data {      /* A process was switched to on a given CPU.   */
         u64     when;           /* When did this occur?                        */
         u32     exec_time;      /* Time the current job has executed.          */
         u8      __unused[4];
-
-};
+} __attribute__((packed));
 
 struct st_switch_away_data {    /* A process was switched away from on a given CPU. */
         u64     when;
         u64     exec_time;
-};
+} __attribute__((packed));
 
 struct st_completion_data {     /* A job completed. */
         u64     when;
@@ -56,35 +55,92 @@ struct st_completion_data {	/* A job completed. */
                                  * next task automatically; set to 0 otherwise.
                                  */
         u8      __uflags:7;
-        u8      __unused[7];
-};
+        u16 nv_int_count;
+        u8      __unused[5];
+} __attribute__((packed));
 
 struct st_block_data {          /* A task blocks. */
         u64     when;
         u64     __unused;
-};
+} __attribute__((packed));
 
 struct st_resume_data {         /* A task resumes. */
         u64     when;
         u64     __unused;
-};
+} __attribute__((packed));
 
 struct st_action_data {
         u64     when;
         u8      action;
         u8      __unused[7];
-};
+} __attribute__((packed));
 
 struct st_sys_release_data {
         u64     when;
         u64     release;
-};
+} __attribute__((packed));
+
+
+struct st_tasklet_release_data {
+        u64 when;
+        u64 __unused;
+} __attribute__((packed));
+
+struct st_tasklet_begin_data {
+        u64 when;
+        u16 exe_pid;
+        u8  __unused[6];
+} __attribute__((packed));
+
+struct st_tasklet_end_data {
+        u64 when;
+        u16 exe_pid;
+        u8      flushed;
+        u8      __unused[5];
+} __attribute__((packed));
+
+
+struct st_work_release_data {
+        u64 when;
+        u64 __unused;
+} __attribute__((packed));
+
+struct st_work_begin_data {
+        u64 when;
+        u16 exe_pid;
+        u8      __unused[6];
+} __attribute__((packed));
+
+struct st_work_end_data {
+        u64 when;
+        u16 exe_pid;
+        u8      flushed;
+        u8      __unused[5];
+} __attribute__((packed));
+
+struct st_effective_priority_change_data {
+        u64 when;
+        u16 inh_pid;
+        u8      __unused[6];
+} __attribute__((packed));
+
+struct st_nv_interrupt_begin_data {
+        u64 when;
+        u32 device;
+        u8  __unused[4];
+} __attribute__((packed));
+
+struct st_nv_interrupt_end_data {
+        u64 when;
+        u32 device;
+        u8  __unused[4];
+} __attribute__((packed));
 
 #define DATA(x) struct st_ ## x ## _data x;
 
 typedef enum {
-        ST_NAME = 1,            /* Start at one, so that we can spot
-                                 * uninitialized records. */
+    ST_NAME = 1, /* Start at one, so that we can spot
+                                  * uninitialized records. */
         ST_PARAM,
         ST_RELEASE,
         ST_ASSIGNED,
@@ -94,7 +150,16 @@ typedef enum {
         ST_BLOCK,
         ST_RESUME,
         ST_ACTION,
-        ST_SYS_RELEASE
+        ST_SYS_RELEASE,
+        ST_TASKLET_RELEASE,
+        ST_TASKLET_BEGIN,
+        ST_TASKLET_END,
+        ST_WORK_RELEASE,
+        ST_WORK_BEGIN,
+        ST_WORK_END,
+        ST_EFF_PRIO_CHANGE,
+        ST_NV_INTERRUPT_BEGIN,
+        ST_NV_INTERRUPT_END,
 } st_event_record_type_t;
 
 struct st_event_record {
@@ -113,8 +178,17 @@ struct st_event_record {
                 DATA(resume);
                 DATA(action);
                 DATA(sys_release);
+                DATA(tasklet_release);
+                DATA(tasklet_begin);
+                DATA(tasklet_end);
+                DATA(work_release);
+                DATA(work_begin);
+                DATA(work_end);
+                DATA(effective_priority_change);
+                DATA(nv_interrupt_begin);
+                DATA(nv_interrupt_end);
         } data;
-};
+} __attribute__((packed));
 
 #undef DATA
 
@@ -129,6 +203,8 @@ struct st_event_record {
         ft_event1(id, callback, task)
 #define SCHED_TRACE2(id, callback, task, xtra) \
         ft_event2(id, callback, task, xtra)
+#define SCHED_TRACE3(id, callback, task, xtra1, xtra2) \
+        ft_event3(id, callback, task, xtra1, xtra2)
 
 /* provide prototypes; needed on sparc64 */
 #ifndef NO_TASK_TRACE_DECLS
@@ -155,12 +231,45 @@ feather_callback void do_sched_trace_action(unsigned long id,
 feather_callback void do_sched_trace_sys_release(unsigned long id,
                                                  lt_t* start);
 
+
+feather_callback void do_sched_trace_tasklet_release(unsigned long id,
+                                                                                                   struct task_struct* owner);
+feather_callback void do_sched_trace_tasklet_begin(unsigned long id,
+                                                                                                  struct task_struct* owner);
+feather_callback void do_sched_trace_tasklet_end(unsigned long id,
+                                                                                                 struct task_struct* owner,
+                                                                                                 unsigned long flushed);
+
+feather_callback void do_sched_trace_work_release(unsigned long id,
+                                                                                                         struct task_struct* owner);
+feather_callback void do_sched_trace_work_begin(unsigned long id,
+                                                                                                struct task_struct* owner,
+                                                                                                struct task_struct* exe);
+feather_callback void do_sched_trace_work_end(unsigned long id,
+                                                                                          struct task_struct* owner,
+                                                                                          struct task_struct* exe,
+                                                                                          unsigned long flushed);
+
+feather_callback void do_sched_trace_eff_prio_change(unsigned long id,
+                                                                                          struct task_struct* task,
+                                                                                          struct task_struct* inh);
+
+feather_callback void do_sched_trace_nv_interrupt_begin(unsigned long id,
+                                                                                                u32 device);
+feather_callback void do_sched_trace_nv_interrupt_end(unsigned long id,
+                                                                                                unsigned long unused);
+
+
+/* returns true if we're tracing an interrupt on current CPU */
+/* int is_interrupt_tracing_active(void); */
+
 #endif
 
 #else
 
 #define SCHED_TRACE(id, callback, task)        /* no tracing */
 #define SCHED_TRACE2(id, callback, task, xtra) /* no tracing */
+#define SCHED_TRACE3(id, callback, task, xtra1, xtra2)
 
 #endif
 
@@ -193,6 +302,35 @@ feather_callback void do_sched_trace_sys_release(unsigned long id,
         SCHED_TRACE(SCHED_TRACE_BASE_ID + 10, do_sched_trace_sys_release, when)
 
 
+#define sched_trace_tasklet_release(t) \
+        SCHED_TRACE(SCHED_TRACE_BASE_ID + 11, do_sched_trace_tasklet_release, t)
+
+#define sched_trace_tasklet_begin(t) \
+        SCHED_TRACE(SCHED_TRACE_BASE_ID + 12, do_sched_trace_tasklet_begin, t)
+
+#define sched_trace_tasklet_end(t, flushed) \
+        SCHED_TRACE2(SCHED_TRACE_BASE_ID + 13, do_sched_trace_tasklet_end, t, flushed)
+
+
+#define sched_trace_work_release(t) \
+        SCHED_TRACE(SCHED_TRACE_BASE_ID + 14, do_sched_trace_work_release, t)
+
+#define sched_trace_work_begin(t, e) \
+        SCHED_TRACE2(SCHED_TRACE_BASE_ID + 15, do_sched_trace_work_begin, t, e)
+
+#define sched_trace_work_end(t, e, flushed) \
+        SCHED_TRACE3(SCHED_TRACE_BASE_ID + 16, do_sched_trace_work_end, t, e, flushed)
+
+
+#define sched_trace_eff_prio_change(t, inh) \
+        SCHED_TRACE2(SCHED_TRACE_BASE_ID + 17, do_sched_trace_eff_prio_change, t, inh)
+
+
+#define sched_trace_nv_interrupt_begin(d) \
+        SCHED_TRACE(SCHED_TRACE_BASE_ID + 18, do_sched_trace_nv_interrupt_begin, d)
+#define sched_trace_nv_interrupt_end() \
+        SCHED_TRACE(SCHED_TRACE_BASE_ID + 19, do_sched_trace_nv_interrupt_end, 0ul)
+
 #define sched_trace_quantum_boundary() /* NOT IMPLEMENTED */
 
 #endif /* __KERNEL__ */
diff --git a/include/litmus/sched_trace_external.h b/include/litmus/sched_trace_external.h
new file mode 100644
index 000000000000..c2c872639880
--- /dev/null
+++ b/include/litmus/sched_trace_external.h
@@ -0,0 +1,42 @@
+/*
+ * sched_trace.h -- record scheduler events to a byte stream for offline analysis.
+ */
+#ifndef _LINUX_SCHED_TRACE_EXTERNAL_H_
+#define _LINUX_SCHED_TRACE_EXTERNAL_H_
+
+extern void __sched_trace_tasklet_begin_external(struct task_struct* t);
+static inline void sched_trace_tasklet_begin_external(struct task_struct* t)
+{
+        __sched_trace_tasklet_begin_external(t);
+}
+
+extern void __sched_trace_tasklet_end_external(struct task_struct* t, unsigned long flushed);
+static inline void sched_trace_tasklet_end_external(struct task_struct* t, unsigned long flushed)
+{
+        __sched_trace_tasklet_end_external(t, flushed);
+}
+
+extern void __sched_trace_work_begin_external(struct task_struct* t, struct task_struct* e);
+static inline void sched_trace_work_begin_external(struct task_struct* t, struct task_struct* e)
+{
+        __sched_trace_work_begin_external(t, e);
+}
+
+extern void __sched_trace_work_end_external(struct task_struct* t, struct task_struct* e, unsigned long f);
+static inline void sched_trace_work_end_external(struct task_struct* t, struct task_struct* e, unsigned long f)
+{
+        __sched_trace_work_end_external(t, e, f);
+}
+
+extern void __sched_trace_nv_interrupt_begin_external(u32 device);
+static inline void sched_trace_nv_interrupt_begin_external(u32 device)
+{
+        __sched_trace_nv_interrupt_begin_external(device);
+}
+
+extern void __sched_trace_nv_interrupt_end_external(void);
+static inline void sched_trace_nv_interrupt_end_external(void)
+{
+        __sched_trace_nv_interrupt_end_external();
+}
+#endif
diff --git a/include/litmus/unistd_32.h b/include/litmus/unistd_32.h
index 94264c27d9ac..c6efc4c40af2 100644
--- a/include/litmus/unistd_32.h
+++ b/include/litmus/unistd_32.h
@@ -17,5 +17,6 @@
 #define __NR_wait_for_ts_release __LSC(9)
 #define __NR_release_ts         __LSC(10)
 #define __NR_null_call          __LSC(11)
+#define __NR_register_nv_device                 __LSC(12)
 
-#define NR_litmus_syscalls 12
+#define NR_litmus_syscalls 13
diff --git a/include/litmus/unistd_64.h b/include/litmus/unistd_64.h
index d5ced0d2642c..b44a7c33bdf8 100644
--- a/include/litmus/unistd_64.h
+++ b/include/litmus/unistd_64.h
@@ -29,5 +29,8 @@ __SYSCALL(__NR_wait_for_ts_release, sys_wait_for_ts_release)
 __SYSCALL(__NR_release_ts, sys_release_ts)
 #define __NR_null_call                          __LSC(11)
 __SYSCALL(__NR_null_call, sys_null_call)
+#define __NR_register_nv_device                 __LSC(12)
+__SYSCALL(__NR_register_nv_device, sys_register_nv_device)
 
-#define NR_litmus_syscalls 12
+
+#define NR_litmus_syscalls 13