Merge branch 'wip-gpu-rtas12' into wip-slave-threads

Conflicts:
	include/litmus/unistd_32.h
	include/litmus/unistd_64.h
	litmus/litmus.c

27 files changed, 1596 insertions, 41 deletions

diff --git a/include/linux/completion.h b/include/linux/completion.h
index 9d727271c9fe..cff405c4dd3a 100644
--- a/include/linux/completion.h
+++ b/include/linux/completion.h
@@ -76,6 +76,7 @@ static inline void init_completion(struct completion *x)
         init_waitqueue_head(&x->wait);
 }
 
+extern void __wait_for_completion_locked(struct completion *);
 extern void wait_for_completion(struct completion *);
 extern int wait_for_completion_interruptible(struct completion *x);
 extern int wait_for_completion_killable(struct completion *x);
diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index f6efed0039ed..57a7bc8807be 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -445,6 +445,7 @@ static inline void __raise_softirq_irqoff(unsigned int nr)
 
 extern void raise_softirq_irqoff(unsigned int nr);
 extern void raise_softirq(unsigned int nr);
+extern void wakeup_softirqd(void);
 
 /* This is the worklist that queues up per-cpu softirq work.
  *
@@ -500,6 +501,10 @@ struct tasklet_struct
         atomic_t count;
         void (*func)(unsigned long);
         unsigned long data;
+
+#if defined(CONFIG_LITMUS_SOFTIRQD) || defined(CONFIG_LITMUS_PAI_SOFTIRQD)
+        struct task_struct *owner;
+#endif
 };
 
 #define DECLARE_TASKLET(name, func, data) \
@@ -537,6 +542,7 @@ static inline void tasklet_unlock_wait(struct tasklet_struct *t)
 #define tasklet_unlock(t) do { } while (0)
 #endif
 
+extern void ___tasklet_schedule(struct tasklet_struct *t);
 extern void __tasklet_schedule(struct tasklet_struct *t);
 
 static inline void tasklet_schedule(struct tasklet_struct *t)
@@ -545,6 +551,7 @@ static inline void tasklet_schedule(struct tasklet_struct *t)
                 __tasklet_schedule(t);
 }
 
+extern void ___tasklet_hi_schedule(struct tasklet_struct *t);
 extern void __tasklet_hi_schedule(struct tasklet_struct *t);
 
 static inline void tasklet_hi_schedule(struct tasklet_struct *t)
@@ -553,6 +560,7 @@ static inline void tasklet_hi_schedule(struct tasklet_struct *t)
                 __tasklet_hi_schedule(t);
 }
 
+extern void ___tasklet_hi_schedule_first(struct tasklet_struct *t);
 extern void __tasklet_hi_schedule_first(struct tasklet_struct *t);
 
 /*
@@ -582,7 +590,7 @@ static inline void tasklet_disable(struct tasklet_struct *t)
 }
 
 static inline void tasklet_enable(struct tasklet_struct *t)
-{
+{       
         smp_mb__before_atomic_dec();
         atomic_dec(&t->count);
 }
diff --git a/include/linux/mutex.h b/include/linux/mutex.h
index a940fe435aca..cb47debbf24d 100644
--- a/include/linux/mutex.h
+++ b/include/linux/mutex.h
@@ -126,6 +126,15 @@ static inline int mutex_is_locked(struct mutex *lock)
         return atomic_read(&lock->count) != 1;
 }
 
+/* return non-zero to abort.  only pre-side-effects may abort */
+typedef int (*side_effect_t)(unsigned long);
+extern void mutex_lock_sfx(struct mutex *lock,
+                                                   side_effect_t pre, unsigned long pre_arg,
+                                                   side_effect_t post, unsigned long post_arg);
+extern void mutex_unlock_sfx(struct mutex *lock,
+                                                         side_effect_t pre, unsigned long pre_arg,
+                                                         side_effect_t post, unsigned long post_arg);
+
 /*
  * See kernel/mutex.c for detailed documentation of these APIs.
  * Also see Documentation/mutex-design.txt.
@@ -153,6 +162,7 @@ extern void mutex_lock(struct mutex *lock);
 extern int __must_check mutex_lock_interruptible(struct mutex *lock);
 extern int __must_check mutex_lock_killable(struct mutex *lock);
 
+
 # define mutex_lock_nested(lock, subclass) mutex_lock(lock)
 # define mutex_lock_interruptible_nested(lock, subclass) mutex_lock_interruptible(lock)
 # define mutex_lock_killable_nested(lock, subclass) mutex_lock_killable(lock)
diff --git a/include/linux/semaphore.h b/include/linux/semaphore.h
index 39fa04966aa8..c83fc2b65f01 100644
--- a/include/linux/semaphore.h
+++ b/include/linux/semaphore.h
@@ -43,4 +43,13 @@ extern int __must_check down_trylock(struct semaphore *sem);
 extern int __must_check down_timeout(struct semaphore *sem, long jiffies);
 extern void up(struct semaphore *sem);
 
+extern void __down(struct semaphore *sem);
+extern void __up(struct semaphore *sem);
+
+struct semaphore_waiter {
+        struct list_head list;
+        struct task_struct *task;
+        int up;
+};
+
 #endif /* __LINUX_SEMAPHORE_H */
diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h
index f584aba78ca9..1ec2ec7d4e3b 100644
--- a/include/linux/workqueue.h
+++ b/include/linux/workqueue.h
@@ -83,6 +83,9 @@ struct work_struct {
 #ifdef CONFIG_LOCKDEP
         struct lockdep_map lockdep_map;
 #endif
+#ifdef CONFIG_LITMUS_SOFTIRQD
+        struct task_struct *owner;
+#endif
 };
 
 #define WORK_DATA_INIT()        ATOMIC_LONG_INIT(WORK_STRUCT_NO_CPU)
@@ -115,11 +118,25 @@ struct execute_work {
 #define __WORK_INIT_LOCKDEP_MAP(n, k)
 #endif
 
+#ifdef CONFIG_LITMUS_SOFTIRQD
+#define __WORK_INIT_OWNER() \
+        .owner = NULL,
+
+#define PREPARE_OWNER(_work, _owner) \
+        do { \
+                (_work)->owner = (_owner); \
+        } while(0)
+#else
+#define __WORK_INIT_OWNER()
+#define PREPARE_OWNER(_work, _owner)
+#endif
+
 #define __WORK_INITIALIZER(n, f) {                              \
         .data = WORK_DATA_STATIC_INIT(),                        \
         .entry  = { &(n).entry, &(n).entry },                   \
         .func = (f),                                            \
         __WORK_INIT_LOCKDEP_MAP(#n, &(n))                       \
+        __WORK_INIT_OWNER() \
         }
 
 #define __DELAYED_WORK_INITIALIZER(n, f) {                      \
@@ -357,6 +374,7 @@ extern int queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
 extern void flush_workqueue(struct workqueue_struct *wq);
 extern void flush_scheduled_work(void);
 
+extern int __schedule_work(struct work_struct *work);
 extern int schedule_work(struct work_struct *work);
 extern int schedule_work_on(int cpu, struct work_struct *work);
 extern int schedule_delayed_work(struct delayed_work *work, unsigned long delay);
diff --git a/include/litmus/budget.h b/include/litmus/budget.h
index 33344ee8d5f9..763b31c0e9f6 100644
--- a/include/litmus/budget.h
+++ b/include/litmus/budget.h
@@ -5,6 +5,9 @@
  * the next task. */
 void update_enforcement_timer(struct task_struct* t);
 
+/* Send SIG_BUDGET to a real-time task. */
+void send_sigbudget(struct task_struct* t);
+
 inline static int budget_exhausted(struct task_struct* t)
 {
         return get_exec_time(t) >= get_exec_cost(t);
@@ -19,10 +22,21 @@ inline static lt_t budget_remaining(struct task_struct* t)
                 return 0;
 }
 
-#define budget_enforced(t) (tsk_rt(t)->task_params.budget_policy != NO_ENFORCEMENT)
+#define budget_enforced(t) (\
+        tsk_rt(t)->task_params.budget_policy != NO_ENFORCEMENT)
+
+#define budget_precisely_tracked(t) (\
+        tsk_rt(t)->task_params.budget_policy == PRECISE_ENFORCEMENT || \
+        tsk_rt(t)->task_params.budget_signal_policy == PRECISE_SIGNALS)
+
+#define budget_signalled(t) (\
+        tsk_rt(t)->task_params.budget_signal_policy != NO_SIGNALS)
+
+#define budget_precisely_signalled(t) (\
+        tsk_rt(t)->task_params.budget_policy == PRECISE_SIGNALS)
 
-#define budget_precisely_enforced(t) (tsk_rt(t)->task_params.budget_policy \
-                                      == PRECISE_ENFORCEMENT)
+#define sigbudget_sent(t) (\
+        test_bit(RT_JOB_SIG_BUDGET_SENT, &tsk_rt(t)->job_params.flags))
 
 static inline int requeue_preempted_job(struct task_struct* t)
 {
diff --git a/include/litmus/edf_common.h b/include/litmus/edf_common.h
index bbaf22ea7f12..63dff7efe8fb 100644
--- a/include/litmus/edf_common.h
+++ b/include/litmus/edf_common.h
@@ -20,6 +20,18 @@ int edf_higher_prio(struct task_struct* first,
 
 int edf_ready_order(struct bheap_node* a, struct bheap_node* b);
 
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+/* binheap_nodes must be embedded within 'struct litmus_lock' */
+int edf_max_heap_order(struct binheap_node *a, struct binheap_node *b);
+int edf_min_heap_order(struct binheap_node *a, struct binheap_node *b);
+int edf_max_heap_base_priority_order(struct binheap_node *a, struct binheap_node *b);
+int edf_min_heap_base_priority_order(struct binheap_node *a, struct binheap_node *b);
+
+int __edf_higher_prio(struct task_struct* first, comparison_mode_t first_mode,
+                                          struct task_struct* second, comparison_mode_t second_mode);
+
+#endif
+
 int edf_preemption_needed(rt_domain_t* rt, struct task_struct *t);
 
 #endif
diff --git a/include/litmus/fdso.h b/include/litmus/fdso.h
index f2115b83f1e4..1469c0fd0460 100644
--- a/include/litmus/fdso.h
+++ b/include/litmus/fdso.h
@@ -24,9 +24,18 @@ typedef enum  {
         MPCP_VS_SEM     = 3,
         DPCP_SEM        = 4,
 
-        PCP_SEM         = 5,
+        PCP_SEM     = 5,
 
-        MAX_OBJ_TYPE    = 5
+        RSM_MUTEX       = 6,
+        IKGLP_SEM       = 7,
+        KFMLP_SEM       = 8,
+
+        IKGLP_SIMPLE_GPU_AFF_OBS        = 9,
+        IKGLP_GPU_AFF_OBS                       = 10,
+        KFMLP_SIMPLE_GPU_AFF_OBS        = 11,
+        KFMLP_GPU_AFF_OBS                       = 12,
+
+        MAX_OBJ_TYPE    = 12
 } obj_type_t;
 
 struct inode_obj_id {
@@ -70,8 +79,11 @@ static inline void* od_lookup(int od, obj_type_t type)
 }
 
 #define lookup_fmlp_sem(od)((struct pi_semaphore*)  od_lookup(od, FMLP_SEM))
+#define lookup_kfmlp_sem(od)((struct pi_semaphore*)  od_lookup(od, KFMLP_SEM))
 #define lookup_srp_sem(od) ((struct srp_semaphore*) od_lookup(od, SRP_SEM))
 #define lookup_ics(od)     ((struct ics*)           od_lookup(od, ICS_ID))
 
+#define lookup_rsm_mutex(od)((struct litmus_lock*)  od_lookup(od, FMLP_SEM))
+
 
 #endif
diff --git a/include/litmus/fpmath.h b/include/litmus/fpmath.h
new file mode 100644
index 000000000000..04d4bcaeae96
--- /dev/null
+++ b/include/litmus/fpmath.h
@@ -0,0 +1,145 @@
+#ifndef __FP_MATH_H__
+#define __FP_MATH_H__
+
+#ifndef __KERNEL__
+#include <stdint.h>
+#define abs(x) (((x) < 0) ? -(x) : x)
+#endif
+
+// Use 64-bit because we want to track things at the nanosecond scale.
+// This can lead to very large numbers.
+typedef int64_t fpbuf_t;
+typedef struct
+{
+        fpbuf_t val;
+} fp_t;
+
+#define FP_SHIFT 10
+#define ROUND_BIT (FP_SHIFT - 1)
+
+#define _fp(x) ((fp_t) {x})
+
+#ifdef __KERNEL__
+static const fp_t LITMUS_FP_ZERO = {.val = 0};
+static const fp_t LITMUS_FP_ONE = {.val = (1 << FP_SHIFT)};
+#endif
+
+static inline fp_t FP(fpbuf_t x)
+{
+        return _fp(((fpbuf_t) x) << FP_SHIFT);
+}
+
+/* divide two integers to obtain a fixed point value  */
+static inline fp_t _frac(fpbuf_t a, fpbuf_t b)
+{
+        return _fp(FP(a).val / (b));
+}
+
+static inline fpbuf_t _point(fp_t x)
+{
+        return (x.val % (1 << FP_SHIFT));
+
+}
+
+#define fp2str(x) x.val
+/*(x.val >> FP_SHIFT), (x.val % (1 << FP_SHIFT)) */
+#define _FP_  "%ld/1024"
+
+static inline fpbuf_t _floor(fp_t x)
+{
+        return x.val >> FP_SHIFT;
+}
+
+/* FIXME: negative rounding */
+static inline fpbuf_t _round(fp_t x)
+{
+        return _floor(x) + ((x.val >> ROUND_BIT) & 1);
+}
+
+/* multiply two fixed point values */
+static inline fp_t _mul(fp_t a, fp_t b)
+{
+        return _fp((a.val * b.val) >> FP_SHIFT);
+}
+
+static inline fp_t _div(fp_t a, fp_t b)
+{
+#if !defined(__KERNEL__) && !defined(unlikely)
+#define unlikely(x) (x)
+#define DO_UNDEF_UNLIKELY
+#endif
+        /* try not to overflow */
+        if (unlikely(  a.val > (2l << ((sizeof(fpbuf_t)*8) - FP_SHIFT)) ))
+                return _fp((a.val / b.val) << FP_SHIFT);
+        else
+                return _fp((a.val << FP_SHIFT) / b.val);
+#ifdef DO_UNDEF_UNLIKELY
+#undef unlikely
+#undef DO_UNDEF_UNLIKELY
+#endif
+}
+
+static inline fp_t _add(fp_t a, fp_t b)
+{
+        return _fp(a.val + b.val);
+}
+
+static inline fp_t _sub(fp_t a, fp_t b)
+{
+        return _fp(a.val - b.val);
+}
+
+static inline fp_t _neg(fp_t x)
+{
+        return _fp(-x.val);
+}
+
+static inline fp_t _abs(fp_t x)
+{
+        return _fp(abs(x.val));
+}
+
+/* works the same as casting float/double to integer */
+static inline fpbuf_t _fp_to_integer(fp_t x)
+{
+        return _floor(_abs(x)) * ((x.val > 0) ? 1 : -1);
+}
+
+static inline fp_t _integer_to_fp(fpbuf_t x)
+{
+        return _frac(x,1);
+}
+
+static inline int _leq(fp_t a, fp_t b)
+{
+        return a.val <= b.val;
+}
+
+static inline int _geq(fp_t a, fp_t b)
+{
+        return a.val >= b.val;
+}
+
+static inline int _lt(fp_t a, fp_t b)
+{
+        return a.val < b.val;
+}
+
+static inline int _gt(fp_t a, fp_t b)
+{
+        return a.val > b.val;
+}
+
+static inline int _eq(fp_t a, fp_t b)
+{
+        return a.val == b.val;
+}
+
+static inline fp_t _max(fp_t a, fp_t b)
+{
+        if (a.val < b.val)
+                return b;
+        else
+                return a;
+}
+#endif
diff --git a/include/litmus/gpu_affinity.h b/include/litmus/gpu_affinity.h
new file mode 100644
index 000000000000..d64a15cbf2a5
--- /dev/null
+++ b/include/litmus/gpu_affinity.h
@@ -0,0 +1,50 @@
+#ifndef LITMUS_GPU_AFFINITY_H
+#define LITMUS_GPU_AFFINITY_H
+
+#include <litmus/rt_param.h>
+#include <litmus/sched_plugin.h>
+#include <litmus/litmus.h>
+
+void update_gpu_estimate(struct task_struct* t, lt_t observed);
+gpu_migration_dist_t gpu_migration_distance(int a, int b);
+
+static inline void reset_gpu_tracker(struct task_struct* t)
+{
+        t->rt_param.accum_gpu_time = 0;
+}
+
+static inline void start_gpu_tracker(struct task_struct* t)
+{
+        t->rt_param.gpu_time_stamp = litmus_clock();
+}
+
+static inline void stop_gpu_tracker(struct task_struct* t)
+{
+        lt_t now = litmus_clock();
+        t->rt_param.accum_gpu_time += (now - t->rt_param.gpu_time_stamp);
+}
+
+static inline lt_t get_gpu_time(struct task_struct* t)
+{
+        return t->rt_param.accum_gpu_time;
+}
+
+static inline lt_t get_gpu_estimate(struct task_struct* t, gpu_migration_dist_t dist)
+{
+//      int i;
+//      fpbuf_t temp = _fp_to_integer(t->rt_param.gpu_migration_est[dist].est);
+//      lt_t val = (temp >= 0) ? temp : 0;  // never allow negative estimates...
+        lt_t val = t->rt_param.gpu_migration_est[dist].avg;
+
+//      WARN_ON(temp < 0);
+
+        // lower-bound a distant migration to be at least equal to the level
+        // below it.
+//      for(i = dist-1; (val == 0) && (i >= MIG_LOCAL); --i) {
+//              val = _fp_to_integer(t->rt_param.gpu_migration_est[i].est);
+//      }
+
+        return ((val > 0) ? val : dist+1);
+}
+
+#endif
diff --git a/include/litmus/ikglp_lock.h b/include/litmus/ikglp_lock.h
new file mode 100644
index 000000000000..0b89c8135360
--- /dev/null
+++ b/include/litmus/ikglp_lock.h
@@ -0,0 +1,160 @@
+#ifndef LITMUS_IKGLP_H
+#define LITMUS_IKGLP_H
+
+#include <litmus/litmus.h>
+#include <litmus/binheap.h>
+#include <litmus/locking.h>
+
+#ifdef CONFIG_LITMUS_AFFINITY_LOCKING
+#include <litmus/kexclu_affinity.h>
+
+struct ikglp_affinity;
+#endif
+
+typedef struct ikglp_heap_node
+{
+        struct task_struct *task;
+        struct binheap_node node;
+} ikglp_heap_node_t;
+
+struct fifo_queue;
+struct ikglp_wait_state;
+
+typedef struct ikglp_donee_heap_node
+{
+        struct task_struct *task;
+        struct fifo_queue *fq;
+        struct ikglp_wait_state *donor_info;  // cross-linked with ikglp_wait_state_t of donor
+
+        struct binheap_node node;
+} ikglp_donee_heap_node_t;
+
+// Maintains the state of a request as it goes through the IKGLP
+typedef struct ikglp_wait_state {
+        struct task_struct *task;  // pointer back to the requesting task
+
+        // Data for while waiting in FIFO Queue
+        wait_queue_t fq_node;
+        ikglp_heap_node_t global_heap_node;
+        ikglp_donee_heap_node_t donee_heap_node;
+
+        // Data for while waiting in PQ
+        ikglp_heap_node_t pq_node;
+
+        // Data for while waiting as a donor
+        ikglp_donee_heap_node_t *donee_info;  // cross-linked with donee's ikglp_donee_heap_node_t
+        struct nested_info prio_donation;
+        struct binheap_node node;
+} ikglp_wait_state_t;
+
+/* struct for semaphore with priority inheritance */
+struct fifo_queue
+{
+        wait_queue_head_t wait;
+        struct task_struct* owner;
+
+        // used for bookkeepping
+        ikglp_heap_node_t global_heap_node;
+        ikglp_donee_heap_node_t donee_heap_node;
+
+        struct task_struct* hp_waiter;
+        int count; /* number of waiters + holder */
+
+        struct nested_info nest;
+};
+
+struct ikglp_semaphore
+{
+        struct litmus_lock litmus_lock;
+
+        raw_spinlock_t  lock;
+        raw_spinlock_t  real_lock;
+
+        int nr_replicas; // AKA k
+        int m;
+
+        int max_fifo_len; // max len of a fifo queue
+        int nr_in_fifos;
+
+        struct binheap top_m;  // min heap, base prio
+        int top_m_size;  // number of nodes in top_m
+
+        struct binheap not_top_m; // max heap, base prio
+
+        struct binheap donees;  // min-heap, base prio
+        struct fifo_queue *shortest_fifo_queue; // pointer to shortest fifo queue
+
+        /* data structures for holding requests */
+        struct fifo_queue *fifo_queues; // array nr_replicas in length
+        struct binheap priority_queue;  // max-heap, base prio
+        struct binheap donors;  // max-heap, base prio
+
+#ifdef CONFIG_LITMUS_AFFINITY_LOCKING
+        struct ikglp_affinity *aff_obs;
+#endif
+};
+
+static inline struct ikglp_semaphore* ikglp_from_lock(struct litmus_lock* lock)
+{
+        return container_of(lock, struct ikglp_semaphore, litmus_lock);
+}
+
+int ikglp_lock(struct litmus_lock* l);
+int ikglp_unlock(struct litmus_lock* l);
+int ikglp_close(struct litmus_lock* l);
+void ikglp_free(struct litmus_lock* l);
+struct litmus_lock* ikglp_new(int m, struct litmus_lock_ops*, void* __user arg);
+
+
+
+#if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA)
+
+struct ikglp_queue_info
+{
+        struct fifo_queue* q;
+        lt_t estimated_len;
+        int *nr_cur_users;
+};
+
+struct ikglp_affinity_ops
+{
+        struct fifo_queue* (*advise_enqueue)(struct ikglp_affinity* aff, struct task_struct* t);        // select FIFO
+        ikglp_wait_state_t* (*advise_steal)(struct ikglp_affinity* aff, struct fifo_queue* dst);        // select steal from FIFO
+        ikglp_donee_heap_node_t* (*advise_donee_selection)(struct ikglp_affinity* aff, struct task_struct* t);  // select a donee
+        ikglp_wait_state_t* (*advise_donor_to_fq)(struct ikglp_affinity* aff, struct fifo_queue* dst);  // select a donor to move to PQ
+
+        void (*notify_enqueue)(struct ikglp_affinity* aff, struct fifo_queue* fq, struct task_struct* t);       // fifo enqueue
+        void (*notify_dequeue)(struct ikglp_affinity* aff, struct fifo_queue* fq, struct task_struct* t);       // fifo dequeue
+        void (*notify_acquired)(struct ikglp_affinity* aff, struct fifo_queue* fq, struct task_struct* t);      // replica acquired
+        void (*notify_freed)(struct ikglp_affinity* aff, struct fifo_queue* fq, struct task_struct* t);         // replica freed
+        int (*replica_to_resource)(struct ikglp_affinity* aff, struct fifo_queue* fq);          // convert a replica # to a GPU (includes offsets and simult user folding)
+};
+
+struct ikglp_affinity
+{
+        struct affinity_observer obs;
+        struct ikglp_affinity_ops *ops;
+        struct ikglp_queue_info *q_info;
+        int *nr_cur_users_on_rsrc;
+        int offset;
+        int nr_simult;
+        int nr_rsrc;
+        int relax_max_fifo_len;
+};
+
+static inline struct ikglp_affinity* ikglp_aff_obs_from_aff_obs(struct affinity_observer* aff_obs)
+{
+        return container_of(aff_obs, struct ikglp_affinity, obs);
+}
+
+int ikglp_aff_obs_close(struct affinity_observer*);
+void ikglp_aff_obs_free(struct affinity_observer*);
+struct affinity_observer* ikglp_gpu_aff_obs_new(struct affinity_observer_ops*,
+                                                                                                void* __user arg);
+struct affinity_observer* ikglp_simple_gpu_aff_obs_new(struct affinity_observer_ops*,
+                                                                                                void* __user arg);
+#endif
+
+
+
+#endif
diff --git a/include/litmus/kexclu_affinity.h b/include/litmus/kexclu_affinity.h
new file mode 100644
index 000000000000..f6355de49074
--- /dev/null
+++ b/include/litmus/kexclu_affinity.h
@@ -0,0 +1,35 @@
+#ifndef LITMUS_AFF_OBS_H
+#define LITMUS_AFF_OBS_H
+
+#include <litmus/locking.h>
+
+struct affinity_observer_ops;
+
+struct affinity_observer
+{
+        struct affinity_observer_ops* ops;
+        int type;
+        int ident;
+
+        struct litmus_lock* lock;  // the lock under observation
+};
+
+typedef int (*aff_obs_open_t)(struct affinity_observer* aff_obs,
+                                                          void* __user arg);
+typedef int (*aff_obs_close_t)(struct affinity_observer* aff_obs);
+typedef void (*aff_obs_free_t)(struct affinity_observer* aff_obs);
+
+struct affinity_observer_ops
+{
+        aff_obs_open_t open;
+        aff_obs_close_t close;
+        aff_obs_free_t deallocate;
+};
+
+struct litmus_lock* get_lock_from_od(int od);
+
+void affinity_observer_new(struct affinity_observer* aff,
+                                                   struct affinity_observer_ops* ops,
+                                                   struct affinity_observer_args* args);
+
+#endif
diff --git a/include/litmus/kfmlp_lock.h b/include/litmus/kfmlp_lock.h
new file mode 100644
index 000000000000..5f0aae6e6f42
--- /dev/null
+++ b/include/litmus/kfmlp_lock.h
@@ -0,0 +1,97 @@
+#ifndef LITMUS_KFMLP_H
+#define LITMUS_KFMLP_H
+
+#include <litmus/litmus.h>
+#include <litmus/locking.h>
+
+#ifdef CONFIG_LITMUS_AFFINITY_LOCKING
+#include <litmus/kexclu_affinity.h>
+
+struct kfmlp_affinity;
+#endif
+
+/* struct for semaphore with priority inheritance */
+struct kfmlp_queue
+{
+        wait_queue_head_t wait;
+        struct task_struct* owner;
+        struct task_struct* hp_waiter;
+        int count; /* number of waiters + holder */
+};
+
+struct kfmlp_semaphore
+{
+        struct litmus_lock litmus_lock;
+
+        spinlock_t      lock;
+
+        int num_resources; /* aka k */
+
+        struct kfmlp_queue *queues; /* array */
+        struct kfmlp_queue *shortest_queue; /* pointer to shortest queue */
+
+#ifdef CONFIG_LITMUS_AFFINITY_LOCKING
+        struct kfmlp_affinity *aff_obs;
+#endif
+};
+
+static inline struct kfmlp_semaphore* kfmlp_from_lock(struct litmus_lock* lock)
+{
+        return container_of(lock, struct kfmlp_semaphore, litmus_lock);
+}
+
+int kfmlp_lock(struct litmus_lock* l);
+int kfmlp_unlock(struct litmus_lock* l);
+int kfmlp_close(struct litmus_lock* l);
+void kfmlp_free(struct litmus_lock* l);
+struct litmus_lock* kfmlp_new(struct litmus_lock_ops*, void* __user arg);
+
+#if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA)
+
+struct kfmlp_queue_info
+{
+        struct kfmlp_queue* q;
+        lt_t estimated_len;
+        int *nr_cur_users;
+};
+
+struct kfmlp_affinity_ops
+{
+        struct kfmlp_queue* (*advise_enqueue)(struct kfmlp_affinity* aff, struct task_struct* t);
+        struct task_struct* (*advise_steal)(struct kfmlp_affinity* aff, wait_queue_t** to_steal, struct kfmlp_queue** to_steal_from);
+        void (*notify_enqueue)(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t);
+        void (*notify_dequeue)(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t);
+        void (*notify_acquired)(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t);
+        void (*notify_freed)(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t);
+        int (*replica_to_resource)(struct kfmlp_affinity* aff, struct kfmlp_queue* fq);
+};
+
+struct kfmlp_affinity
+{
+        struct affinity_observer obs;
+        struct kfmlp_affinity_ops *ops;
+        struct kfmlp_queue_info *q_info;
+        int *nr_cur_users_on_rsrc;
+        int offset;
+        int nr_simult;
+        int nr_rsrc;
+};
+
+static inline struct kfmlp_affinity* kfmlp_aff_obs_from_aff_obs(struct affinity_observer* aff_obs)
+{
+        return container_of(aff_obs, struct kfmlp_affinity, obs);
+}
+
+int kfmlp_aff_obs_close(struct affinity_observer*);
+void kfmlp_aff_obs_free(struct affinity_observer*);
+struct affinity_observer* kfmlp_gpu_aff_obs_new(struct affinity_observer_ops*,
+                                                                                        void* __user arg);
+struct affinity_observer* kfmlp_simple_gpu_aff_obs_new(struct affinity_observer_ops*,
+                                                                                                void* __user arg);
+
+
+#endif
+
+#endif
+
+
diff --git a/include/litmus/litmus.h b/include/litmus/litmus.h
index 338245abd6ed..1d70ab713571 100644
--- a/include/litmus/litmus.h
+++ b/include/litmus/litmus.h
@@ -26,6 +26,7 @@ static inline int in_list(struct list_head* list)
                 );
 }
 
+
 struct task_struct* __waitqueue_remove_first(wait_queue_head_t *wq);
 
 #define NO_CPU                  0xffffffff
@@ -62,8 +63,12 @@ void litmus_exit_task(struct task_struct *tsk);
 /* job_param macros */
 #define get_exec_time(t)    (tsk_rt(t)->job_params.exec_time)
 #define get_deadline(t)         (tsk_rt(t)->job_params.deadline)
+#define get_period(t)           (tsk_rt(t)->task_params.period)
 #define get_release(t)          (tsk_rt(t)->job_params.release)
+#define get_lateness(t)     (tsk_rt(t)->job_params.lateness)
 
+#define effective_priority(t) ((!(tsk_rt(t)->inh_task)) ? t : tsk_rt(t)->inh_task)
+#define base_priority(t) (t)
 
 #define is_hrt(t)               \
         (tsk_rt(t)->task_params.cls == RT_CLASS_HARD)
@@ -101,10 +106,12 @@ static inline lt_t litmus_clock(void)
 #define earlier_deadline(a, b) (lt_before(\
         (a)->rt_param.job_params.deadline,\
         (b)->rt_param.job_params.deadline))
+#define shorter_period(a, b) (lt_before(\
+        (a)->rt_param.task_params.period,\
+        (b)->rt_param.task_params.period))
 #define earlier_release(a, b)  (lt_before(\
         (a)->rt_param.job_params.release,\
         (b)->rt_param.job_params.release))
-
 void preempt_if_preemptable(struct task_struct* t, int on_cpu);
 
 #ifdef CONFIG_LITMUS_LOCKING
diff --git a/include/litmus/litmus_softirq.h b/include/litmus/litmus_softirq.h
new file mode 100644
index 000000000000..1eb5ea1a6c4b
--- /dev/null
+++ b/include/litmus/litmus_softirq.h
@@ -0,0 +1,199 @@
+#ifndef __LITMUS_SOFTIRQ_H
+#define __LITMUS_SOFTIRQ_H
+
+#include <linux/interrupt.h>
+#include <linux/workqueue.h>
+
+/*
+   Threaded tasklet handling for Litmus.  Tasklets
+   are scheduled with the priority of the tasklet's
+   owner---that is, the RT task on behalf the tasklet
+   runs.
+
+   Tasklets are current scheduled in FIFO order with
+   NO priority inheritance for "blocked" tasklets.
+
+   klitirqd assumes the priority of the owner of the
+   tasklet when the tasklet is next to execute.
+
+   Currently, hi-tasklets are scheduled before
+   low-tasklets, regardless of priority of low-tasklets.
+   And likewise, low-tasklets are scheduled before work
+   queue objects.  This priority inversion probably needs
+   to be fixed, though it is not an issue if our work with
+   GPUs as GPUs are owned (and associated klitirqds) for
+   exclusive time periods, thus no inversions can
+   occur.
+ */
+
+
+
+#define NR_LITMUS_SOFTIRQD CONFIG_NR_LITMUS_SOFTIRQD
+
+/* Spawns NR_LITMUS_SOFTIRQD klitirqd daemons.
+   Actual launch of threads is deffered to kworker's
+   workqueue, so daemons will likely not be immediately
+   running when this function returns, though the required
+   data will be initialized.
+
+   @affinity_set: an array expressing the processor affinity
+    for each of the NR_LITMUS_SOFTIRQD daemons.  May be set
+    to NULL for global scheduling.
+
+        - Examples -
+        8-CPU system with two CPU clusters:
+                affinity[] = {0, 0, 0, 0, 3, 3, 3, 3}
+                NOTE: Daemons not actually bound to specified CPU, but rather
+                cluster in which the CPU resides.
+
+        8-CPU system, partitioned:
+                affinity[] = {0, 1, 2, 3, 4, 5, 6, 7}
+
+        FIXME: change array to a CPU topology or array of cpumasks
+
+ */
+void spawn_klitirqd(int* affinity);
+
+
+/* Raises a flag to tell klitirqds to terminate.
+   Termination is async, so some threads may be running
+   after function return. */
+void kill_klitirqd(void);
+
+
+/* Returns 1 if all NR_LITMUS_SOFTIRQD klitirqs are ready
+   to handle tasklets. 0, otherwise.*/
+int klitirqd_is_ready(void);
+
+/* Returns 1 if no NR_LITMUS_SOFTIRQD klitirqs are ready
+   to handle tasklets. 0, otherwise.*/
+int klitirqd_is_dead(void);
+
+/* Flushes all pending work out to the OS for regular
+ * tasklet/work processing of the specified 'owner'
+ *
+ * PRECOND: klitirqd_thread must have a clear entry
+ * in the GPU registry, otherwise this call will become
+ * a no-op as work will loop back to the klitirqd_thread.
+ *
+ * Pass NULL for owner to flush ALL pending items.
+ */
+void flush_pending(struct task_struct* klitirqd_thread,
+                                   struct task_struct* owner);
+
+struct task_struct* get_klitirqd(unsigned int k_id);
+
+
+extern int __litmus_tasklet_schedule(
+        struct tasklet_struct *t,
+        unsigned int k_id);
+
+/* schedule a tasklet on klitirqd #k_id */
+static inline int litmus_tasklet_schedule(
+    struct tasklet_struct *t,
+    unsigned int k_id)
+{
+        int ret = 0;
+        if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))
+                ret = __litmus_tasklet_schedule(t, k_id);
+        return(ret);
+}
+
+/* for use by __tasklet_schedule() */
+static inline int _litmus_tasklet_schedule(
+    struct tasklet_struct *t,
+    unsigned int k_id)
+{
+    return(__litmus_tasklet_schedule(t, k_id));
+}
+
+
+
+
+extern int __litmus_tasklet_hi_schedule(struct tasklet_struct *t,
+                                         unsigned int k_id);
+
+/* schedule a hi tasklet on klitirqd #k_id */
+static inline int litmus_tasklet_hi_schedule(struct tasklet_struct *t,
+                                              unsigned int k_id)
+{
+        int ret = 0;
+        if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))
+                ret = __litmus_tasklet_hi_schedule(t, k_id);
+        return(ret);
+}
+
+/* for use by __tasklet_hi_schedule() */
+static inline int _litmus_tasklet_hi_schedule(struct tasklet_struct *t,
+                                               unsigned int k_id)
+{
+    return(__litmus_tasklet_hi_schedule(t, k_id));
+}
+
+
+
+
+
+extern int __litmus_tasklet_hi_schedule_first(
+    struct tasklet_struct *t,
+    unsigned int k_id);
+
+/* schedule a hi tasklet on klitirqd #k_id on next go-around */
+/* PRECONDITION: Interrupts must be disabled. */
+static inline int litmus_tasklet_hi_schedule_first(
+    struct tasklet_struct *t,
+    unsigned int k_id)
+{
+        int ret = 0;
+        if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))
+                ret = __litmus_tasklet_hi_schedule_first(t, k_id);
+        return(ret);
+}
+
+/* for use by __tasklet_hi_schedule_first() */
+static inline int _litmus_tasklet_hi_schedule_first(
+    struct tasklet_struct *t,
+    unsigned int k_id)
+{
+    return(__litmus_tasklet_hi_schedule_first(t, k_id));
+}
+
+
+
+//////////////
+
+extern int __litmus_schedule_work(
+        struct work_struct* w,
+        unsigned int k_id);
+
+static inline int litmus_schedule_work(
+        struct work_struct* w,
+        unsigned int k_id)
+{
+        return(__litmus_schedule_work(w, k_id));
+}
+
+
+
+///////////// mutex operations for client threads.
+
+void down_and_set_stat(struct task_struct* t,
+                                         enum klitirqd_sem_status to_set,
+                                         struct mutex* sem);
+
+void __down_and_reset_and_set_stat(struct task_struct* t,
+                                enum klitirqd_sem_status to_reset,
+                                enum klitirqd_sem_status to_set,
+                                struct mutex* sem);
+
+void up_and_set_stat(struct task_struct* t,
+                                        enum klitirqd_sem_status to_set,
+                                        struct mutex* sem);
+
+
+
+void release_klitirqd_lock(struct task_struct* t);
+
+int reacquire_klitirqd_lock(struct task_struct* t);
+
+#endif
diff --git a/include/litmus/locking.h b/include/litmus/locking.h
index 4d7b870cb443..296bbf6f7af0 100644
--- a/include/litmus/locking.h
+++ b/include/litmus/locking.h
@@ -1,28 +1,160 @@
 #ifndef LITMUS_LOCKING_H
 #define LITMUS_LOCKING_H
 
+#include <linux/list.h>
+
 struct litmus_lock_ops;
 
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+struct nested_info
+{
+        struct litmus_lock *lock;
+        struct task_struct *hp_waiter_eff_prio;
+        struct task_struct **hp_waiter_ptr;
+    struct binheap_node hp_binheap_node;
+};
+
+static inline struct task_struct* top_priority(struct binheap* handle) {
+        if(!binheap_empty(handle)) {
+                return (struct task_struct*)(binheap_top_entry(handle, struct nested_info, hp_binheap_node)->hp_waiter_eff_prio);
+        }
+        return NULL;
+}
+
+void print_hp_waiters(struct binheap_node* n, int depth);
+#endif
+
+
 /* Generic base struct for LITMUS^RT userspace semaphores.
  * This structure should be embedded in protocol-specific semaphores.
  */
 struct litmus_lock {
         struct litmus_lock_ops *ops;
         int type;
+
+        int ident;
+
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+        struct nested_info nest;
+//#ifdef CONFIG_DEBUG_SPINLOCK
+        char cheat_lockdep[2];
+        struct lock_class_key key;
+//#endif
+#endif
 };
 
+#ifdef CONFIG_LITMUS_DGL_SUPPORT
+
+#define MAX_DGL_SIZE CONFIG_LITMUS_MAX_DGL_SIZE
+
+typedef struct dgl_wait_state {
+        struct task_struct *task;       /* task waiting on DGL */
+        struct litmus_lock *locks[MAX_DGL_SIZE];        /* requested locks in DGL */
+        int size;                       /* size of the DGL */
+        int nr_remaining;       /* nr locks remainging before DGL is complete */
+        int last_primary;       /* index lock in locks[] that has active priority */
+        wait_queue_t wq_nodes[MAX_DGL_SIZE];
+} dgl_wait_state_t;
+
+void wake_or_wait_on_next_lock(dgl_wait_state_t *dgl_wait);
+void select_next_lock(dgl_wait_state_t* dgl_wait /*, struct litmus_lock* prev_lock*/);
+
+void init_dgl_waitqueue_entry(wait_queue_t *wq_node, dgl_wait_state_t* dgl_wait);
+int dgl_wake_up(wait_queue_t *wq_node, unsigned mode, int sync, void *key);
+void __waitqueue_dgl_remove_first(wait_queue_head_t *wq, dgl_wait_state_t** dgl_wait, struct task_struct **task);
+#endif
+
+typedef int (*lock_op_t)(struct litmus_lock *l);
+typedef lock_op_t lock_close_t;
+typedef lock_op_t lock_lock_t;
+typedef lock_op_t lock_unlock_t;
+
+typedef int (*lock_open_t)(struct litmus_lock *l, void* __user arg);
+typedef void (*lock_free_t)(struct litmus_lock *l);
+
 struct litmus_lock_ops {
         /* Current task tries to obtain / drop a reference to a lock.
          * Optional methods, allowed by default. */
-        int (*open)(struct litmus_lock*, void* __user);
-        int (*close)(struct litmus_lock*);
+        lock_open_t open;
+        lock_close_t close;
 
         /* Current tries to lock/unlock this lock (mandatory methods). */
-        int (*lock)(struct litmus_lock*);
-        int (*unlock)(struct litmus_lock*);
+        lock_lock_t lock;
+        lock_unlock_t unlock;
 
         /* The lock is no longer being referenced (mandatory method). */
-        void (*deallocate)(struct litmus_lock*);
+        lock_free_t deallocate;
+
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+        void (*propagate_increase_inheritance)(struct litmus_lock* l, struct task_struct* t, raw_spinlock_t* to_unlock, unsigned long irqflags);
+        void (*propagate_decrease_inheritance)(struct litmus_lock* l, struct task_struct* t, raw_spinlock_t* to_unlock, unsigned long irqflags);
+#endif
+
+#ifdef CONFIG_LITMUS_DGL_SUPPORT
+        raw_spinlock_t* (*get_dgl_spin_lock)(struct litmus_lock *l);
+        int (*dgl_lock)(struct litmus_lock *l, dgl_wait_state_t* dgl_wait, wait_queue_t* wq_node);
+        int (*is_owner)(struct litmus_lock *l, struct task_struct *t);
+        void (*enable_priority)(struct litmus_lock *l, dgl_wait_state_t* dgl_wait);
+#endif
 };
 
+
+/*
+ Nested inheritance can be achieved with fine-grain locking when there is
+ no need for DGL support, presuming locks are acquired in a partial order
+ (no cycles!).  However, DGLs allow locks to be acquired in any order.  This
+ makes nested inheritance very difficult (we don't yet know a solution) to
+ realize with fine-grain locks, so we use a big lock instead.
+
+ Code contains both fine-grain and coarse-grain methods together, side-by-side.
+ Each lock operation *IS NOT* surrounded by ifdef/endif to help make code more
+ readable.  However, this leads to the odd situation where both code paths
+ appear together in code as if they were both active together.
+
+ THIS IS NOT REALLY THE CASE!  ONLY ONE CODE PATH IS ACTUALLY ACTIVE!
+
+ Example:
+        lock_global_irqsave(coarseLock, flags);
+        lock_fine_irqsave(fineLock, flags);
+
+ Reality (coarse):
+        lock_global_irqsave(coarseLock, flags);
+        //lock_fine_irqsave(fineLock, flags);
+
+ Reality (fine):
+        //lock_global_irqsave(coarseLock, flags);
+        lock_fine_irqsave(fineLock, flags);
+
+ Be careful when you read code involving nested inheritance.
+ */
+#if defined(CONFIG_LITMUS_DGL_SUPPORT)
+/* DGL requires a big lock to implement nested inheritance */
+#define lock_global_irqsave(lock, flags)                raw_spin_lock_irqsave((lock), (flags))
+#define lock_global(lock)                                               raw_spin_lock((lock))
+#define unlock_global_irqrestore(lock, flags)   raw_spin_unlock_irqrestore((lock), (flags))
+#define unlock_global(lock)                                             raw_spin_unlock((lock))
+
+/* fine-grain locking are no-ops with DGL support */
+#define lock_fine_irqsave(lock, flags)
+#define lock_fine(lock)
+#define unlock_fine_irqrestore(lock, flags)
+#define unlock_fine(lock)
+
+#elif defined(CONFIG_LITMUS_NESTED_LOCKING)
+/* Use fine-grain locking when DGLs are disabled. */
+/* global locking are no-ops without DGL support */
+#define lock_global_irqsave(lock, flags)
+#define lock_global(lock)
+#define unlock_global_irqrestore(lock, flags)
+#define unlock_global(lock)
+
+#define lock_fine_irqsave(lock, flags)                  raw_spin_lock_irqsave((lock), (flags))
+#define lock_fine(lock)                                                 raw_spin_lock((lock))
+#define unlock_fine_irqrestore(lock, flags)             raw_spin_unlock_irqrestore((lock), (flags))
+#define unlock_fine(lock)                                               raw_spin_unlock((lock))
+
 #endif
+
+
+#endif
+
diff --git a/include/litmus/nvidia_info.h b/include/litmus/nvidia_info.h
new file mode 100644
index 000000000000..97c9577141db
--- /dev/null
+++ b/include/litmus/nvidia_info.h
@@ -0,0 +1,46 @@
+#ifndef __LITMUS_NVIDIA_H
+#define __LITMUS_NVIDIA_H
+
+#include <linux/interrupt.h>
+
+
+#include <litmus/litmus_softirq.h>
+
+
+//#define NV_DEVICE_NUM NR_LITMUS_SOFTIRQD
+#define NV_DEVICE_NUM CONFIG_NV_DEVICE_NUM
+#define NV_MAX_SIMULT_USERS CONFIG_NV_MAX_SIMULT_USERS
+
+int init_nvidia_info(void);
+void shutdown_nvidia_info(void);
+
+int is_nvidia_func(void* func_addr);
+
+void dump_nvidia_info(const struct tasklet_struct *t);
+
+
+// Returns the Nvidia device # associated with provided tasklet and work_struct.
+u32 get_tasklet_nv_device_num(const struct tasklet_struct *t);
+u32 get_work_nv_device_num(const struct work_struct *t);
+
+
+int init_nv_device_reg(void);
+//int get_nv_device_id(struct task_struct* owner);
+
+
+int reg_nv_device(int reg_device_id, int register_device, struct task_struct *t);
+
+struct task_struct* get_nv_max_device_owner(u32 target_device_id);
+//int is_nv_device_owner(u32 target_device_id);
+
+void lock_nv_registry(u32 reg_device_id, unsigned long* flags);
+void unlock_nv_registry(u32 reg_device_id, unsigned long* flags);
+
+#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
+void pai_check_priority_increase(struct task_struct *t, int reg_device_id);
+void pai_check_priority_decrease(struct task_struct *t, int reg_device_id);
+#endif
+
+//void increment_nv_int_count(u32 device);
+
+#endif
diff --git a/include/litmus/preempt.h b/include/litmus/preempt.h
index 380b886d78ff..8f3a9ca2d4e3 100644
--- a/include/litmus/preempt.h
+++ b/include/litmus/preempt.h
@@ -26,12 +26,12 @@ const char* sched_state_name(int s);
                                     (x), #x, __FUNCTION__);             \
         } while (0);
 
+//#define TRACE_SCHED_STATE_CHANGE(x, y, cpu) /* ignore */
 #define TRACE_SCHED_STATE_CHANGE(x, y, cpu)                             \
         TRACE_STATE("[P%d] 0x%x (%s) -> 0x%x (%s)\n",                   \
                     cpu,  (x), sched_state_name(x),                     \
                     (y), sched_state_name(y))
 
-
 typedef enum scheduling_state {
         TASK_SCHEDULED    = (1 << 0),  /* The currently scheduled task is the one that
                                         * should be scheduled, and the processor does not
diff --git a/include/litmus/rsm_lock.h b/include/litmus/rsm_lock.h
new file mode 100644
index 000000000000..a15189683de4
--- /dev/null
+++ b/include/litmus/rsm_lock.h
@@ -0,0 +1,54 @@
+#ifndef LITMUS_RSM_H
+#define LITMUS_RSM_H
+
+#include <litmus/litmus.h>
+#include <litmus/binheap.h>
+#include <litmus/locking.h>
+
+/* struct for semaphore with priority inheritance */
+struct rsm_mutex {
+        struct litmus_lock litmus_lock;
+
+        /* current resource holder */
+        struct task_struct *owner;
+
+        /* highest-priority waiter */
+        struct task_struct *hp_waiter;
+
+        /* FIFO queue of waiting tasks -- for now.  time stamp in the future. */
+        wait_queue_head_t       wait;
+
+        /* we do some nesting within spinlocks, so we can't use the normal
+         sleeplocks found in wait_queue_head_t. */
+        raw_spinlock_t          lock;
+};
+
+static inline struct rsm_mutex* rsm_mutex_from_lock(struct litmus_lock* lock)
+{
+        return container_of(lock, struct rsm_mutex, litmus_lock);
+}
+
+#ifdef CONFIG_LITMUS_DGL_SUPPORT
+int rsm_mutex_is_owner(struct litmus_lock *l, struct task_struct *t);
+int rsm_mutex_dgl_lock(struct litmus_lock *l, dgl_wait_state_t* dgl_wait, wait_queue_t* wq_node);
+void rsm_mutex_enable_priority(struct litmus_lock *l, dgl_wait_state_t* dgl_wait);
+#endif
+
+void rsm_mutex_propagate_increase_inheritance(struct litmus_lock* l,
+                                                                                          struct task_struct* t,
+                                                                                          raw_spinlock_t* to_unlock,
+                                                                                          unsigned long irqflags);
+
+void rsm_mutex_propagate_decrease_inheritance(struct litmus_lock* l,
+                                                                                          struct task_struct* t,
+                                                                                          raw_spinlock_t* to_unlock,
+                                                                                          unsigned long irqflags);
+
+int rsm_mutex_lock(struct litmus_lock* l);
+int rsm_mutex_unlock(struct litmus_lock* l);
+int rsm_mutex_close(struct litmus_lock* l);
+void rsm_mutex_free(struct litmus_lock* l);
+struct litmus_lock* rsm_mutex_new(struct litmus_lock_ops*);
+
+
+#endif
\ No newline at end of file
diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h
index 21430623a940..02b750a9570b 100644
--- a/include/litmus/rt_param.h
+++ b/include/litmus/rt_param.h
@@ -5,6 +5,8 @@
 #ifndef _LINUX_RT_PARAM_H_
 #define _LINUX_RT_PARAM_H_
 
+#include <litmus/fpmath.h>
+
 /* Litmus time type. */
 typedef unsigned long long lt_t;
 
@@ -30,9 +32,15 @@ typedef enum {
 typedef enum {
         NO_ENFORCEMENT,      /* job may overrun unhindered */
         QUANTUM_ENFORCEMENT, /* budgets are only checked on quantum boundaries */
-        PRECISE_ENFORCEMENT  /* budgets are enforced with hrtimers */
+        PRECISE_ENFORCEMENT, /* budgets are enforced with hrtimers */
 } budget_policy_t;
 
+typedef enum {
+        NO_SIGNALS,                     /* job receives no signals when it exhausts its budget */
+        QUANTUM_SIGNALS,        /* budget signals are only sent on quantum boundaries */
+        PRECISE_SIGNALS,        /* budget signals are triggered with hrtimers */
+} budget_signal_policy_t;
+
 /* We use the common priority interpretation "lower index == higher priority",
  * which is commonly used in fixed-priority schedulability analysis papers.
  * So, a numerically lower priority value implies higher scheduling priority,
@@ -62,6 +70,7 @@ struct rt_task {
         unsigned int    priority;
         task_class_t    cls;
         budget_policy_t budget_policy; /* ignored by pfair */
+        budget_signal_policy_t budget_signal_policy; /* currently ignored by pfair */
 };
 
 union np_flag {
@@ -74,6 +83,19 @@ union np_flag {
         } np;
 };
 
+struct affinity_observer_args
+{
+        int lock_od;
+};
+
+struct gpu_affinity_observer_args
+{
+        struct affinity_observer_args obs;
+        int replica_to_gpu_offset;
+        int nr_simult_users;
+        int relaxed_rules;
+};
+
 /* The definition of the data that is shared between the kernel and real-time
  * tasks via a shared page (see litmus/ctrldev.c).
  *
@@ -97,6 +119,9 @@ struct control_page {
 /* don't export internal data structures to user space (liblitmus) */
 #ifdef __KERNEL__
 
+#include <litmus/binheap.h>
+#include <linux/semaphore.h>
+
 struct _rt_domain;
 struct bheap_node;
 struct release_heap;
@@ -110,6 +135,12 @@ struct rt_job {
         /* How much service has this job received so far? */
         lt_t    exec_time;
 
+        /* By how much did the prior job miss its deadline by?
+         * Value differs from tardiness in that lateness may
+         * be negative (when job finishes before its deadline).
+         */
+        long long       lateness;
+
         /* Which job is this. This is used to let user space
          * specify which job to wait for, which is important if jobs
          * overrun. If we just call sys_sleep_next_period() then we
@@ -118,10 +149,54 @@ struct rt_job {
          * Increase this sequence number when a job is released.
          */
         unsigned int    job_no;
+
+        /* bits:
+         * 0th: Set if a budget exhaustion signal has already been sent for
+         *      the current job. */
+        unsigned long   flags;
 };
 
+#define RT_JOB_SIG_BUDGET_SENT  0
+
 struct pfair_param;
 
+enum klitirqd_sem_status
+{
+        NEED_TO_REACQUIRE,
+        REACQUIRING,
+        NOT_HELD,
+        HELD
+};
+
+typedef enum gpu_migration_dist
+{
+        // TODO: Make this variable against NR_NVIDIA_GPUS
+        MIG_LOCAL = 0,
+        MIG_NEAR = 1,
+        MIG_MED = 2,
+        MIG_FAR = 3,    // 8 GPUs in a binary tree hierarchy
+        MIG_NONE = 4,
+
+        MIG_LAST = MIG_NONE
+} gpu_migration_dist_t;
+
+typedef struct feedback_est{
+        fp_t est;
+        fp_t accum_err;
+} feedback_est_t;
+
+
+#define AVG_EST_WINDOW_SIZE 20
+
+typedef struct avg_est{
+        lt_t history[AVG_EST_WINDOW_SIZE];
+        uint16_t count;
+        uint16_t idx;
+        lt_t sum;
+        lt_t std;
+        lt_t avg;
+} avg_est_t;
+
 /*      RT task parameters for scheduling extensions
  *      These parameters are inherited during clone and therefore must
  *      be explicitly set up before the task set is launched.
@@ -136,6 +211,50 @@ struct rt_param {
         /* is the task present? (true if it can be scheduled) */
         unsigned int            present:1;
 
+#ifdef CONFIG_LITMUS_SOFTIRQD
+    /* proxy threads have minimum priority by default */
+    unsigned int        is_proxy_thread:1;
+
+        /* pointer to klitirqd currently working on this
+           task_struct's behalf.  only set by the task pointed
+           to by klitirqd.
+
+           ptr only valid if is_proxy_thread == 0
+         */
+        struct task_struct* cur_klitirqd;
+
+        /* Used to implement mutual execution exclusion between
+         * job and klitirqd execution.  Job must always hold
+         * it's klitirqd_sem to execute.  klitirqd instance
+         * must hold the semaphore before executing on behalf
+         * of a job.
+         */
+        struct mutex                            klitirqd_sem;
+
+        /* status of held klitirqd_sem, even if the held klitirqd_sem is from
+           another task (only proxy threads do this though).
+         */
+        atomic_t                                        klitirqd_sem_stat;
+#endif
+
+#ifdef CONFIG_LITMUS_NVIDIA
+        /* number of top-half interrupts handled on behalf of current job */
+        atomic_t                                        nv_int_count;
+        long unsigned int                       held_gpus;  // bitmap of held GPUs.
+
+#ifdef CONFIG_LITMUS_AFFINITY_LOCKING
+        avg_est_t gpu_migration_est[MIG_LAST+1];
+
+        gpu_migration_dist_t    gpu_migration;
+        int                             last_gpu;
+
+        lt_t accum_gpu_time;
+        lt_t gpu_time_stamp;
+
+        unsigned int suspend_gpu_tracker_on_block:1;
+#endif
+#endif
+
 #ifdef CONFIG_LITMUS_LOCKING
         /* Is the task being priority-boosted by a locking protocol? */
         unsigned int            priority_boosted:1;
@@ -155,11 +274,20 @@ struct rt_param {
          * could point to self if PI does not result in
          * an increased task priority.
          */
-         struct task_struct*    inh_task;
+        struct task_struct*     inh_task;
+
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+        raw_spinlock_t                  hp_blocked_tasks_lock;
+        struct binheap                  hp_blocked_tasks;
+
+        /* pointer to lock upon which is currently blocked */
+        struct litmus_lock* blocked_lock;
+#endif
 
         
          struct task_struct*    hp_group;
          unsigned int                   is_slave:1;
+         unsigned int                   has_slaves:1;
 
 
 #ifdef CONFIG_NP_SECTION
diff --git a/include/litmus/sched_plugin.h b/include/litmus/sched_plugin.h
index 6e7cabdddae8..24a6858b4b0b 100644
--- a/include/litmus/sched_plugin.h
+++ b/include/litmus/sched_plugin.h
@@ -11,6 +11,12 @@
 #include <litmus/locking.h>
 #endif
 
+#ifdef CONFIG_LITMUS_AFFINITY_LOCKING
+#include <litmus/kexclu_affinity.h>
+#endif
+
+#include <linux/interrupt.h>
+
 /************************ setup/tear down ********************/
 
 typedef long (*activate_plugin_t) (void);
@@ -29,7 +35,6 @@ typedef struct task_struct* (*schedule_t)(struct task_struct * prev);
  */
 typedef void (*finish_switch_t)(struct task_struct *prev);
 
-
 /********************* task state changes ********************/
 
 /* Called to setup a new real-time task.
@@ -58,6 +63,47 @@ typedef void (*task_exit_t)    (struct task_struct *);
 typedef long (*allocate_lock_t) (struct litmus_lock **lock, int type,
                                  void* __user config);
 
+struct affinity_observer;
+typedef long (*allocate_affinity_observer_t) (
+                                                                struct affinity_observer **aff_obs, int type,
+                                                                void* __user config);
+
+typedef void (*increase_prio_t)(struct task_struct* t, struct task_struct* prio_inh);
+typedef void (*decrease_prio_t)(struct task_struct* t, struct task_struct* prio_inh);
+typedef void (*nested_increase_prio_t)(struct task_struct* t, struct task_struct* prio_inh,
+                                                                          raw_spinlock_t *to_unlock, unsigned long irqflags);
+typedef void (*nested_decrease_prio_t)(struct task_struct* t, struct task_struct* prio_inh,
+                                                                          raw_spinlock_t *to_unlock, unsigned long irqflags);
+
+typedef void (*increase_prio_klitirq_t)(struct task_struct* klitirqd,
+                                        struct task_struct* old_owner,
+                                        struct task_struct* new_owner);
+typedef void (*decrease_prio_klitirqd_t)(struct task_struct* klitirqd,
+                                         struct task_struct* old_owner);
+
+
+typedef int (*enqueue_pai_tasklet_t)(struct tasklet_struct* tasklet);
+typedef void (*change_prio_pai_tasklet_t)(struct task_struct *old_prio,
+                                                                                  struct task_struct *new_prio);
+typedef void (*run_tasklets_t)(struct task_struct* next);
+
+typedef raw_spinlock_t* (*get_dgl_spinlock_t) (struct task_struct *t);
+
+
+typedef int (*higher_prio_t)(struct task_struct* a, struct task_struct* b);
+
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+
+typedef enum
+{
+        BASE,
+        EFFECTIVE
+} comparison_mode_t;
+
+typedef int (*__higher_prio_t)(struct task_struct* a, comparison_mode_t a_mod,
+                                                          struct task_struct* b, comparison_mode_t b_mod);
+#endif
+
 
 /********************* sys call backends  ********************/
 /* This function causes the caller to sleep until the next release */
@@ -88,14 +134,40 @@ struct sched_plugin {
         /*      task state changes      */
         admit_task_t            admit_task;
 
-        task_new_t              task_new;
+    task_new_t                  task_new;
         task_wake_up_t          task_wake_up;
         task_block_t            task_block;
         task_exit_t             task_exit;
 
+        higher_prio_t           compare;
+
 #ifdef CONFIG_LITMUS_LOCKING
         /*      locking protocols       */
         allocate_lock_t         allocate_lock;
+        increase_prio_t         increase_prio;
+        decrease_prio_t         decrease_prio;
+#endif
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+        nested_increase_prio_t nested_increase_prio;
+        nested_decrease_prio_t nested_decrease_prio;
+        __higher_prio_t         __compare;
+#endif
+#ifdef CONFIG_LITMUS_DGL_SUPPORT
+        get_dgl_spinlock_t      get_dgl_spinlock;
+#endif
+
+#ifdef CONFIG_LITMUS_AFFINITY_LOCKING
+        allocate_affinity_observer_t allocate_aff_obs;
+#endif
+
+#ifdef CONFIG_LITMUS_SOFTIRQD
+    increase_prio_klitirq_t             increase_prio_klitirqd;
+    decrease_prio_klitirqd_t    decrease_prio_klitirqd;
+#endif
+#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
+        enqueue_pai_tasklet_t           enqueue_pai_tasklet;
+        change_prio_pai_tasklet_t       change_prio_pai_tasklet;
+        run_tasklets_t                          run_tasklets;
 #endif
 } __attribute__ ((__aligned__(SMP_CACHE_BYTES)));
 
diff --git a/include/litmus/sched_trace.h b/include/litmus/sched_trace.h
index 82bde8241298..7af12f49c600 100644
--- a/include/litmus/sched_trace.h
+++ b/include/litmus/sched_trace.h
@@ -10,13 +10,14 @@ struct st_trace_header {
         u8      type;           /* Of what type is this record?  */
         u8      cpu;            /* On which CPU was it recorded? */
         u16     pid;            /* PID of the task.              */
-        u32     job;            /* The job sequence number.      */
-};
+        u32 job:24;             /* The job sequence number.      */
+        u8  extra;
+} __attribute__((packed));
 
 #define ST_NAME_LEN 16
 struct st_name_data {
         char    cmd[ST_NAME_LEN];/* The name of the executable of this process. */
-};
+} __attribute__((packed));
 
 struct st_param_data {          /* regular params */
         u32     wcet;
@@ -25,30 +26,29 @@ struct st_param_data {		/* regular params */
         u8      partition;
         u8      class;
         u8      __unused[2];
-};
+} __attribute__((packed));
 
 struct st_release_data {        /* A job is was/is going to be released. */
         u64     release;        /* What's the release time?              */
         u64     deadline;       /* By when must it finish?               */
-};
+} __attribute__((packed));
 
 struct st_assigned_data {       /* A job was asigned to a CPU.           */
         u64     when;
         u8      target;         /* Where should it execute?              */
         u8      __unused[7];
-};
+} __attribute__((packed));
 
 struct st_switch_to_data {      /* A process was switched to on a given CPU.   */
         u64     when;           /* When did this occur?                        */
         u32     exec_time;      /* Time the current job has executed.          */
         u8      __unused[4];
-
-};
+} __attribute__((packed));
 
 struct st_switch_away_data {    /* A process was switched away from on a given CPU. */
         u64     when;
         u64     exec_time;
-};
+} __attribute__((packed));
 
 struct st_completion_data {     /* A job completed. */
         u64     when;
@@ -56,35 +56,108 @@ struct st_completion_data {	/* A job completed. */
                                  * next task automatically; set to 0 otherwise.
                                  */
         u8      __uflags:7;
-        u8      __unused[7];
-};
+        u16 nv_int_count;
+        u8      __unused[5];
+} __attribute__((packed));
 
 struct st_block_data {          /* A task blocks. */
         u64     when;
         u64     __unused;
-};
+} __attribute__((packed));
 
 struct st_resume_data {         /* A task resumes. */
         u64     when;
         u64     __unused;
-};
+} __attribute__((packed));
 
 struct st_action_data {
         u64     when;
         u8      action;
         u8      __unused[7];
-};
+} __attribute__((packed));
 
 struct st_sys_release_data {
         u64     when;
         u64     release;
-};
+} __attribute__((packed));
+
+
+struct st_tasklet_release_data {
+        u64 when;
+        u64 __unused;
+} __attribute__((packed));
+
+struct st_tasklet_begin_data {
+        u64 when;
+        u16 exe_pid;
+        u8  __unused[6];
+} __attribute__((packed));
+
+struct st_tasklet_end_data {
+        u64 when;
+        u16 exe_pid;
+        u8      flushed;
+        u8      __unused[5];
+} __attribute__((packed));
+
+
+struct st_work_release_data {
+        u64 when;
+        u64 __unused;
+} __attribute__((packed));
+
+struct st_work_begin_data {
+        u64 when;
+        u16 exe_pid;
+        u8      __unused[6];
+} __attribute__((packed));
+
+struct st_work_end_data {
+        u64 when;
+        u16 exe_pid;
+        u8      flushed;
+        u8      __unused[5];
+} __attribute__((packed));
+
+struct st_effective_priority_change_data {
+        u64 when;
+        u16 inh_pid;
+        u8      __unused[6];
+} __attribute__((packed));
+
+struct st_nv_interrupt_begin_data {
+        u64 when;
+        u32 device;
+        u32 serialNumber;
+} __attribute__((packed));
+
+struct st_nv_interrupt_end_data {
+        u64 when;
+        u32 device;
+        u32 serialNumber;
+} __attribute__((packed));
+
+struct st_prediction_err_data {
+        u64 distance;
+        u64 rel_err;
+} __attribute__((packed));
+
+struct st_migration_data {
+        u64 observed;
+        u64 estimated;
+} __attribute__((packed));
+
+struct migration_info {
+        u64 observed;
+        u64 estimated;
+        u8 distance;
+} __attribute__((packed));
 
 #define DATA(x) struct st_ ## x ## _data x;
 
 typedef enum {
-        ST_NAME = 1,            /* Start at one, so that we can spot
-                                 * uninitialized records. */
+    ST_NAME = 1, /* Start at one, so that we can spot
+                                  * uninitialized records. */
         ST_PARAM,
         ST_RELEASE,
         ST_ASSIGNED,
@@ -94,7 +167,19 @@ typedef enum {
         ST_BLOCK,
         ST_RESUME,
         ST_ACTION,
-        ST_SYS_RELEASE
+        ST_SYS_RELEASE,
+        ST_TASKLET_RELEASE,
+        ST_TASKLET_BEGIN,
+        ST_TASKLET_END,
+        ST_WORK_RELEASE,
+        ST_WORK_BEGIN,
+        ST_WORK_END,
+        ST_EFF_PRIO_CHANGE,
+        ST_NV_INTERRUPT_BEGIN,
+        ST_NV_INTERRUPT_END,
+
+        ST_PREDICTION_ERR,
+        ST_MIGRATION,
 } st_event_record_type_t;
 
 struct st_event_record {
@@ -113,8 +198,20 @@ struct st_event_record {
                 DATA(resume);
                 DATA(action);
                 DATA(sys_release);
+                DATA(tasklet_release);
+                DATA(tasklet_begin);
+                DATA(tasklet_end);
+                DATA(work_release);
+                DATA(work_begin);
+                DATA(work_end);
+                DATA(effective_priority_change);
+                DATA(nv_interrupt_begin);
+                DATA(nv_interrupt_end);
+
+                DATA(prediction_err);
+                DATA(migration);
         } data;
-};
+} __attribute__((packed));
 
 #undef DATA
 
@@ -129,6 +226,8 @@ struct st_event_record {
         ft_event1(id, callback, task)
 #define SCHED_TRACE2(id, callback, task, xtra) \
         ft_event2(id, callback, task, xtra)
+#define SCHED_TRACE3(id, callback, task, xtra1, xtra2) \
+        ft_event3(id, callback, task, xtra1, xtra2)
 
 /* provide prototypes; needed on sparc64 */
 #ifndef NO_TASK_TRACE_DECLS
@@ -155,12 +254,58 @@ feather_callback void do_sched_trace_action(unsigned long id,
 feather_callback void do_sched_trace_sys_release(unsigned long id,
                                                  lt_t* start);
 
+
+feather_callback void do_sched_trace_tasklet_release(unsigned long id,
+                                                                                                   struct task_struct* owner);
+feather_callback void do_sched_trace_tasklet_begin(unsigned long id,
+                                                                                                  struct task_struct* owner);
+feather_callback void do_sched_trace_tasklet_end(unsigned long id,
+                                                                                                 struct task_struct* owner,
+                                                                                                 unsigned long flushed);
+
+feather_callback void do_sched_trace_work_release(unsigned long id,
+                                                                                                         struct task_struct* owner);
+feather_callback void do_sched_trace_work_begin(unsigned long id,
+                                                                                                struct task_struct* owner,
+                                                                                                struct task_struct* exe);
+feather_callback void do_sched_trace_work_end(unsigned long id,
+                                                                                          struct task_struct* owner,
+                                                                                          struct task_struct* exe,
+                                                                                          unsigned long flushed);
+
+feather_callback void do_sched_trace_eff_prio_change(unsigned long id,
+                                                                                          struct task_struct* task,
+                                                                                          struct task_struct* inh);
+
+feather_callback void do_sched_trace_nv_interrupt_begin(unsigned long id,
+                                                                                                u32 device);
+feather_callback void do_sched_trace_nv_interrupt_end(unsigned long id,
+                                                                                                unsigned long unused);
+
+feather_callback void do_sched_trace_prediction_err(unsigned long id,
+                                                                                                          struct task_struct* task,
+                                                                                                          gpu_migration_dist_t* distance,
+                                                                                                          fp_t* rel_err);
+
+
+
+
+
+feather_callback void do_sched_trace_migration(unsigned long id,
+                                                                                          struct task_struct* task,
+                                                                                          struct migration_info* mig_info);
+
+
+/* returns true if we're tracing an interrupt on current CPU */
+/* int is_interrupt_tracing_active(void); */
+
 #endif
 
 #else
 
 #define SCHED_TRACE(id, callback, task)        /* no tracing */
 #define SCHED_TRACE2(id, callback, task, xtra) /* no tracing */
+#define SCHED_TRACE3(id, callback, task, xtra1, xtra2)
 
 #endif
 
@@ -252,6 +397,41 @@ feather_callback void do_sched_trace_sys_release(unsigned long id,
                 trace_litmus_sys_release(when);                         \
         } while (0)
 
+#define sched_trace_tasklet_release(t) \
+        SCHED_TRACE(SCHED_TRACE_BASE_ID + 11, do_sched_trace_tasklet_release, t)
+
+#define sched_trace_tasklet_begin(t) \
+        SCHED_TRACE(SCHED_TRACE_BASE_ID + 12, do_sched_trace_tasklet_begin, t)
+
+#define sched_trace_tasklet_end(t, flushed) \
+        SCHED_TRACE2(SCHED_TRACE_BASE_ID + 13, do_sched_trace_tasklet_end, t, flushed)
+
+
+#define sched_trace_work_release(t) \
+        SCHED_TRACE(SCHED_TRACE_BASE_ID + 14, do_sched_trace_work_release, t)
+
+#define sched_trace_work_begin(t, e) \
+        SCHED_TRACE2(SCHED_TRACE_BASE_ID + 15, do_sched_trace_work_begin, t, e)
+
+#define sched_trace_work_end(t, e, flushed) \
+        SCHED_TRACE3(SCHED_TRACE_BASE_ID + 16, do_sched_trace_work_end, t, e, flushed)
+
+
+#define sched_trace_eff_prio_change(t, inh) \
+        SCHED_TRACE2(SCHED_TRACE_BASE_ID + 17, do_sched_trace_eff_prio_change, t, inh)
+
+
+#define sched_trace_nv_interrupt_begin(d) \
+        SCHED_TRACE(SCHED_TRACE_BASE_ID + 18, do_sched_trace_nv_interrupt_begin, d)
+#define sched_trace_nv_interrupt_end(d) \
+        SCHED_TRACE(SCHED_TRACE_BASE_ID + 19, do_sched_trace_nv_interrupt_end, d)
+
+#define sched_trace_prediction_err(t, dist, rel_err) \
+        SCHED_TRACE3(SCHED_TRACE_BASE_ID + 20, do_sched_trace_prediction_err, t, dist, rel_err)
+
+#define sched_trace_migration(t, mig_info) \
+        SCHED_TRACE2(SCHED_TRACE_BASE_ID + 21, do_sched_trace_migration, t, mig_info)
+
 #define sched_trace_quantum_boundary() /* NOT IMPLEMENTED */
 
 #endif /* __KERNEL__ */
diff --git a/include/litmus/sched_trace_external.h b/include/litmus/sched_trace_external.h
new file mode 100644
index 000000000000..e70e45e4cf51
--- /dev/null
+++ b/include/litmus/sched_trace_external.h
@@ -0,0 +1,78 @@
+/*
+ * sched_trace.h -- record scheduler events to a byte stream for offline analysis.
+ */
+#ifndef _LINUX_SCHED_TRACE_EXTERNAL_H_
+#define _LINUX_SCHED_TRACE_EXTERNAL_H_
+
+
+#ifdef CONFIG_SCHED_TASK_TRACE
+extern void __sched_trace_tasklet_begin_external(struct task_struct* t);
+static inline void sched_trace_tasklet_begin_external(struct task_struct* t)
+{
+        __sched_trace_tasklet_begin_external(t);
+}
+
+extern void __sched_trace_tasklet_end_external(struct task_struct* t, unsigned long flushed);
+static inline void sched_trace_tasklet_end_external(struct task_struct* t, unsigned long flushed)
+{
+        __sched_trace_tasklet_end_external(t, flushed);
+}
+
+extern void __sched_trace_work_begin_external(struct task_struct* t, struct task_struct* e);
+static inline void sched_trace_work_begin_external(struct task_struct* t, struct task_struct* e)
+{
+        __sched_trace_work_begin_external(t, e);
+}
+
+extern void __sched_trace_work_end_external(struct task_struct* t, struct task_struct* e, unsigned long f);
+static inline void sched_trace_work_end_external(struct task_struct* t, struct task_struct* e, unsigned long f)
+{
+        __sched_trace_work_end_external(t, e, f);
+}
+
+#ifdef CONFIG_LITMUS_NVIDIA
+extern void __sched_trace_nv_interrupt_begin_external(u32 device);
+static inline void sched_trace_nv_interrupt_begin_external(u32 device)
+{
+        __sched_trace_nv_interrupt_begin_external(device);
+}
+
+extern void __sched_trace_nv_interrupt_end_external(u32 device);
+static inline void sched_trace_nv_interrupt_end_external(u32 device)
+{
+        __sched_trace_nv_interrupt_end_external(device);
+}
+#endif
+
+#else
+
+// no tracing.
+static inline void sched_trace_tasklet_begin_external(struct task_struct* t){}
+static inline void sched_trace_tasklet_end_external(struct task_struct* t, unsigned long flushed){}
+static inline void sched_trace_work_begin_external(struct task_struct* t, struct task_struct* e){}
+static inline void sched_trace_work_end_external(struct task_struct* t, struct task_struct* e, unsigned long f){}
+
+#ifdef CONFIG_LITMUS_NVIDIA
+static inline void sched_trace_nv_interrupt_begin_external(u32 device){}
+static inline void sched_trace_nv_interrupt_end_external(u32 device){}
+#endif
+
+#endif
+
+
+#ifdef CONFIG_LITMUS_NVIDIA
+
+#define EX_TS(evt) \
+extern void __##evt(void); \
+static inline void EX_##evt(void) { __##evt(); }
+
+EX_TS(TS_NV_TOPISR_START)
+EX_TS(TS_NV_TOPISR_END)
+EX_TS(TS_NV_BOTISR_START)
+EX_TS(TS_NV_BOTISR_END)
+EX_TS(TS_NV_RELEASE_BOTISR_START)
+EX_TS(TS_NV_RELEASE_BOTISR_END)
+
+#endif
+
+#endif
diff --git a/include/litmus/signal.h b/include/litmus/signal.h
new file mode 100644
index 000000000000..b3d82b294984
--- /dev/null
+++ b/include/litmus/signal.h
@@ -0,0 +1,47 @@
+#ifndef LITMUS_SIGNAL_H
+#define LITMUS_SIGNAL_H
+
+#ifdef __KERNEL__
+#include <linux/signal.h>
+#else
+#include <signal.h>
+#endif
+
+/* Signals used by Litmus to asynchronously communicate events
+ * to real-time tasks.
+ * 
+ * Signal values overlap with [SIGRTMIN, SIGRTMAX], so beware of
+ * application-level conflicts when dealing with COTS user-level
+ * code.
+ */
+
+/* Sent to a Litmus task when all of the following conditions are true:
+ * (1) The task has exhausted its budget.
+ * (2) budget_signal_policy is QUANTUM_SIGNALS or PRECISE_SIGNALS.
+ *
+ * Note: If a task does not have a registered handler for SIG_BUDGET,
+ * the signal will cause the task to terminate (default action).
+ */
+
+/* Assigned values start at SIGRTMAX and decrease, hopefully reducing
+ * likelihood of user-level conflicts.
+ */
+#define SIG_BUDGET                                      (SIGRTMAX - 0)
+
+/*
+Future signals could include:
+
+#define SIG_DEADLINE_MISS                       (SIGRTMAX - 1)
+#define SIG_CRIT_LEVEL_CHANGE           (SIGRTMAX - 2)
+*/
+
+#define SIGLITMUSMIN                            SIG_BUDGET
+
+#ifdef __KERNEL__
+#if (SIGLITMUSMIN < SIGRTMIN)
+/* no compile-time check in user-space since SIGRTMIN may be a variable. */
+#error "Too many LITMUS^RT signals!"
+#endif
+#endif
+
+#endif
diff --git a/include/litmus/trace.h b/include/litmus/trace.h
index e809376d6487..e078aee4234d 100644
--- a/include/litmus/trace.h
+++ b/include/litmus/trace.h
@@ -103,14 +103,46 @@ feather_callback void save_task_latency(unsigned long event, unsigned long when_
 #define TS_LOCK_START                   TIMESTAMP(170)
 #define TS_LOCK_SUSPEND                 TIMESTAMP(171)
 #define TS_LOCK_RESUME                  TIMESTAMP(172)
-#define TS_LOCK_END                     TIMESTAMP(173)
+#define TS_LOCK_END                             TIMESTAMP(173)
+
+#ifdef CONFIG_LITMUS_DGL_SUPPORT
+#define TS_DGL_LOCK_START                       TIMESTAMP(175)
+#define TS_DGL_LOCK_SUSPEND                     TIMESTAMP(176)
+#define TS_DGL_LOCK_RESUME                      TIMESTAMP(177)
+#define TS_DGL_LOCK_END                         TIMESTAMP(178)
+#endif
 
 #define TS_UNLOCK_START                 TIMESTAMP(180)
 #define TS_UNLOCK_END                   TIMESTAMP(181)
 
+#ifdef CONFIG_LITMUS_DGL_SUPPORT
+#define TS_DGL_UNLOCK_START                     TIMESTAMP(185)
+#define TS_DGL_UNLOCK_END                       TIMESTAMP(186)
+#endif
+
 #define TS_SEND_RESCHED_START(c)        CTIMESTAMP(190, c)
 #define TS_SEND_RESCHED_END             DTIMESTAMP(191, TSK_UNKNOWN)
 
 #define TS_RELEASE_LATENCY(when)        LTIMESTAMP(208, &(when))
 
+
+#ifdef CONFIG_LITMUS_NVIDIA
+
+#define TS_NV_TOPISR_START              TIMESTAMP(200)
+#define TS_NV_TOPISR_END                TIMESTAMP(201)
+
+#define TS_NV_BOTISR_START              TIMESTAMP(202)
+#define TS_NV_BOTISR_END                TIMESTAMP(203)
+
+#define TS_NV_RELEASE_BOTISR_START      TIMESTAMP(204)
+#define TS_NV_RELEASE_BOTISR_END        TIMESTAMP(205)
+
+#endif
+
+#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
+#define TS_NV_SCHED_BOTISR_START        TIMESTAMP(206)
+#define TS_NV_SCHED_BOTISR_END          TIMESTAMP(207)
+#endif
+
+
 #endif /* !_SYS_TRACE_H_ */
diff --git a/include/litmus/unistd_32.h b/include/litmus/unistd_32.h
index bcb8f1183b4f..4fd92956d13f 100644
--- a/include/litmus/unistd_32.h
+++ b/include/litmus/unistd_32.h
@@ -17,7 +17,10 @@
 #define __NR_wait_for_ts_release __LSC(9)
 #define __NR_release_ts         __LSC(10)
 #define __NR_null_call          __LSC(11)
+#define __NR_litmus_dgl_lock    __LSC(12)
+#define __NR_litmus_dgl_unlock  __LSC(13)
+#define __NR_register_nv_device                 __LSC(14)
 
-#define __NR_slave_non_rt_threads       _LSC(12)
+#define __NR_slave_non_rt_threads       _LSC(15)
 
-#define NR_litmus_syscalls 13
+#define NR_litmus_syscalls 16
diff --git a/include/litmus/unistd_64.h b/include/litmus/unistd_64.h
index 5f56d5947343..abb45c181e8e 100644
--- a/include/litmus/unistd_64.h
+++ b/include/litmus/unistd_64.h
@@ -29,8 +29,14 @@ __SYSCALL(__NR_wait_for_ts_release, sys_wait_for_ts_release)
 __SYSCALL(__NR_release_ts, sys_release_ts)
 #define __NR_null_call                          __LSC(11)
 __SYSCALL(__NR_null_call, sys_null_call)
+#define __NR_litmus_dgl_lock            __LSC(12)
+__SYSCALL(__NR_litmus_dgl_lock, sys_litmus_dgl_lock)
+#define __NR_litmus_dgl_unlock          __LSC(13)
+__SYSCALL(__NR_litmus_dgl_unlock, sys_litmus_dgl_unlock)
+#define __NR_register_nv_device                 __LSC(14)
+__SYSCALL(__NR_register_nv_device, sys_register_nv_device)
 
-#define __NR_slave_non_rt_threads       __LSC(12)
+#define __NR_slave_non_rt_threads       __LSC(15)
 __SYSCALL(__NR_slave_non_rt_threads, sys_slave_non_rt_threads)
 
-#define NR_litmus_syscalls 13
+#define NR_litmus_syscalls 16