9 files changed, 143 insertions, 166 deletions

diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index 57a7bc8807be..9fc31289a1bb 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -507,6 +507,12 @@ struct tasklet_struct
 #endif
 };
 
+struct tasklet_head
+{
+        struct tasklet_struct *head;
+        struct tasklet_struct **tail;
+};
+
 #define DECLARE_TASKLET(name, func, data) \
 struct tasklet_struct name = { NULL, 0, ATOMIC_INIT(0), func, data }
 
diff --git a/include/litmus/aux_tasks.h b/include/litmus/aux_tasks.h
index 87745c1c0df0..255bbafcc6b7 100644
--- a/include/litmus/aux_tasks.h
+++ b/include/litmus/aux_tasks.h
@@ -3,9 +3,6 @@
 
 struct task_struct;
 
-/* admit an aux task with default parameters */
-//int admit_aux_task(struct task_struct *t);
-
 int make_aux_task_if_required(struct task_struct *t);
 
 /* call on an aux task when it exits real-time */
@@ -17,13 +14,6 @@ long enable_aux_task_owner(struct task_struct *t);
 /* call when an aux_owner exits real-time */
 long disable_aux_task_owner(struct task_struct *t);
 
-
-/* collectivelly make all aux tasks in the process of leader inherit from hp */
-//int aux_tasks_increase_priority(struct task_struct *leader, struct task_struct *hp);
-
-/* collectivelly make all aux tasks in the process of leader inherit from hp */
-//int aux_tasks_decrease_priority(struct task_struct *leader, struct task_struct *hp);
-
 /* call when an aux_owner increases its priority */
 int aux_task_owner_increase_priority(struct task_struct *t);
 
diff --git a/include/litmus/litmus_softirq.h b/include/litmus/litmus_softirq.h
index 46fe89148505..52e3f7e74ab1 100644
--- a/include/litmus/litmus_softirq.h
+++ b/include/litmus/litmus_softirq.h
@@ -5,129 +5,113 @@
 #include <linux/workqueue.h>
 
 /*
-   Threaded tasklet handling for Litmus.  Tasklets
-   are scheduled with the priority of the tasklet's
-   owner---that is, the RT task on behalf the tasklet
-   runs.
-
-   Tasklets are current scheduled in FIFO order with
-   NO priority inheritance for "blocked" tasklets.
+   Threaded tasklet/workqueue handling for Litmus.
+   Items are scheduled in the following order: hi-tasklet,
+   lo-tasklet, workqueue.  Items are scheduled in FIFO order
+   within each of these classes.
 
    klmirqd assumes the priority of the owner of the
    tasklet when the tasklet is next to execute.
 
-   Currently, hi-tasklets are scheduled before
-   low-tasklets, regardless of priority of low-tasklets.
-   And likewise, low-tasklets are scheduled before work
-   queue objects.  This priority inversion probably needs
-   to be fixed, though it is not an issue if our work with
-   GPUs as GPUs are owned (and associated klmirqds) for
-   exclusive time periods, thus no inversions can
-   occur.
+   The base-priority of a klimirqd thread is below all regular
+   real-time tasks, but above all other Linux scheduling
+   classes (klmirqd threads are within the SHCED_LITMUS class).
+   Regular real-time tasks may increase the priority of
+   a klmirqd thread, but klmirqd is unaware of this
+   (this was not the case in prior incarnations of klmirqd).
  */
 
 
-
-#define NR_LITMUS_SOFTIRQD CONFIG_NR_LITMUS_SOFTIRQD
-
-/* Spawns NR_LITMUS_SOFTIRQD klmirqd daemons.
-   Actual launch of threads is deffered to kworker's
-   workqueue, so daemons will likely not be immediately
-   running when this function returns, though the required
-   data will be initialized.
-
-   @affinity_set: an array expressing the processor affinity
-    for each of the NR_LITMUS_SOFTIRQD daemons.  May be set
-    to NULL for global scheduling.
-
-        - Examples -
-        8-CPU system with two CPU clusters:
-                affinity[] = {0, 0, 0, 0, 3, 3, 3, 3}
-                NOTE: Daemons not actually bound to specified CPU, but rather
-                cluster in which the CPU resides.
-
-        8-CPU system, partitioned:
-                affinity[] = {0, 1, 2, 3, 4, 5, 6, 7}
-
-        FIXME: change array to a CPU topology or array of cpumasks
-
- */
-void spawn_klmirqd(int* affinity);
-
+/* Initialize klmirqd */
+void init_klmirqd(void);
 
 /* Raises a flag to tell klmirqds to terminate.
-   Termination is async, so some threads may be running
-   after function return. */
+ Termination is async, so some threads may be running
+ after function return. */
 void kill_klmirqd(void);
 
+void kill_klmirqd_thread(struct task_struct* klmirqd_thread);
 
 /* Returns 1 if all NR_LITMUS_SOFTIRQD klitirqs are ready
-   to handle tasklets. 0, otherwise.*/
+ to handle tasklets. 0, otherwise.*/
 int klmirqd_is_ready(void);
 
 /* Returns 1 if no NR_LITMUS_SOFTIRQD klitirqs are ready
-   to handle tasklets. 0, otherwise.*/
+ to handle tasklets. 0, otherwise.*/
 int klmirqd_is_dead(void);
 
-/* Flushes all pending work out to the OS for regular
- * tasklet/work processing of the specified 'owner'
- *
- * PRECOND: klmirqd_thread must have a clear entry
- * in the GPU registry, otherwise this call will become
- * a no-op as work will loop back to the klmirqd_thread.
- *
- * Pass NULL for owner to flush ALL pending items.
+
+typedef int (*klmirqd_cb_t) (void *arg);
+
+typedef struct
+{
+        klmirqd_cb_t func;
+        void* arg;
+} klmirqd_callback_t;
+
+/* Launches a klmirqd thread with the provided affinity.
+
+   Actual launch of threads is deffered to kworker's
+   workqueue, so daemons will likely not be immediately
+   running when this function returns, though the required
+   data will be initialized.
+
+   cpu == -1 for no affinity
  */
-void flush_pending(struct task_struct* klmirqd_thread,
-                                   struct task_struct* owner);
+int launch_klmirqd_thread(int cpu, klmirqd_callback_t* cb);
 
-struct task_struct* get_klmirqd(unsigned int k_id);
 
+/* Flushes all pending work out to the OS for regular
+ * tasklet/work processing.
+ */
+void flush_pending(struct task_struct* klmirqd_thread);
 
 extern int __litmus_tasklet_schedule(
         struct tasklet_struct *t,
-        unsigned int k_id);
+                struct task_struct *klmirqd_thread);
 
 /* schedule a tasklet on klmirqd #k_id */
 static inline int litmus_tasklet_schedule(
     struct tasklet_struct *t,
-    unsigned int k_id)
+    struct task_struct *klmirqd_thread)
 {
         int ret = 0;
-        if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))
-                ret = __litmus_tasklet_schedule(t, k_id);
+        if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
+                ret = __litmus_tasklet_schedule(t, klmirqd_thread);
+        }
         return(ret);
 }
 
 /* for use by __tasklet_schedule() */
 static inline int _litmus_tasklet_schedule(
     struct tasklet_struct *t,
-    unsigned int k_id)
+    struct task_struct *klmirqd_thread)
 {
-    return(__litmus_tasklet_schedule(t, k_id));
+    return(__litmus_tasklet_schedule(t, klmirqd_thread));
 }
 
 
 
 
 extern int __litmus_tasklet_hi_schedule(struct tasklet_struct *t,
-                                         unsigned int k_id);
+                                        struct task_struct *klmirqd_thread);
 
 /* schedule a hi tasklet on klmirqd #k_id */
 static inline int litmus_tasklet_hi_schedule(struct tasklet_struct *t,
-                                              unsigned int k_id)
+                                             struct task_struct *klmirqd_thread)
 {
         int ret = 0;
-        if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))
-                ret = __litmus_tasklet_hi_schedule(t, k_id);
+        if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
+                ret = __litmus_tasklet_hi_schedule(t, klmirqd_thread);
+        }
         return(ret);
 }
 
 /* for use by __tasklet_hi_schedule() */
 static inline int _litmus_tasklet_hi_schedule(struct tasklet_struct *t,
-                                               unsigned int k_id)
+                                              struct task_struct *klmirqd_thread)
 {
-    return(__litmus_tasklet_hi_schedule(t, k_id));
+    return(__litmus_tasklet_hi_schedule(t, klmirqd_thread));
 }
 
 
@@ -136,26 +120,27 @@ static inline int _litmus_tasklet_hi_schedule(struct tasklet_struct *t,
 
 extern int __litmus_tasklet_hi_schedule_first(
     struct tasklet_struct *t,
-    unsigned int k_id);
+    struct task_struct *klmirqd_thread);
 
 /* schedule a hi tasklet on klmirqd #k_id on next go-around */
 /* PRECONDITION: Interrupts must be disabled. */
 static inline int litmus_tasklet_hi_schedule_first(
     struct tasklet_struct *t,
-    unsigned int k_id)
+    struct task_struct *klmirqd_thread)
 {
         int ret = 0;
-        if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))
-                ret = __litmus_tasklet_hi_schedule_first(t, k_id);
+        if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
+                ret = __litmus_tasklet_hi_schedule_first(t, klmirqd_thread);
+        }
         return(ret);
 }
 
 /* for use by __tasklet_hi_schedule_first() */
 static inline int _litmus_tasklet_hi_schedule_first(
     struct tasklet_struct *t,
-    unsigned int k_id)
+    struct task_struct *klmirqd_thread)
 {
-    return(__litmus_tasklet_hi_schedule_first(t, k_id));
+    return(__litmus_tasklet_hi_schedule_first(t, klmirqd_thread));
 }
 
 
@@ -164,36 +149,13 @@ static inline int _litmus_tasklet_hi_schedule_first(
 
 extern int __litmus_schedule_work(
         struct work_struct* w,
-        unsigned int k_id);
+        struct task_struct *klmirqd_thread);
 
 static inline int litmus_schedule_work(
         struct work_struct* w,
-        unsigned int k_id)
+        struct task_struct *klmirqd_thread)
 {
-        return(__litmus_schedule_work(w, k_id));
+        return(__litmus_schedule_work(w, klmirqd_thread));
 }
 
-
-
-///////////// mutex operations for client threads.
-
-void down_and_set_stat(struct task_struct* t,
-                                         enum klmirqd_sem_status to_set,
-                                         struct mutex* sem);
-
-void __down_and_reset_and_set_stat(struct task_struct* t,
-                                enum klmirqd_sem_status to_reset,
-                                enum klmirqd_sem_status to_set,
-                                struct mutex* sem);
-
-void up_and_set_stat(struct task_struct* t,
-                                        enum klmirqd_sem_status to_set,
-                                        struct mutex* sem);
-
-
-
-void release_klmirqd_lock(struct task_struct* t);
-
-int reacquire_klmirqd_lock(struct task_struct* t);
-
 #endif
diff --git a/include/litmus/nvidia_info.h b/include/litmus/nvidia_info.h
index 97c9577141db..6f354c8b00ac 100644
--- a/include/litmus/nvidia_info.h
+++ b/include/litmus/nvidia_info.h
@@ -6,10 +6,9 @@
 
 #include <litmus/litmus_softirq.h>
 
-
-//#define NV_DEVICE_NUM NR_LITMUS_SOFTIRQD
 #define NV_DEVICE_NUM CONFIG_NV_DEVICE_NUM
-#define NV_MAX_SIMULT_USERS CONFIG_NV_MAX_SIMULT_USERS
+
+/* Functions used for decoding NVIDIA blobs. */
 
 int init_nvidia_info(void);
 void shutdown_nvidia_info(void);
@@ -18,29 +17,33 @@ int is_nvidia_func(void* func_addr);
 
 void dump_nvidia_info(const struct tasklet_struct *t);
 
-
 // Returns the Nvidia device # associated with provided tasklet and work_struct.
 u32 get_tasklet_nv_device_num(const struct tasklet_struct *t);
 u32 get_work_nv_device_num(const struct work_struct *t);
 
 
-int init_nv_device_reg(void);
-//int get_nv_device_id(struct task_struct* owner);
 
-
-int reg_nv_device(int reg_device_id, int register_device, struct task_struct *t);
+/* Functions for figuring out the priority of GPU-using tasks */
 
 struct task_struct* get_nv_max_device_owner(u32 target_device_id);
-//int is_nv_device_owner(u32 target_device_id);
-
-void lock_nv_registry(u32 reg_device_id, unsigned long* flags);
-void unlock_nv_registry(u32 reg_device_id, unsigned long* flags);
 
-#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
-void pai_check_priority_increase(struct task_struct *t, int reg_device_id);
-void pai_check_priority_decrease(struct task_struct *t, int reg_device_id);
+#ifdef CONFIG_LITMUS_SOFTIRQD
+struct task_struct* get_nv_klmirqd_thread(u32 target_device_id);
 #endif
 
-//void increment_nv_int_count(u32 device);
+/* call when the GPU-holding task, t, blocks */
+long enable_gpu_owner(struct task_struct *t);
+
+/* call when the GPU-holding task, t, resumes */
+long disable_gpu_owner(struct task_struct *t);
+
+/* call when the GPU-holding task, t, increases its priority */
+int gpu_owner_increase_priority(struct task_struct *t);
+
+/* call when the GPU-holding task, t, decreases its priority */
+int gpu_owner_decrease_priority(struct task_struct *t);
+
+
+int reg_nv_device(int reg_device_id, int reg_action, struct task_struct *t);
 
 #endif
diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h
index 47301c04d862..c8ee64569dbb 100644
--- a/include/litmus/rt_param.h
+++ b/include/litmus/rt_param.h
@@ -128,6 +128,10 @@ struct control_page {
 #include <litmus/binheap.h>
 #include <linux/semaphore.h>
 
+#ifdef CONFIG_LITMUS_SOFTIRQD
+#include <linux/interrupt.h>
+#endif
+
 struct _rt_domain;
 struct bheap_node;
 struct release_heap;
@@ -205,6 +209,38 @@ typedef struct avg_est{
         lt_t avg;
 } avg_est_t;
 
+
+
+#ifdef CONFIG_LITMUS_SOFTIRQD
+//struct tasklet_head
+//{
+//      struct tasklet_struct *head;
+//      struct tasklet_struct **tail;
+//};
+
+struct klmirqd_info
+{
+        struct task_struct*             klmirqd;
+    struct task_struct*     current_owner;
+    unsigned int                        terminating:1;
+
+        raw_spinlock_t                  lock;
+
+        u32                                             pending;
+        atomic_t                                num_hi_pending;
+        atomic_t                                num_low_pending;
+        atomic_t                                num_work_pending;
+
+        /* in order of priority */
+        struct tasklet_head     pending_tasklets_hi;
+        struct tasklet_head             pending_tasklets;
+        struct list_head                worklist;
+
+        struct list_head                klmirqd_reg;
+};
+#endif
+
+
 /*      RT task parameters for scheduling extensions
  *      These parameters are inherited during clone and therefore must
  *      be explicitly set up before the task set is launched.
@@ -221,34 +257,21 @@ struct rt_param {
 
 #ifdef CONFIG_LITMUS_SOFTIRQD
     /* proxy threads have minimum priority by default */
-    unsigned int        is_proxy_thread:1;
-
-        /* pointer to klmirqd currently working on this
-           task_struct's behalf.  only set by the task pointed
-           to by klmirqd.
+    unsigned int        is_interrupt_thread:1;
 
-           ptr only valid if is_proxy_thread == 0
-         */
-        struct task_struct* cur_klmirqd;
-
-        /* Used to implement mutual execution exclusion between
-         * job and klmirqd execution.  Job must always hold
-         * it's klmirqd_sem to execute.  klmirqd instance
-         * must hold the semaphore before executing on behalf
-         * of a job.
-         */
-        struct mutex                            klmirqd_sem;
-
-        /* status of held klmirqd_sem, even if the held klmirqd_sem is from
-           another task (only proxy threads do this though).
+        /* pointer to data used by klmirqd thread.
+         *
+         * ptr only valid if is_interrupt_thread == 1
          */
-        atomic_t                                        klmirqd_sem_stat;
+        struct klmirqd_info* klmirqd_info;
 #endif
 
 #ifdef CONFIG_LITMUS_NVIDIA
         /* number of top-half interrupts handled on behalf of current job */
         atomic_t                                        nv_int_count;
         long unsigned int                       held_gpus;  // bitmap of held GPUs.
+        struct binheap_node                     gpu_owner_node; // just one GPU for now...
+        unsigned int                            hide_from_gpu:1;
 
 #ifdef CONFIG_LITMUS_AFFINITY_LOCKING
         avg_est_t gpu_migration_est[MIG_LAST+1];
@@ -370,6 +393,13 @@ struct rt_param {
         struct control_page * ctrl_page;
 };
 
+//#ifdef CONFIG_LITMUS_SOFTIRQD
+//struct klmirqd_data
+//{
+//      struct binheap          klmirqd_users;
+//};
+//#endif
+
 #ifdef CONFIG_REALTIME_AUX_TASKS
 struct aux_data
 {
diff --git a/include/litmus/sched_plugin.h b/include/litmus/sched_plugin.h
index e8127f427d56..a13d1a2992fe 100644
--- a/include/litmus/sched_plugin.h
+++ b/include/litmus/sched_plugin.h
@@ -79,12 +79,6 @@ typedef void (*nested_increase_prio_t)(struct task_struct* t, struct task_struct
 typedef void (*nested_decrease_prio_t)(struct task_struct* t, struct task_struct* prio_inh,
                                                                           raw_spinlock_t *to_unlock, unsigned long irqflags);
 
-typedef void (*increase_prio_klitirq_t)(struct task_struct* klmirqd,
-                                        struct task_struct* old_owner,
-                                        struct task_struct* new_owner);
-typedef void (*decrease_prio_klmirqd_t)(struct task_struct* klmirqd,
-                                         struct task_struct* old_owner);
-
 
 typedef int (*enqueue_pai_tasklet_t)(struct tasklet_struct* tasklet);
 typedef void (*change_prio_pai_tasklet_t)(struct task_struct *old_prio,
@@ -166,11 +160,6 @@ struct sched_plugin {
 #ifdef CONFIG_LITMUS_AFFINITY_LOCKING
         allocate_affinity_observer_t allocate_aff_obs;
 #endif
-
-#ifdef CONFIG_LITMUS_SOFTIRQD
-    increase_prio_klitirq_t             increase_prio_klmirqd;
-    decrease_prio_klmirqd_t     decrease_prio_klmirqd;
-#endif
 #ifdef CONFIG_LITMUS_PAI_SOFTIRQD
         enqueue_pai_tasklet_t           enqueue_pai_tasklet;
         change_prio_pai_tasklet_t       change_prio_pai_tasklet;
diff --git a/include/litmus/signal.h b/include/litmus/signal.h
index b3d82b294984..38c3207951e0 100644
--- a/include/litmus/signal.h
+++ b/include/litmus/signal.h
@@ -9,7 +9,7 @@
 
 /* Signals used by Litmus to asynchronously communicate events
  * to real-time tasks.
- * 
+ *
  * Signal values overlap with [SIGRTMIN, SIGRTMAX], so beware of
  * application-level conflicts when dealing with COTS user-level
  * code.
diff --git a/include/litmus/unistd_32.h b/include/litmus/unistd_32.h
index c86b743408ed..7265ffadf555 100644
--- a/include/litmus/unistd_32.h
+++ b/include/litmus/unistd_32.h
@@ -19,8 +19,7 @@
 #define __NR_null_call          __LSC(11)
 #define __NR_litmus_dgl_lock    __LSC(12)
 #define __NR_litmus_dgl_unlock  __LSC(13)
-#define __NR_register_nv_device                 __LSC(14)
 
-#define __NR_set_aux_tasks              _LSC(15)
+#define __NR_set_aux_tasks              _LSC(14)
 
-#define NR_litmus_syscalls 16
+#define NR_litmus_syscalls 15
diff --git a/include/litmus/unistd_64.h b/include/litmus/unistd_64.h
index 3825bc129dbd..51e730124dde 100644
--- a/include/litmus/unistd_64.h
+++ b/include/litmus/unistd_64.h
@@ -33,10 +33,8 @@ __SYSCALL(__NR_null_call, sys_null_call)
 __SYSCALL(__NR_litmus_dgl_lock, sys_litmus_dgl_lock)
 #define __NR_litmus_dgl_unlock          __LSC(13)
 __SYSCALL(__NR_litmus_dgl_unlock, sys_litmus_dgl_unlock)
-#define __NR_register_nv_device         __LSC(14)
-__SYSCALL(__NR_register_nv_device, sys_register_nv_device)
 
-#define __NR_set_aux_tasks                      __LSC(15)
+#define __NR_set_aux_tasks                      __LSC(14)
 __SYSCALL(__NR_set_aux_tasks, sys_set_aux_tasks)
 
-#define NR_litmus_syscalls 16
+#define NR_litmus_syscalls 15