Fix several klmirqd bugs.

1) Deadlock in litmus_task_exit()-- added litmus_pre_task_exit()
   to be called without the Linux runqueue lock held.
2) Prioritization of base-prio klmirqd/aux threads vs. normal
   real-time tasks.
3) Initialization of gpu owner binheap node moved to *after*
   memset(0) of rt_params.
4) Exit path of klmirqd threads.

8 files changed, 138 insertions, 28 deletions

diff --git a/include/litmus/litmus.h b/include/litmus/litmus.h
index 711b88e2b3d1..54f33e835682 100644
--- a/include/litmus/litmus.h
+++ b/include/litmus/litmus.h
@@ -38,6 +38,8 @@ void litmus_exec(void);
 void exit_litmus(struct task_struct *dead_tsk);
 
 long litmus_admit_task(struct task_struct *tsk);
+
+void litmus_pre_exit_task(struct task_struct *tsk); // called before litmus_exit_task, but without run queue locks held
 void litmus_exit_task(struct task_struct *tsk);
 
 #define is_realtime(t)          ((t)->policy == SCHED_LITMUS)
diff --git a/kernel/sched.c b/kernel/sched.c
index 840f87bce097..a1f10984adb3 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -5288,6 +5288,9 @@ recheck:
                 if (retval)
                         return retval;
         }
+        else if (p->policy == SCHED_LITMUS) {
+                litmus_pre_exit_task(p);
+        }
 
         /*
          * make sure no PI-waiters arrive (or leave) while we are
diff --git a/litmus/Kconfig b/litmus/Kconfig
index 8ca66b4d687c..b704e893e9be 100644
--- a/litmus/Kconfig
+++ b/litmus/Kconfig
@@ -470,6 +470,14 @@ config CUDA_3_2
 
 endchoice
 
+config LITMUS_NV_KLMIRQD_DEBUG
+          bool "Raise fake sporadic tasklets to test nv klimirqd threads."
+          depends on LITMUS_NVIDIA && LITMUS_SOFTIRQD
+          default n
+          help
+                Causes tasklets to be sporadically dispatched to waiting klmirqd
+                threads.
+
 endmenu
 
 endmenu
diff --git a/litmus/edf_common.c b/litmus/edf_common.c
index 27b728a55669..255e4f36e413 100644
--- a/litmus/edf_common.c
+++ b/litmus/edf_common.c
@@ -119,8 +119,15 @@ int edf_higher_prio(struct task_struct* first, struct task_struct* second)
                                 goto aux_tie_break;
                         }
                         else {
+
                                 // make the aux thread lowest priority real-time task
-                                int temp = (first_lo_aux) ? !is_realtime(second) : !is_realtime(first);
+                                int temp = 0;
+                                if (first_lo_aux && is_realtime(second)) {
+//                                      temp = 0;
+                                }
+                                else if(second_lo_aux && is_realtime(first)) {
+                                        temp = 1;
+                                }
                                 TRACE_CUR("%s/%d >> %s/%d --- %d\n", first->comm, first->pid, second->comm, second->pid, temp);
                                 return temp;
                         }
@@ -149,8 +156,15 @@ int edf_higher_prio(struct task_struct* first, struct task_struct* second)
                                 goto klmirqd_tie_break;
                         }
                         else {
-                                // make the klmirqd thread (second) lowest priority real-time task
-                                int temp = (first_lo_klmirqd) ? !is_realtime(second) : !is_realtime(first);
+                                // make the klmirqd thread the lowest-priority real-time task
+                                // but (above low-prio aux tasks and Linux tasks)
+                                int temp = 0;
+                                if (first_lo_klmirqd && is_realtime(second)) {
+//                                      temp = 0;
+                                }
+                                else if(second_lo_klmirqd && is_realtime(first)) {
+                                        temp = 1;
+                                }
                                 TRACE_CUR("%s/%d >> %s/%d --- %d\n", first->comm, first->pid, second->comm, second->pid, temp);
                                 return temp;
                         }
diff --git a/litmus/litmus.c b/litmus/litmus.c
index f98aa9d778a2..1aada57176de 100644
--- a/litmus/litmus.c
+++ b/litmus/litmus.c
@@ -355,8 +355,8 @@ static void reinit_litmus_state(struct task_struct* p, int restore)
                 ctrl_page   = p->rt_param.ctrl_page;
         }
 
-#ifdef CONFIG_LITMUS_NESTED_LOCKING
-        prio_order = p->rt_param.hp_blocked_tasks.compare;
+#ifdef CONFIG_LITMUS_NVIDIA
+        WARN_ON(p->rt_param.held_gpus != 0);
 #endif
 
 #ifdef CONFIG_LITMUS_LOCKING
@@ -367,15 +367,7 @@ static void reinit_litmus_state(struct task_struct* p, int restore)
 #endif
 
 #ifdef CONFIG_LITMUS_NESTED_LOCKING
-//      WARN_ON(p->rt_param.blocked_lock);
-//    WARN_ON(!binheap_empty(&p->rt_param.hp_blocked_tasks));
-#endif
-
-
-#ifdef CONFIG_LITMUS_NVIDIA
-        WARN_ON(p->rt_param.held_gpus != 0);
-
-        INIT_BINHEAP_NODE(&p->rt_param.gpu_owner_node);
+        prio_order = p->rt_param.hp_blocked_tasks.compare;
 #endif
 
         /* Cleanup everything else. */
@@ -384,8 +376,9 @@ static void reinit_litmus_state(struct task_struct* p, int restore)
 #ifdef CONFIG_REALTIME_AUX_TASKS
         /* also clear out the aux_data. the !restore case is only called on
          * fork (initial thread creation). */
-        if (!restore)
+        if (!restore) {
                 memset(&p->aux_data, 0, sizeof(p->aux_data));
+        }
 #endif
 
         /* Restore preserved fields. */
@@ -394,6 +387,10 @@ static void reinit_litmus_state(struct task_struct* p, int restore)
                 p->rt_param.ctrl_page   = ctrl_page;
         }
 
+#ifdef CONFIG_LITMUS_NVIDIA
+        INIT_BINHEAP_NODE(&p->rt_param.gpu_owner_node);
+#endif
+
 #if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_AFFINITY_LOCKING)
         init_gpu_affinity_state(p);
 #endif
@@ -494,15 +491,20 @@ out:
         return retval;
 }
 
-void litmus_exit_task(struct task_struct* tsk)
+void litmus_pre_exit_task(struct task_struct* tsk)
 {
         if (is_realtime(tsk)) {
-                sched_trace_task_completion(tsk, 1);
-
                 if (tsk_rt(tsk)->rsrc_exit_cb) {
                         int ret = tsk_rt(tsk)->rsrc_exit_cb(tsk);
                         WARN_ON(ret != 0);
                 }
+        }
+}
+
+void litmus_exit_task(struct task_struct* tsk)
+{
+        if (is_realtime(tsk)) {
+                sched_trace_task_completion(tsk, 1);
 
                 litmus->task_exit(tsk);
 
@@ -637,8 +639,10 @@ void exit_litmus(struct task_struct *dead_tsk)
         }
 
         /* main cleanup only for RT tasks */
-        if (is_realtime(dead_tsk))
+        if (is_realtime(dead_tsk)) {
+                litmus_pre_exit_task(dead_tsk); /* todo: double check that no Linux rq lock is held */
                 litmus_exit_task(dead_tsk);
+        }
 }
 
 
diff --git a/litmus/litmus_softirq.c b/litmus/litmus_softirq.c
index 9c5ecab5e8d9..be06405021c5 100644
--- a/litmus/litmus_softirq.c
+++ b/litmus/litmus_softirq.c
@@ -1163,3 +1163,7 @@ int __litmus_schedule_work(struct work_struct *w, struct task_struct* klmirqd_th
 }
 EXPORT_SYMBOL(__litmus_schedule_work);
 
+
+
+
+
diff --git a/litmus/nvidia_info.c b/litmus/nvidia_info.c
index 3d38b168d9ba..059a7e7ac715 100644
--- a/litmus/nvidia_info.c
+++ b/litmus/nvidia_info.c
@@ -258,6 +258,8 @@ void dump_nvidia_info(const struct tasklet_struct *t)
 static struct module* nvidia_mod = NULL;
 
 
+
+
 #if 0
 static int nvidia_ready_module_notify(struct notifier_block *self,
                                 unsigned long val, void *data)
@@ -390,6 +392,10 @@ typedef struct {
         struct task_struct* thread;
         int ready:1;  /* todo: make threads check for the ready flag */
 #endif
+
+#ifdef CONFIG_LITMUS_NV_KLMIRQD_DEBUG
+        struct tasklet_struct nv_klmirqd_dbg_tasklet;
+#endif
 }nv_device_registry_t;
 
 
@@ -397,8 +403,6 @@ static nv_device_registry_t NV_DEVICE_REG[NV_DEVICE_NUM];
 
 
 
-
-
 #ifdef CONFIG_LITMUS_SOFTIRQD
 static int nvidia_klmirqd_cb(void *arg)
 {
@@ -417,6 +421,63 @@ static int nvidia_klmirqd_cb(void *arg)
 }
 #endif
 
+#ifdef CONFIG_LITMUS_NV_KLMIRQD_DEBUG
+struct nv_klmirqd_dbg_timer_struct
+{
+        struct hrtimer timer;
+};
+
+static struct nv_klmirqd_dbg_timer_struct nv_klmirqd_dbg_timer;
+
+static void nv_klmirqd_arm_dbg_timer(lt_t relative_time)
+{
+        lt_t when_to_fire = litmus_clock() + relative_time;
+
+        TRACE("next nv tasklet in %d ns\n", relative_time);
+
+        __hrtimer_start_range_ns(&nv_klmirqd_dbg_timer.timer,
+                                                         ns_to_ktime(when_to_fire),
+                                                         0,
+                                                         HRTIMER_MODE_ABS_PINNED,
+                                                         0);
+}
+
+static void nv_klmirqd_dbg_tasklet_func(unsigned long arg)
+{
+        lt_t now = litmus_clock();
+        nv_device_registry_t *reg = (nv_device_registry_t*)arg;
+        int gpunum = reg - &NV_DEVICE_REG[0];
+
+        TRACE("nv klmirqd routine invoked for GPU %d!\n", gpunum);
+
+        /* set up the next timer */
+        nv_klmirqd_arm_dbg_timer(now % (NSEC_PER_MSEC * 10)); // within the next 10ms.
+}
+
+
+static enum hrtimer_restart nvklmirqd_timer_func(struct hrtimer *timer)
+{
+        lt_t now = litmus_clock();
+        int gpu = (int)(now % num_online_gpus());
+        nv_device_registry_t *reg;
+
+        TRACE("nvklmirqd_timer invoked!\n");
+
+        reg = &NV_DEVICE_REG[gpu];
+
+        if (reg->thread && reg->ready) {
+                TRACE("Adding a tasklet for GPU %d\n", gpu);
+                litmus_tasklet_schedule(&reg->nv_klmirqd_dbg_tasklet, reg->thread);
+        }
+        else {
+                TRACE("nv klmirqd is not ready!\n");
+                nv_klmirqd_arm_dbg_timer(now % (NSEC_PER_MSEC * 10)); // within the next 10ms.
+        }
+
+        return HRTIMER_NORESTART;
+}
+#endif
+
 
 static int gpu_owner_max_priority_order(struct binheap_node *a,
                                                                                         struct binheap_node *b)
@@ -451,6 +512,10 @@ static int init_nv_device_reg(void)
                 raw_spin_lock_init(&NV_DEVICE_REG[i].lock);
                 INIT_BINHEAP_HANDLE(&NV_DEVICE_REG[i].owners, gpu_owner_max_priority_order);
 
+#ifdef CONFIG_LITMUS_NV_KLMIRQD_DEBUG
+                tasklet_init(&NV_DEVICE_REG[i].nv_klmirqd_dbg_tasklet, nv_klmirqd_dbg_tasklet_func, (unsigned long)&NV_DEVICE_REG[i]);
+#endif
+
 #ifdef CONFIG_LITMUS_SOFTIRQD
                 {
                         int default_cpu = litmus->map_gpu_to_cpu(i);
@@ -466,6 +531,12 @@ static int init_nv_device_reg(void)
 #endif
         }
 
+#ifdef CONFIG_LITMUS_NV_KLMIRQD_DEBUG
+        hrtimer_init(&nv_klmirqd_dbg_timer.timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
+        nv_klmirqd_dbg_timer.timer.function = nvklmirqd_timer_func;
+        nv_klmirqd_arm_dbg_timer(NSEC_PER_MSEC * 1000);
+#endif
+
         return(1);
 }
 
@@ -578,7 +649,7 @@ static int gpu_klmirqd_decrease_priority(struct task_struct *klmirqd, struct tas
 
 
 
-/* call when an aux_owner becomes real-time */
+/* call when an gpu owner becomes real-time */
 long enable_gpu_owner(struct task_struct *t)
 {
         long retval = 0;
@@ -631,7 +702,7 @@ out:
         return retval;
 }
 
-/* call when an aux_owner exits real-time */
+/* call when an gpu owner exits real-time */
 long disable_gpu_owner(struct task_struct *t)
 {
         long retval = 0;
@@ -773,9 +844,9 @@ int gpu_owner_decrease_priority(struct task_struct *t)
 
         gpu = find_first_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus));
 
-        if (!binheap_is_in_heap(&tsk_rt(t)->aux_task_owner_node)) {
+        if (!binheap_is_in_heap(&tsk_rt(t)->gpu_owner_node)) {
                 WARN_ON(!is_running(t));
-                TRACE_CUR("aux tasks may not inherit from %s/%d on GPU %d\n",
+                TRACE_CUR("nv klmirqd may not inherit from %s/%d on GPU %d\n",
                                   t->comm, t->pid, gpu);
                 goto out;
         }
@@ -865,6 +936,10 @@ int reg_nv_device(int reg_device_id, int reg_action, struct task_struct *t)
 
 
 
+
+
+
+
 #ifdef CONFIG_LITMUS_PAI_SOFTIRQD
 //void pai_check_priority_increase(struct task_struct *t, int reg_device_id)
 //{
diff --git a/litmus/sched_cedf.c b/litmus/sched_cedf.c
index 46de8041cf59..a454832b2aa8 100644
--- a/litmus/sched_cedf.c
+++ b/litmus/sched_cedf.c
@@ -1717,17 +1717,17 @@ static long cedf_activate_plugin(void)
         printk(KERN_INFO "C-EDF: %d cluster(s) of size = %d\n",
                         num_clusters, cluster_size);
 
-        
+
 #if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_SOFTIRQD)
         num_gpu_clusters = min(num_clusters, num_online_gpus());
         gpu_cluster_size = num_online_gpus() / num_gpu_clusters;
-        
+
         if (((num_online_gpus() % gpu_cluster_size) != 0) ||
                 (num_gpu_clusters != num_clusters)) {
                 printk(KERN_WARNING "C-EDF: GPUs not uniformly distributed among CPU clusters.\n");
         }
 #endif
-        
+
         /* initialize clusters */
         cedf = kmalloc(num_clusters * sizeof(cedf_domain_t), GFP_ATOMIC);
         for (i = 0; i < num_clusters; i++) {