4 files changed, 158 insertions, 22 deletions

diff --git a/include/litmus/nvidia_info.h b/include/litmus/nvidia_info.h
index f28d8f0a7f40..c5bf3dc0a210 100644
--- a/include/litmus/nvidia_info.h
+++ b/include/litmus/nvidia_info.h
@@ -9,5 +9,8 @@ int is_nvidia_func(void *func);
 
 int is_nvidia_in_callstack(void);
 
+// Returns the Nvidia device # associated with the
+// provided tasklet.
+u32 get_nv_device_num(const struct tasklet_struct *t);
 
 #endif
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 266cea2b9721..6f977ebc2770 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -29,6 +29,10 @@
 #include <trace/events/irq.h>
 
 #include <asm/irq.h>
+
+#include <litmus/sched_trace.h>
+#include <litmus/nvidia_info.h>
+
 /*
    - No shared variables, all the data are CPU local.
    - If a softirq needs serialization, let it serialize itself
@@ -54,7 +58,7 @@ EXPORT_SYMBOL(irq_stat);
 
 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
 
-static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
+static DEFINE_PER_CPU(struct task_struct *, ksoftirqd) = NULL;
 
 char *softirq_to_name[NR_SOFTIRQS] = {
         "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
@@ -177,6 +181,7 @@ void local_bh_enable_ip(unsigned long ip)
 }
 EXPORT_SYMBOL(local_bh_enable_ip);
 
+
 /*
  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
  * and we fall back to softirqd after that.
@@ -186,35 +191,36 @@ EXPORT_SYMBOL(local_bh_enable_ip);
  * we want to handle softirqs as soon as possible, but they
  * should not be able to lock up the box.
  */
+#ifndef LITMUS_THREAD_ALL_SOFTIRQ
 #define MAX_SOFTIRQ_RESTART 10
+#else
+#define MAX_SOFTIRQ_RESTART 20
+#endif
 
-asmlinkage void __do_softirq(void)
+static void ____do_softirq(void)
 {
-        struct softirq_action *h;
         __u32 pending;
-        int max_restart = MAX_SOFTIRQ_RESTART;
+        
+        struct softirq_action *h;
         int cpu;
-
+        
         pending = local_softirq_pending();
+        
         account_system_vtime(current);
-
-        __local_bh_disable((unsigned long)__builtin_return_address(0));
-        lockdep_softirq_enter();
-
+        
         cpu = smp_processor_id();
-restart:
-        /* Reset the pending bitmask before enabling irqs */
-        set_softirq_pending(0);
 
+        set_softirq_pending(0);
+        
         local_irq_enable();
-
+        
         h = softirq_vec;
-
+        
         do {
                 if (pending & 1) {
                         int prev_count = preempt_count();
                         kstat_incr_softirqs_this_cpu(h - softirq_vec);
-
+                        
                         trace_softirq_entry(h, softirq_vec);
                         h->action(h);
                         trace_softirq_exit(h, softirq_vec);
@@ -226,26 +232,68 @@ restart:
                                        h->action, prev_count, preempt_count());
                                 preempt_count() = prev_count;
                         }
-
+                        
                         rcu_bh_qs(cpu);
                 }
                 h++;
                 pending >>= 1;
         } while (pending);
-
+        
         local_irq_disable();
+}
+
+static void ___do_softirq(void)
+{
+        __u32 pending;
+
+        //struct softirq_action *h;
+        int max_restart = MAX_SOFTIRQ_RESTART;
+        //int cpu;
+
+        pending = local_softirq_pending();
+
+restart:
+        ____do_softirq();
 
         pending = local_softirq_pending();
         if (pending && --max_restart)
                 goto restart;
 
         if (pending)
+        {
                 wakeup_softirqd();
+        }
+}
 
+asmlinkage void __do_softirq(void)
+{
+#ifdef LITMUS_THREAD_ALL_SOFTIRQ
+        /* Skip straight to wakeup_softirqd() if we're using 
+         LITMUS_THREAD_ALL_SOFTIRQ.  This will cause ALL softirqs
+         and tasklets be threaded, either on Litmus klitirqd
+         threads or Linux ksoftirqd threads. */
+        struct task_struct *tsk = __get_cpu_var(ksoftirqd);
+        
+        if(tsk)
+        {
+                if(local_softirq_pending())
+                        wakeup_softirqd();
+                return;
+        }
+#endif
+        
+        /*
+         * 'immediate' softirq execution:
+         */
+        __local_bh_disable((unsigned long)__builtin_return_address(0));
+        lockdep_softirq_enter();
+        
+        ___do_softirq();
+        
         lockdep_softirq_exit();
-
+        
         account_system_vtime(current);
-        _local_bh_enable();
+        _local_bh_enable();     
 }
 
 #ifndef __ARCH_HAS_DO_SOFTIRQ
@@ -361,6 +409,26 @@ void __tasklet_schedule(struct tasklet_struct *t)
 {
         unsigned long flags;
 
+        if(is_nvidia_func(t->func))
+        {
+                u32 nvidiaDevice = get_nv_device_num(t);
+                
+                TRACE("%s: Handling Nvidia tasklet for device %u\n",
+                          __FUNCTION__, nvidiaDevice);
+                
+                BUG_ON(nvidiaDevice > 7); /* for Bonham.  remove for general support. */
+                
+                /*
+                 TODO:
+                 1) Ask Litmus which task owns GPU <nvidiaDevice>. (API to be defined.)
+                 2) If there is an owner, set t->owner to the owner's task struct.
+                   3) Call litmus_tasklet_schedule() and return (don't execute the rest
+                      of __tasklet_schedule()).
+                 4) If there is no owner, then continue on to the regular
+                        tasklet processing below.
+                 */
+        }
+        
         local_irq_save(flags);
         t->next = NULL;
         *__get_cpu_var(tasklet_vec).tail = t;
@@ -375,6 +443,24 @@ void __tasklet_hi_schedule(struct tasklet_struct *t)
 {
         unsigned long flags;
 
+        if(is_nvidia_func(t->func))
+        {
+                u32 nvidiaDevice = get_nv_device_num(t);
+                
+                TRACE("%s: Handling Nvidia tasklet for device %u\n",
+                          __FUNCTION__, nvidiaDevice);
+                
+                /*
+                 TODO:
+                 1) Ask Litmus which task owns GPU <nvidiaDevice>. (API to be defined.)
+                 2) If there is an owner, set t->owner to the owner's task struct.
+                   3) Call litmus_tasklet_hi_schedule() and return (don't execute the rest
+                      of __tasklet_schedule()).
+                 4) If there is no owner, then continue on to the regular
+                    tasklet processing below.
+                 */
+        }       
+        
         local_irq_save(flags);
         t->next = NULL;
         *__get_cpu_var(tasklet_hi_vec).tail = t;
@@ -389,6 +475,24 @@ void __tasklet_hi_schedule_first(struct tasklet_struct *t)
 {
         BUG_ON(!irqs_disabled());
 
+        if(is_nvidia_func(t->func))
+        {
+                u32 nvidiaDevice = get_nv_device_num(t);
+                
+                TRACE("%s: Handling Nvidia tasklet for device %u\n",
+                          __FUNCTION__, nvidiaDevice);
+                
+                /*
+                 TODO:
+                 1) Ask Litmus which task owns GPU <nvidiaDevice>. (API to be defined.)
+                 2) If there is an owner, set t->owner to the owner's task struct.
+                   3) Call litmus_tasklet_hi_schedule first() and return
+                          (don't execute the rest of __tasklet_schedule()).
+                 4) If there is no owner, then continue on to the regular
+                    tasklet processing below.
+                 */
+        }               
+        
         t->next = __get_cpu_var(tasklet_hi_vec).head;
         __get_cpu_var(tasklet_hi_vec).head = t;
         __raise_softirq_irqoff(HI_SOFTIRQ);
@@ -698,6 +802,8 @@ void __init softirq_init(void)
 
 static int run_ksoftirqd(void * __bind_cpu)
 {
+        unsigned long flags;
+        
         set_current_state(TASK_INTERRUPTIBLE);
 
         while (!kthread_should_stop()) {
@@ -716,7 +822,11 @@ static int run_ksoftirqd(void * __bind_cpu)
                            don't process */
                         if (cpu_is_offline((long)__bind_cpu))
                                 goto wait_to_die;
-                        do_softirq();
+                        
+                        local_irq_save(flags);
+                        ____do_softirq();
+                        local_irq_restore(flags);
+                        
                         preempt_enable_no_resched();
                         cond_resched();
                         preempt_disable();
diff --git a/litmus/Kconfig b/litmus/Kconfig
index 42c5ff67c8c1..3e5ed05b0ccf 100644
--- a/litmus/Kconfig
+++ b/litmus/Kconfig
@@ -184,6 +184,17 @@ endmenu
 
 menu "Interrupt Handling"
 
+config LITMUS_THREAD_ALL_SOFTIRQ
+       bool "Process all softirqs in ksoftirqd threads."
+       default n
+       help
+         Thread all softirqs in ksoftirqd daemon threads,
+         similar to PREEMPT_RT.  I/O throughput will
+         will drop with this enabled, but latencies
+         due to interrupts will be reduced.
+
+         If unsure, say No.
+
 config LITMUS_SOFTIRQD
        bool "Spawn klitirqd interrupt handling threads."
            depends on LITMUS_LOCKING
diff --git a/litmus/nvidia_info.c b/litmus/nvidia_info.c
index 05f8356a8117..c16879f54363 100644
--- a/litmus/nvidia_info.c
+++ b/litmus/nvidia_info.c
@@ -14,7 +14,7 @@ int init_nvidia_info(void)
         
         if(nvidia_mod != NULL)
         {
-                TRACE("%s : Found NVIDIA module. Core Code: %x to %x\n", __FUNCTION__,
+                TRACE("%s : Found NVIDIA module. Core Code: %p to %p\n", __FUNCTION__,
                           (void*)(nvidia_mod->module_core),
                           (void*)(nvidia_mod->module_core) + nvidia_mod->core_size);
                 return(0);
@@ -35,7 +35,7 @@ int is_nvidia_func(void *func_addr)
         {
                 ret = within_module_core((long unsigned int)func_addr, nvidia_mod);
                 
-                TRACE("%s : %x is in NVIDIA module : %d\n",
+                TRACE("%s : %p is in NVIDIA module : %d\n",
                           __FUNCTION__, func_addr, ret);
         }
         else
@@ -53,3 +53,15 @@ int is_nvidia_in_callstack(void)
         TRACE("%s : NOT IMPLEMENTED!\n", __FUNCTION__);
         return(1);
 }
+
+
+u32 get_nv_device_num(const struct tasklet_struct *t)
+{
+        // offset determined though observed behavior of the NV driver.
+        const int DEVICE_NUM_OFFSET = 0x278;
+        
+        void* state = (void*)(t->data);
+        void* device_num_ptr = state + DEVICE_NUM_OFFSET;
+        
+        return(*((u32*)device_num_ptr));
+}