Full patch for klitirqd with Nvidia GPU support.

1 files changed, 252 insertions, 26 deletions

diff --git a/kernel/softirq.c b/kernel/softirq.c
index 07b4f1b1a73a..be4b8fab3637 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -29,6 +29,14 @@
 #include <trace/events/irq.h>
 
 #include <asm/irq.h>
+
+#include <litmus/litmus.h>
+#include <litmus/sched_trace.h>
+
+#ifdef CONFIG_LITMUS_NVIDIA
+#include <litmus/nvidia_info.h>
+#endif
+
 /*
    - No shared variables, all the data are CPU local.
    - If a softirq needs serialization, let it serialize itself
@@ -54,7 +62,7 @@ EXPORT_SYMBOL(irq_stat);
 
 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
 
-static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
+static DEFINE_PER_CPU(struct task_struct *, ksoftirqd) = NULL;
 
 char *softirq_to_name[NR_SOFTIRQS] = {
         "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
@@ -177,6 +185,7 @@ void local_bh_enable_ip(unsigned long ip)
 }
 EXPORT_SYMBOL(local_bh_enable_ip);
 
+
 /*
  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
  * and we fall back to softirqd after that.
@@ -187,34 +196,30 @@ EXPORT_SYMBOL(local_bh_enable_ip);
  * should not be able to lock up the box.
  */
 #define MAX_SOFTIRQ_RESTART 10
-
-asmlinkage void __do_softirq(void)
+static void ____do_softirq(void)
 {
-        struct softirq_action *h;
         __u32 pending;
-        int max_restart = MAX_SOFTIRQ_RESTART;
+        
+        struct softirq_action *h;
         int cpu;
-
+        
         pending = local_softirq_pending();
+        
         account_system_vtime(current);
-
-        __local_bh_disable((unsigned long)__builtin_return_address(0));
-        lockdep_softirq_enter();
-
+        
         cpu = smp_processor_id();
-restart:
-        /* Reset the pending bitmask before enabling irqs */
-        set_softirq_pending(0);
 
+        set_softirq_pending(0);
+        
         local_irq_enable();
-
+        
         h = softirq_vec;
-
+        
         do {
                 if (pending & 1) {
                         int prev_count = preempt_count();
                         kstat_incr_softirqs_this_cpu(h - softirq_vec);
-
+                        
                         trace_softirq_entry(h, softirq_vec);
                         h->action(h);
                         trace_softirq_exit(h, softirq_vec);
@@ -226,26 +231,70 @@ restart:
                                        h->action, prev_count, preempt_count());
                                 preempt_count() = prev_count;
                         }
-
+                        
                         rcu_bh_qs(cpu);
                 }
                 h++;
                 pending >>= 1;
         } while (pending);
-
+        
         local_irq_disable();
+}
+
+static void ___do_softirq(void)
+{
+        __u32 pending;
+
+        //struct softirq_action *h;
+        int max_restart = MAX_SOFTIRQ_RESTART;
+        //int cpu;
+
+        pending = local_softirq_pending();
+
+restart:
+        ____do_softirq();
 
         pending = local_softirq_pending();
         if (pending && --max_restart)
                 goto restart;
 
         if (pending)
+        {
                 wakeup_softirqd();
+        }
+}
 
+asmlinkage void __do_softirq(void)
+{
+#ifdef LITMUS_THREAD_ALL_SOFTIRQ
+        /* Skip straight to wakeup_softirqd() if we're using 
+         LITMUS_THREAD_ALL_SOFTIRQ (unless there's really high prio-stuff waiting.). */
+        struct task_struct *tsk = __get_cpu_var(ksoftirqd);
+        
+        if(tsk)
+        {
+                __u32 pending = local_softirq_pending();
+                const __u32 high_prio_softirq = (1<<HI_SOFTIRQ) | (1<<TIMER_SOFTIRQ) | (1<<HRTIMER_SOFTIRQ);
+                if(pending && !(pending & high_prio_softirq))
+                {
+                        wakeup_softirqd();
+                        return;
+                }
+        }
+#endif
+        
+        /*
+         * 'immediate' softirq execution:
+         */
+        __local_bh_disable((unsigned long)__builtin_return_address(0));
+        lockdep_softirq_enter();
+        
+        ___do_softirq();
+        
         lockdep_softirq_exit();
-
+        
         account_system_vtime(current);
-        _local_bh_enable();
+        _local_bh_enable();     
 }
 
 #ifndef __ARCH_HAS_DO_SOFTIRQ
@@ -357,8 +406,64 @@ struct tasklet_head
 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 
+
 void __tasklet_schedule(struct tasklet_struct *t)
 {
+#ifdef CONFIG_LITMUS_NVIDIA
+        if(is_nvidia_func(t->func))
+        {
+                u32 nvidia_device = get_tasklet_nv_device_num(t);       
+                //              TRACE("%s: Handling NVIDIA tasklet for device\t%u\tat\t%llu\n",
+                //                        __FUNCTION__, nvidia_device,litmus_clock());
+
+                unsigned long flags;
+                struct task_struct* device_owner;
+
+                lock_nv_registry(nvidia_device, &flags);
+
+                device_owner = get_nv_device_owner(nvidia_device);
+
+                if(device_owner==NULL)
+                {
+                        t->owner = NULL;
+                }
+                else
+                {
+                        if(is_realtime(device_owner))
+                        {
+                                TRACE("%s: Handling NVIDIA tasklet for device %u at %llu\n",
+                                          __FUNCTION__, nvidia_device,litmus_clock());                          
+                                TRACE("%s: the owner task %d of NVIDIA Device %u is RT-task\n",
+                                          __FUNCTION__,device_owner->pid,nvidia_device);
+
+                                t->owner = device_owner;
+                                sched_trace_tasklet_release(t->owner);
+                                if(likely(_litmus_tasklet_schedule(t,nvidia_device)))
+                                {
+                                        unlock_nv_registry(nvidia_device, &flags);
+                                        return;
+                                }
+                                else
+                                {
+                                        t->owner = NULL; /* fall through to normal scheduling */
+                                }
+                        }
+                        else
+                        {
+                                t->owner = NULL;
+                        }
+                }
+                unlock_nv_registry(nvidia_device, &flags);
+        }
+#endif
+
+        ___tasklet_schedule(t);
+}
+EXPORT_SYMBOL(__tasklet_schedule);
+
+
+void ___tasklet_schedule(struct tasklet_struct *t)
+{
         unsigned long flags;
 
         local_irq_save(flags);
@@ -368,11 +473,65 @@ void __tasklet_schedule(struct tasklet_struct *t)
         raise_softirq_irqoff(TASKLET_SOFTIRQ);
         local_irq_restore(flags);
 }
+EXPORT_SYMBOL(___tasklet_schedule);
 
-EXPORT_SYMBOL(__tasklet_schedule);
 
 void __tasklet_hi_schedule(struct tasklet_struct *t)
 {
+#ifdef CONFIG_LITMUS_NVIDIA
+        if(is_nvidia_func(t->func))
+        {       
+                u32 nvidia_device = get_tasklet_nv_device_num(t);
+                //              TRACE("%s: Handling NVIDIA tasklet for device\t%u\tat\t%llu\n",
+                //                        __FUNCTION__, nvidia_device,litmus_clock());
+
+                unsigned long flags;
+                struct task_struct* device_owner;
+                
+                lock_nv_registry(nvidia_device, &flags);
+                
+                device_owner = get_nv_device_owner(nvidia_device);
+
+                if(device_owner==NULL) 
+                {
+                        t->owner = NULL;
+                }
+                else
+                {
+                        if( is_realtime(device_owner))
+                        {
+                                TRACE("%s: Handling NVIDIA tasklet for device %u\tat %llu\n",
+                                          __FUNCTION__, nvidia_device,litmus_clock());                          
+                                TRACE("%s: the owner task %d of NVIDIA Device %u is RT-task\n",
+                                          __FUNCTION__,device_owner->pid,nvidia_device);
+                                
+                                t->owner = device_owner;
+                                sched_trace_tasklet_release(t->owner);
+                                if(likely(_litmus_tasklet_hi_schedule(t,nvidia_device)))
+                                {
+                                        unlock_nv_registry(nvidia_device, &flags);
+                                        return;
+                                }
+                                else
+                                {
+                                        t->owner = NULL; /* fall through to normal scheduling */
+                                }
+                        }
+                        else
+                        {
+                                t->owner = NULL;
+                        }
+                }
+                unlock_nv_registry(nvidia_device, &flags);
+        }
+#endif
+
+        ___tasklet_hi_schedule(t);
+}
+EXPORT_SYMBOL(__tasklet_hi_schedule);
+
+void ___tasklet_hi_schedule(struct tasklet_struct* t)
+{
         unsigned long flags;
 
         local_irq_save(flags);
@@ -382,19 +541,72 @@ void __tasklet_hi_schedule(struct tasklet_struct *t)
         raise_softirq_irqoff(HI_SOFTIRQ);
         local_irq_restore(flags);
 }
-
-EXPORT_SYMBOL(__tasklet_hi_schedule);
+EXPORT_SYMBOL(___tasklet_hi_schedule);
 
 void __tasklet_hi_schedule_first(struct tasklet_struct *t)
 {
         BUG_ON(!irqs_disabled());
+#ifdef CONFIG_LITMUS_NVIDIA     
+        if(is_nvidia_func(t->func))
+        {       
+                u32 nvidia_device = get_tasklet_nv_device_num(t);
+                //              TRACE("%s: Handling NVIDIA tasklet for device\t%u\tat\t%llu\n",
+                //                        __FUNCTION__, nvidia_device,litmus_clock());
+                unsigned long flags;
+                struct task_struct* device_owner;
+                
+                lock_nv_registry(nvidia_device, &flags);
+
+                device_owner = get_nv_device_owner(nvidia_device);
+
+                if(device_owner==NULL)
+                {
+                        t->owner = NULL;
+                }
+                else
+                {
+                        if(is_realtime(device_owner))
+                        {
+                                TRACE("%s: Handling NVIDIA tasklet for device %u at %llu\n",
+                                          __FUNCTION__, nvidia_device,litmus_clock());
+                                
+                                TRACE("%s: the owner task %d of NVIDIA Device %u is RT-task\n",
+                                          __FUNCTION__,device_owner->pid,nvidia_device);
+                                
+                                t->owner = device_owner;
+                                sched_trace_tasklet_release(t->owner);
+                                if(likely(_litmus_tasklet_hi_schedule_first(t,nvidia_device)))
+                                {
+                                        unlock_nv_registry(nvidia_device, &flags);
+                                        return;
+                                }
+                                else
+                                {
+                                        t->owner = NULL; /* fall through to normal scheduling */
+                                }
+                        }
+                        else
+                        {
+                                t->owner = NULL;
+                        }
+                }
+                unlock_nv_registry(nvidia_device, &flags);
+        }
+#endif
+
+        ___tasklet_hi_schedule_first(t);
+}
+EXPORT_SYMBOL(__tasklet_hi_schedule_first);
+
+void ___tasklet_hi_schedule_first(struct tasklet_struct* t)
+{
+        BUG_ON(!irqs_disabled());
 
         t->next = __get_cpu_var(tasklet_hi_vec).head;
         __get_cpu_var(tasklet_hi_vec).head = t;
         __raise_softirq_irqoff(HI_SOFTIRQ);
 }
-
-EXPORT_SYMBOL(__tasklet_hi_schedule_first);
+EXPORT_SYMBOL(___tasklet_hi_schedule_first);
 
 static void tasklet_action(struct softirq_action *a)
 {
@@ -450,6 +662,7 @@ static void tasklet_hi_action(struct softirq_action *a)
                         if (!atomic_read(&t->count)) {
                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
                                         BUG();
+
                                 t->func(t->data);
                                 tasklet_unlock(t);
                                 continue;
@@ -473,8 +686,13 @@ void tasklet_init(struct tasklet_struct *t,
         t->next = NULL;
         t->state = 0;
         atomic_set(&t->count, 0);
+
         t->func = func;
         t->data = data;
+
+#ifdef CONFIG_LITMUS_SOFTIRQD
+        t->owner = NULL;
+#endif
 }
 
 EXPORT_SYMBOL(tasklet_init);
@@ -489,6 +707,7 @@ void tasklet_kill(struct tasklet_struct *t)
                         yield();
                 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
         }
+
         tasklet_unlock_wait(t);
         clear_bit(TASKLET_STATE_SCHED, &t->state);
 }
@@ -694,6 +913,8 @@ void __init softirq_init(void)
 
 static int run_ksoftirqd(void * __bind_cpu)
 {
+        unsigned long flags;
+        
         set_current_state(TASK_INTERRUPTIBLE);
 
         while (!kthread_should_stop()) {
@@ -712,7 +933,11 @@ static int run_ksoftirqd(void * __bind_cpu)
                            don't process */
                         if (cpu_is_offline((long)__bind_cpu))
                                 goto wait_to_die;
-                        do_softirq();
+                        
+                        local_irq_save(flags);
+                        ____do_softirq();
+                        local_irq_restore(flags);
+                        
                         preempt_enable_no_resched();
                         cond_resched();
                         preempt_disable();
@@ -760,6 +985,7 @@ void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
                 if (*i == t) {
                         *i = t->next;
+
                         /* If this was the tail element, move the tail ptr */
                         if (*i == NULL)
                                 per_cpu(tasklet_vec, cpu).tail = i;