1 files changed, 279 insertions, 56 deletions

diff --git a/kernel/softirq.c b/kernel/softirq.c
index 2f2df08df395..ea438a8635d0 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -29,6 +29,15 @@
 #include <trace/events/irq.h>
 
 #include <asm/irq.h>
+
+#include <litmus/litmus.h>
+#include <litmus/sched_trace.h>
+
+#ifdef CONFIG_LITMUS_NVIDIA
+#include <litmus/nvidia_info.h>
+#include <litmus/trace.h>
+#endif
+
 /*
    - No shared variables, all the data are CPU local.
    - If a softirq needs serialization, let it serialize itself
@@ -67,7 +76,7 @@ char *softirq_to_name[NR_SOFTIRQS] = {
  * to the pending events, so lets the scheduler to balance
  * the softirq load for us.
  */
-static void wakeup_softirqd(void)
+void wakeup_softirqd(void)
 {
         /* Interrupts are disabled: no need to stop preemption */
         struct task_struct *tsk = __this_cpu_read(ksoftirqd);
@@ -193,6 +202,7 @@ void local_bh_enable_ip(unsigned long ip)
 }
 EXPORT_SYMBOL(local_bh_enable_ip);
 
+
 /*
  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
  * and we fall back to softirqd after that.
@@ -206,10 +216,10 @@ EXPORT_SYMBOL(local_bh_enable_ip);
 
 asmlinkage void __do_softirq(void)
 {
-        struct softirq_action *h;
-        __u32 pending;
-        int max_restart = MAX_SOFTIRQ_RESTART;
-        int cpu;
+    struct softirq_action *h; 
+    __u32 pending;
+    int max_restart = MAX_SOFTIRQ_RESTART;
+    int cpu;
 
         /* Mark Feather-Trace samples as "disturbed". */
         ft_irq_fired();
@@ -217,57 +227,57 @@ asmlinkage void __do_softirq(void)
         pending = local_softirq_pending();
         account_system_vtime(current);
 
-        __local_bh_disable((unsigned long)__builtin_return_address(0),
-                                SOFTIRQ_OFFSET);
-        lockdep_softirq_enter();
+    __local_bh_disable((unsigned long)__builtin_return_address(0),
+                SOFTIRQ_OFFSET);
+    lockdep_softirq_enter();
 
-        cpu = smp_processor_id();
+    cpu = smp_processor_id();
 restart:
-        /* Reset the pending bitmask before enabling irqs */
-        set_softirq_pending(0);
+    /* Reset the pending bitmask before enabling irqs */
+    set_softirq_pending(0);
 
-        local_irq_enable();
+    local_irq_enable();
 
-        h = softirq_vec;
-
-        do {
-                if (pending & 1) {
-                        unsigned int vec_nr = h - softirq_vec;
-                        int prev_count = preempt_count();
-
-                        kstat_incr_softirqs_this_cpu(vec_nr);
-
-                        trace_softirq_entry(vec_nr);
-                        h->action(h);
-                        trace_softirq_exit(vec_nr);
-                        if (unlikely(prev_count != preempt_count())) {
-                                printk(KERN_ERR "huh, entered softirq %u %s %p"
-                                       "with preempt_count %08x,"
-                                       " exited with %08x?\n", vec_nr,
-                                       softirq_to_name[vec_nr], h->action,
-                                       prev_count, preempt_count());
-                                preempt_count() = prev_count;
-                        }
+    h = softirq_vec;
 
-                        rcu_bh_qs(cpu);
-                }
-                h++;
-                pending >>= 1;
-        } while (pending);
+    do {
+        if (pending & 1) {
+            unsigned int vec_nr = h - softirq_vec;
+            int prev_count = preempt_count();
 
-        local_irq_disable();
+            kstat_incr_softirqs_this_cpu(vec_nr);
 
-        pending = local_softirq_pending();
-        if (pending && --max_restart)
-                goto restart;
+            trace_softirq_entry(vec_nr);
+            h->action(h);
+            trace_softirq_exit(vec_nr);
+            if (unlikely(prev_count != preempt_count())) {
+                printk(KERN_ERR "huh, entered softirq %u %s %p"
+                       "with preempt_count %08x,"
+                       " exited with %08x?\n", vec_nr,
+                       softirq_to_name[vec_nr], h->action,
+                       prev_count, preempt_count());
+                preempt_count() = prev_count;
+            }   
 
-        if (pending)
-                wakeup_softirqd();
+            rcu_bh_qs(cpu);
+        }   
+        h++;
+        pending >>= 1;
+    } while (pending);
 
-        lockdep_softirq_exit();
+    local_irq_disable();
 
-        account_system_vtime(current);
-        __local_bh_enable(SOFTIRQ_OFFSET);
+    pending = local_softirq_pending();
+    if (pending && --max_restart)
+        goto restart;
+
+    if (pending)
+        wakeup_softirqd();
+
+    lockdep_softirq_exit();
+
+    account_system_vtime(current);
+    __local_bh_enable(SOFTIRQ_OFFSET);
 }
 
 #ifndef __ARCH_HAS_DO_SOFTIRQ
@@ -396,17 +406,103 @@ void open_softirq(int nr, void (*action)(struct softirq_action *))
 /*
  * Tasklets
  */
-struct tasklet_head
-{
-        struct tasklet_struct *head;
-        struct tasklet_struct **tail;
-};
 
 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 
+#ifdef CONFIG_LITMUS_NVIDIA
+static int __do_nv_now(struct tasklet_struct* tasklet)
+{
+        int success = 1;
+
+        if(tasklet_trylock(tasklet)) {
+                if (!atomic_read(&tasklet->count)) {
+                        if (!test_and_clear_bit(TASKLET_STATE_SCHED, &tasklet->state)) {    
+                                BUG();
+                        }
+                        tasklet->func(tasklet->data);
+                        tasklet_unlock(tasklet);
+                }
+                else {
+                        success = 0;
+                }
+
+                tasklet_unlock(tasklet);
+        }
+        else {
+                success = 0;
+        }
+
+        return success;
+}
+#endif
+
+
 void __tasklet_schedule(struct tasklet_struct *t)
 {
+#ifdef CONFIG_LITMUS_NVIDIA
+        if(is_nvidia_func(t->func))
+        {
+#if 1
+                // do nvidia tasklets right away and return
+                if(__do_nv_now(t))
+                        return;
+#else
+                u32 nvidia_device = get_tasklet_nv_device_num(t);       
+                //              TRACE("%s: Handling NVIDIA tasklet for device\t%u\tat\t%llu\n",
+                //                        __FUNCTION__, nvidia_device,litmus_clock());
+
+                unsigned long flags;
+                struct task_struct* device_owner;
+
+                lock_nv_registry(nvidia_device, &flags);
+
+                device_owner = get_nv_max_device_owner(nvidia_device);
+
+                if(device_owner==NULL)
+                {
+                        t->owner = NULL;
+                }
+                else
+                {
+                        if(is_realtime(device_owner))
+                        {
+                                TRACE("%s: Handling NVIDIA tasklet for device %u at %llu\n",
+                                          __FUNCTION__, nvidia_device,litmus_clock());                          
+                                TRACE("%s: the owner task %d of NVIDIA Device %u is RT-task\n",
+                                          __FUNCTION__,device_owner->pid,nvidia_device);
+
+                                t->owner = device_owner;
+                                sched_trace_tasklet_release(t->owner);
+
+                                if(likely(_litmus_tasklet_schedule(t,nvidia_device)))
+                                {
+                                        unlock_nv_registry(nvidia_device, &flags);
+                                        return;
+                                }
+                                else
+                                {
+                                        t->owner = NULL; /* fall through to normal scheduling */
+                                }
+                        }
+                        else
+                        {
+                                t->owner = NULL;
+                        }
+                }
+                unlock_nv_registry(nvidia_device, &flags);
+#endif
+        }
+
+#endif
+
+        ___tasklet_schedule(t);
+}
+EXPORT_SYMBOL(__tasklet_schedule);
+
+
+void ___tasklet_schedule(struct tasklet_struct *t)
+{
         unsigned long flags;
 
         local_irq_save(flags);
@@ -416,11 +512,71 @@ void __tasklet_schedule(struct tasklet_struct *t)
         raise_softirq_irqoff(TASKLET_SOFTIRQ);
         local_irq_restore(flags);
 }
+EXPORT_SYMBOL(___tasklet_schedule);
 
-EXPORT_SYMBOL(__tasklet_schedule);
 
 void __tasklet_hi_schedule(struct tasklet_struct *t)
 {
+#ifdef CONFIG_LITMUS_NVIDIA
+        if(is_nvidia_func(t->func))
+        {       
+#if 1
+                // do nvidia tasklets right away and return
+                if(__do_nv_now(t))
+                        return;
+#else
+                u32 nvidia_device = get_tasklet_nv_device_num(t);
+                //              TRACE("%s: Handling NVIDIA tasklet for device\t%u\tat\t%llu\n",
+                //                        __FUNCTION__, nvidia_device,litmus_clock());
+
+                unsigned long flags;
+                struct task_struct* device_owner;
+                
+                lock_nv_registry(nvidia_device, &flags);
+                
+                device_owner = get_nv_max_device_owner(nvidia_device);
+
+                if(device_owner==NULL) 
+                {
+                        t->owner = NULL;
+                }
+                else
+                {
+                        if( is_realtime(device_owner))
+                        {
+                                TRACE("%s: Handling NVIDIA tasklet for device %u\tat %llu\n",
+                                          __FUNCTION__, nvidia_device,litmus_clock());                          
+                                TRACE("%s: the owner task %d of NVIDIA Device %u is RT-task\n",
+                                          __FUNCTION__,device_owner->pid,nvidia_device);
+                                
+                                t->owner = device_owner;
+                                sched_trace_tasklet_release(t->owner);
+                                if(likely(_litmus_tasklet_hi_schedule(t,nvidia_device)))
+                                {
+                                        unlock_nv_registry(nvidia_device, &flags);
+                                        return;
+                                }
+                                else
+                                {
+                                        t->owner = NULL; /* fall through to normal scheduling */
+                                }
+                        }
+                        else
+                        {
+                                t->owner = NULL;
+                        }
+                }
+                unlock_nv_registry(nvidia_device, &flags);
+#endif
+        }
+#endif
+
+        ___tasklet_hi_schedule(t);
+}
+EXPORT_SYMBOL(__tasklet_hi_schedule);
+
+void ___tasklet_hi_schedule(struct tasklet_struct* t)
+{
         unsigned long flags;
 
         local_irq_save(flags);
@@ -430,19 +586,78 @@ void __tasklet_hi_schedule(struct tasklet_struct *t)
         raise_softirq_irqoff(HI_SOFTIRQ);
         local_irq_restore(flags);
 }
-
-EXPORT_SYMBOL(__tasklet_hi_schedule);
+EXPORT_SYMBOL(___tasklet_hi_schedule);
 
 void __tasklet_hi_schedule_first(struct tasklet_struct *t)
 {
         BUG_ON(!irqs_disabled());
+#ifdef CONFIG_LITMUS_NVIDIA     
+        if(is_nvidia_func(t->func))
+        {       
+#if 1
+                // do nvidia tasklets right away and return
+                if(__do_nv_now(t))
+                        return;
+#else
+                u32 nvidia_device = get_tasklet_nv_device_num(t);
+                //              TRACE("%s: Handling NVIDIA tasklet for device\t%u\tat\t%llu\n",
+                //                        __FUNCTION__, nvidia_device,litmus_clock());
+                unsigned long flags;
+                struct task_struct* device_owner;
+                
+                lock_nv_registry(nvidia_device, &flags);
+
+                device_owner = get_nv_max_device_owner(nvidia_device);
+
+                if(device_owner==NULL)
+                {
+                        t->owner = NULL;
+                }
+                else
+                {
+                        if(is_realtime(device_owner))
+                        {
+                                TRACE("%s: Handling NVIDIA tasklet for device %u at %llu\n",
+                                          __FUNCTION__, nvidia_device,litmus_clock());
+                                
+                                TRACE("%s: the owner task %d of NVIDIA Device %u is RT-task\n",
+                                          __FUNCTION__,device_owner->pid,nvidia_device);
+                                
+                                t->owner = device_owner;
+                                sched_trace_tasklet_release(t->owner);
+                                if(likely(_litmus_tasklet_hi_schedule_first(t,nvidia_device)))
+                                {
+                                        unlock_nv_registry(nvidia_device, &flags);
+                                        return;
+                                }
+                                else
+                                {
+                                        t->owner = NULL; /* fall through to normal scheduling */
+                                }
+                        }
+                        else
+                        {
+                                t->owner = NULL;
+                        }
+                }
+                unlock_nv_registry(nvidia_device, &flags);
+#endif
+        }
+#endif
+
+        ___tasklet_hi_schedule_first(t);
+}
+EXPORT_SYMBOL(__tasklet_hi_schedule_first);
+
+void ___tasklet_hi_schedule_first(struct tasklet_struct* t)
+{
+        BUG_ON(!irqs_disabled());
 
         t->next = __this_cpu_read(tasklet_hi_vec.head);
         __this_cpu_write(tasklet_hi_vec.head, t);
         __raise_softirq_irqoff(HI_SOFTIRQ);
 }
-
-EXPORT_SYMBOL(__tasklet_hi_schedule_first);
+EXPORT_SYMBOL(___tasklet_hi_schedule_first);
 
 static void tasklet_action(struct softirq_action *a)
 {
@@ -498,6 +713,7 @@ static void tasklet_hi_action(struct softirq_action *a)
                         if (!atomic_read(&t->count)) {
                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
                                         BUG();
+
                                 t->func(t->data);
                                 tasklet_unlock(t);
                                 continue;
@@ -521,8 +737,13 @@ void tasklet_init(struct tasklet_struct *t,
         t->next = NULL;
         t->state = 0;
         atomic_set(&t->count, 0);
+
         t->func = func;
         t->data = data;
+
+#ifdef CONFIG_LITMUS_SOFTIRQD
+        t->owner = NULL;
+#endif
 }
 
 EXPORT_SYMBOL(tasklet_init);
@@ -537,6 +758,7 @@ void tasklet_kill(struct tasklet_struct *t)
                         yield();
                 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
         }
+
         tasklet_unlock_wait(t);
         clear_bit(TASKLET_STATE_SCHED, &t->state);
 }
@@ -811,6 +1033,7 @@ void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
                 if (*i == t) {
                         *i = t->next;
+
                         /* If this was the tail element, move the tail ptr */
                         if (*i == NULL)
                                 per_cpu(tasklet_vec, cpu).tail = i;