1 files changed, 432 insertions, 2 deletions

diff --git a/litmus/sched_gsn_edf.c b/litmus/sched_gsn_edf.c
index ac7685fe69f0..b40ff7ba4f0e 100644
--- a/litmus/sched_gsn_edf.c
+++ b/litmus/sched_gsn_edf.c
@@ -35,6 +35,10 @@
 #include <litmus/litmus_softirq.h>
 #endif
 
+#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
+#include <linux/interrupt.h>
+#endif
+
 #ifdef CONFIG_LITMUS_NVIDIA
 #include <litmus/nvidia_info.h>
 #endif
@@ -126,6 +130,16 @@ static struct bheap      gsnedf_cpu_heap;
 static rt_domain_t gsnedf;
 #define gsnedf_lock (gsnedf.ready_lock)
 
+#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
+struct tasklet_head
+{
+        struct tasklet_struct *head;
+        struct tasklet_struct **tail;
+};
+
+struct tasklet_head gsnedf_pending_tasklets;
+#endif
+
 
 /* Uncomment this if you want to see all scheduling decisions in the
  * TRACE() log.
@@ -393,6 +407,410 @@ static void gsnedf_tick(struct task_struct* t)
         }
 }
 
+
+
+
+
+
+
+
+
+
+
+
+#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
+
+
+static void __do_lit_tasklet(struct tasklet_struct* tasklet, unsigned long flushed)
+{
+        if (!atomic_read(&tasklet->count)) {
+                sched_trace_tasklet_begin(tasklet->owner);
+                
+                if (!test_and_clear_bit(TASKLET_STATE_SCHED, &tasklet->state))
+                {
+                        BUG();
+                }
+                TRACE("%s: Invoking tasklet with owner pid = %d (flushed = %d).\n", __FUNCTION__, tasklet->owner->pid, flushed);
+                tasklet->func(tasklet->data);
+                tasklet_unlock(tasklet);
+                
+                sched_trace_tasklet_end(tasklet->owner, flushed);
+        }
+        else {
+                BUG();
+        }
+}
+
+
+static void __extract_tasklets(struct task_struct* task, struct tasklet_head* task_tasklets)
+{
+        struct tasklet_struct* step;
+        struct tasklet_struct* tasklet;
+        struct tasklet_struct* prev;
+        
+        task_tasklets->head = NULL;
+        task_tasklets->tail = &(task_tasklets->head);
+        
+        prev = NULL;
+        for(step = gsnedf_pending_tasklets.head; step != NULL; step = step->next)
+        {
+                if(step->owner == task)
+                {
+                        TRACE("%s: Found tasklet to flush: %d\n", __FUNCTION__, step->owner->pid);
+                        
+                        tasklet = step;
+                        
+                        if(prev) {
+                                prev->next = tasklet->next;
+                        }
+                        else if(gsnedf_pending_tasklets.head == tasklet) {
+                                // we're at the head.
+                                gsnedf_pending_tasklets.head = tasklet->next;
+                        }
+                        
+                        if(gsnedf_pending_tasklets.tail == &tasklet) {
+                                // we're at the tail
+                                if(prev) {
+                                        gsnedf_pending_tasklets.tail = &prev;
+                                }
+                                else {
+                                        gsnedf_pending_tasklets.tail = &(gsnedf_pending_tasklets.head);
+                                }
+                        }
+                        
+                        tasklet->next = NULL;
+                        *(task_tasklets->tail) = tasklet;
+                        task_tasklets->tail = &(tasklet->next);
+                }
+                else {
+                        prev = step;
+                }
+        }
+}
+
+static void flush_tasklets(struct task_struct* task)
+{
+        unsigned long flags;
+        struct tasklet_head task_tasklets;
+        struct tasklet_struct* step;
+        
+        raw_spin_lock_irqsave(&gsnedf_lock, flags);
+        __extract_tasklets(task, &task_tasklets);
+        raw_spin_unlock_irqrestore(&gsnedf_lock, flags);
+        
+        if(gsnedf_pending_tasklets.head != NULL) {
+                TRACE("%s: Flushing tasklets for %d...\n", __FUNCTION__, task->pid);
+        }
+        
+        // now execute any flushed tasklets.
+        for(step = gsnedf_pending_tasklets.head; step != NULL; /**/)
+        {
+                struct tasklet_struct* temp = step->next;
+                
+                step->next = NULL;
+                __do_lit_tasklet(step, 1ul);
+                
+                step = temp;
+        }
+}
+
+
+static void do_lit_tasklets(struct task_struct* sched_task)
+{
+        int work_to_do = 1;
+        struct tasklet_struct *tasklet = NULL;
+        //struct tasklet_struct *step;
+        unsigned long flags;
+        
+        while(work_to_do) {
+                // remove tasklet at head of list if it has higher priority.
+                raw_spin_lock_irqsave(&gsnedf_lock, flags);     
+                
+                /*
+                 step = gsnedf_pending_tasklets.head;
+                 TRACE("%s: (BEFORE) dumping tasklet queue...\n", __FUNCTION__);
+                 while(step != NULL){
+                 TRACE("%s: %p (%d)\n", __FUNCTION__, step, step->owner->pid);
+                 step = step->next;
+                 }
+                 TRACE("%s: tail = %p (%d)\n", __FUNCTION__, *(gsnedf_pending_tasklets.tail), (*(gsnedf_pending_tasklets.tail) != NULL) ? (*(gsnedf_pending_tasklets.tail))->owner->pid : -1);
+                 TRACE("%s: done.\n", __FUNCTION__);
+                 */
+                
+                
+                if(gsnedf_pending_tasklets.head != NULL) {
+                        // remove tasklet at head.
+                        tasklet = gsnedf_pending_tasklets.head;
+                        
+                        if(edf_higher_prio(tasklet->owner, sched_task)) {
+                                
+                                if(NULL == tasklet->next) {
+                                        // tasklet is at the head, list only has one element
+                                        TRACE("%s: Tasklet for %d is the last element in tasklet queue.\n", __FUNCTION__, tasklet->owner->pid);
+                                        gsnedf_pending_tasklets.tail = &(gsnedf_pending_tasklets.head);
+                                }
+                                
+                                // remove the tasklet from the queue
+                                gsnedf_pending_tasklets.head = tasklet->next;
+                                
+                                TRACE("%s: Removed tasklet for %d from tasklet queue.\n", __FUNCTION__, tasklet->owner->pid);
+                        }
+                        else {
+                                TRACE("%s: Pending tasklet (%d) does not have priority to run on this CPU (%d).\n", __FUNCTION__, tasklet->owner->pid, smp_processor_id());
+                                tasklet = NULL;
+                        }
+                }
+                else {
+                        TRACE("%s: Tasklet queue is empty.\n", __FUNCTION__);
+                }
+                
+                
+                /*
+                 step = gsnedf_pending_tasklets.head;
+                 TRACE("%s: (AFTER) dumping tasklet queue...\n", __FUNCTION__);
+                 while(step != NULL){
+                 TRACE("%s: %p (%d)\n", __FUNCTION__, step, step->owner->pid);
+                 step = step->next;
+                 }
+                 TRACE("%s: tail = %p (%d)\n", __FUNCTION__, *(gsnedf_pending_tasklets.tail), (*(gsnedf_pending_tasklets.tail) != NULL) ? (*(gsnedf_pending_tasklets.tail))->owner->pid : -1);
+                 TRACE("%s: done.\n", __FUNCTION__);
+                 */
+                
+                
+                raw_spin_unlock_irqrestore(&gsnedf_lock, flags);
+                
+                if(tasklet) {
+                        __do_lit_tasklet(tasklet, 0ul);
+                        tasklet = NULL; 
+                }
+                else {
+                        work_to_do = 0;
+                }
+        }
+        
+        //TRACE("%s: exited.\n", __FUNCTION__);
+}
+
+
+static void run_tasklets(struct task_struct* sched_task)
+{
+#if 0
+        int task_is_rt = is_realtime(sched_task);
+        cedf_domain_t* cluster;
+        
+        if(is_realtime(sched_task)) {
+                cluster = task_cpu_cluster(sched_task);
+        }
+        else {
+                cluster = remote_cluster(get_cpu());
+        }
+        
+        if(cluster && gsnedf_pending_tasklets.head != NULL) {
+                TRACE("%s: There are tasklets to process.\n", __FUNCTION__);
+                
+                do_lit_tasklets(cluster, sched_task);
+        }
+        
+        if(!task_is_rt) {
+                put_cpu_no_resched();
+        }
+#else
+        
+        preempt_disable();
+        
+        if(gsnedf_pending_tasklets.head != NULL) {
+                TRACE("%s: There are tasklets to process.\n", __FUNCTION__);
+                do_lit_tasklets(sched_task);
+        }
+        
+        preempt_enable_no_resched();
+        
+#endif
+}
+
+
+static void __add_pai_tasklet(struct tasklet_struct* tasklet)
+{
+        struct tasklet_struct* step;
+        
+        /*
+         step = gsnedf_pending_tasklets.head;
+         TRACE("%s: (BEFORE) dumping tasklet queue...\n", __FUNCTION__);
+         while(step != NULL){
+         TRACE("%s: %p (%d)\n", __FUNCTION__, step, step->owner->pid);
+         step = step->next;
+         }
+         TRACE("%s: tail = %p (%d)\n", __FUNCTION__, *(gsnedf_pending_tasklets.tail), (*(gsnedf_pending_tasklets.tail) != NULL) ? (*(gsnedf_pending_tasklets.tail))->owner->pid : -1);
+         TRACE("%s: done.\n", __FUNCTION__);
+         */
+        
+        
+        tasklet->next = NULL;  // make sure there are no old values floating around
+        
+        step = gsnedf_pending_tasklets.head;
+        if(step == NULL) {
+                TRACE("%s: tasklet queue empty.  inserting tasklet for %d at head.\n", __FUNCTION__, tasklet->owner->pid);
+                // insert at tail.
+                *(gsnedf_pending_tasklets.tail) = tasklet;
+                gsnedf_pending_tasklets.tail = &(tasklet->next);                
+        }
+        else if((*(gsnedf_pending_tasklets.tail) != NULL) &&
+                        edf_higher_prio((*(gsnedf_pending_tasklets.tail))->owner, tasklet->owner)) {
+                // insert at tail.
+                TRACE("%s: tasklet belongs at end.  inserting tasklet for %d at tail.\n", __FUNCTION__, tasklet->owner->pid);
+                
+                *(gsnedf_pending_tasklets.tail) = tasklet;
+                gsnedf_pending_tasklets.tail = &(tasklet->next);
+        }
+        else {
+                
+        //WARN_ON(1 == 1);
+                
+                // insert the tasklet somewhere in the middle.
+                
+        TRACE("%s: tasklet belongs somewhere in the middle.\n", __FUNCTION__);
+                
+                while(step->next && edf_higher_prio(step->next->owner, tasklet->owner)) {
+                        step = step->next;
+                }
+                
+                // insert tasklet right before step->next.
+                
+                TRACE("%s: inserting tasklet for %d between %d and %d.\n", __FUNCTION__, tasklet->owner->pid, step->owner->pid, (step->next) ? step->next->owner->pid : -1);
+                
+                tasklet->next = step->next;
+                step->next = tasklet;
+                
+                // patch up the head if needed.
+                if(gsnedf_pending_tasklets.head == step)
+                {
+                        TRACE("%s: %d is the new tasklet queue head.\n", __FUNCTION__, tasklet->owner->pid);
+                        gsnedf_pending_tasklets.head = tasklet;
+                }
+        }
+        
+        /*
+         step = gsnedf_pending_tasklets.head;
+         TRACE("%s: (AFTER) dumping tasklet queue...\n", __FUNCTION__);
+         while(step != NULL){
+         TRACE("%s: %p (%d)\n", __FUNCTION__, step, step->owner->pid);
+         step = step->next;
+         }
+         TRACE("%s: tail = %p (%d)\n", __FUNCTION__, *(gsnedf_pending_tasklets.tail), (*(gsnedf_pending_tasklets.tail) != NULL) ? (*(gsnedf_pending_tasklets.tail))->owner->pid : -1);
+         TRACE("%s: done.\n", __FUNCTION__);
+         */
+        
+        // TODO: Maintain this list in priority order.
+        //      tasklet->next = NULL;
+        //      *(gsnedf_pending_tasklets.tail) = tasklet;
+        //      gsnedf_pending_tasklets.tail = &tasklet->next;
+}
+
+static int enqueue_pai_tasklet(struct tasklet_struct* tasklet)
+{
+        cpu_entry_t *targetCPU = NULL;
+        int thisCPU;
+        int runLocal = 0;
+        int runNow = 0;
+        unsigned long flags;
+        
+    if(unlikely((tasklet->owner == NULL) || !is_realtime(tasklet->owner)))
+    {
+        TRACE("%s: No owner associated with this tasklet!\n", __FUNCTION__);
+                return 0;
+    }   
+
+        
+        raw_spin_lock_irqsave(&gsnedf_lock, flags);             
+        
+        thisCPU = smp_processor_id();
+        
+#if 1
+#ifdef CONFIG_SCHED_CPU_AFFINITY
+        {
+                cpu_entry_t* affinity = NULL;
+                
+                // use this CPU if it is in our cluster and isn't running any RT work.
+                if(
+#ifdef CONFIG_RELEASE_MASTER
+                   (thisCPU != gsnedf.release_master) &&
+#endif
+                   (__get_cpu_var(gsnedf_cpu_entries).linked == NULL)) {
+                        affinity = &(__get_cpu_var(gsnedf_cpu_entries));
+                }
+                else {
+                        // this CPU is busy or shouldn't run tasklet in this cluster.
+                        // look for available near by CPUs.
+                        // NOTE: Affinity towards owner and not this CPU.  Is this right?               
+                        affinity = 
+                                gsnedf_get_nearest_available_cpu(
+                                        &per_cpu(gsnedf_cpu_entries, task_cpu(tasklet->owner)));
+                }
+                
+                targetCPU = affinity;
+        }
+#endif
+#endif
+        
+        if (targetCPU == NULL) {
+                targetCPU = lowest_prio_cpu();
+        }
+        
+        if (edf_higher_prio(tasklet->owner, targetCPU->linked)) {
+                if (thisCPU == targetCPU->cpu) {
+                        TRACE("%s: Run tasklet locally (and now).\n", __FUNCTION__);
+                        runLocal = 1;
+                        runNow = 1;
+                }
+                else {
+                        TRACE("%s: Run tasklet remotely (and now).\n", __FUNCTION__);
+                        runLocal = 0;
+                        runNow = 1;
+                }
+        }
+        else {
+                runLocal = 0;
+                runNow = 0;
+        }
+        
+        if(!runLocal) {
+                // enqueue the tasklet
+                __add_pai_tasklet(tasklet);
+        }
+        
+        raw_spin_unlock_irqrestore(&gsnedf_lock, flags);
+        
+        
+        if (runLocal /*&& runNow */) {  // runNow == 1 is implied
+                TRACE("%s: Running tasklet on CPU where it was received.\n", __FUNCTION__);
+                __do_lit_tasklet(tasklet, 0ul);
+        }
+        else if (runNow /*&& !runLocal */) {  // runLocal == 0 is implied
+                TRACE("%s: Triggering CPU %d to run tasklet.\n", __FUNCTION__, targetCPU->cpu);
+                preempt(targetCPU);  // need to be protected by cedf_lock?
+        }
+        else {
+                TRACE("%s: Scheduling of tasklet was deferred.\n", __FUNCTION__);
+        }
+        
+        return(1); // success
+}
+
+
+#endif
+
+
+
+
+
+
+
+
+
+
+
+
+
 /* Getting schedule() right is a bit tricky. schedule() may not make any
  * assumptions on the state of the current task since it may be called for a
  * number of reasons. The reasons include a scheduler_tick() determined that it
@@ -592,7 +1010,7 @@ static void gsnedf_task_new(struct task_struct * t, int on_rq, int running)
 static void gsnedf_task_wake_up(struct task_struct *task)
 {
         unsigned long flags;
-        lt_t now;       
+        //lt_t now;     
         
         TRACE_TASK(task, "wake_up at %llu\n", litmus_clock());
 
@@ -660,6 +1078,10 @@ static void gsnedf_task_exit(struct task_struct * t)
         }
         raw_spin_unlock_irqrestore(&gsnedf_lock, flags);
 
+#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
+        flush_tasklets(t);
+#endif  
+        
         BUG_ON(!is_realtime(t));
         TRACE_TASK(t, "RIP\n");
 }
@@ -1602,6 +2024,11 @@ static long gsnedf_activate_plugin(void)
                 }
 #endif
         }
+        
+#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
+        gsnedf_pending_tasklets.head = NULL;
+        gsnedf_pending_tasklets.tail = &(gsnedf_pending_tasklets.head);
+#endif  
     
 #ifdef CONFIG_LITMUS_SOFTIRQD
     spawn_klitirqd(NULL);
@@ -1636,7 +2063,10 @@ static struct sched_plugin gsn_edf_plugin __cacheline_aligned_in_smp = {
         .set_prio_inh_klitirqd = set_priority_inheritance_klitirqd,
         .clear_prio_inh_klitirqd = clear_priority_inheritance_klitirqd,
 #endif
-
+#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
+        .enqueue_pai_tasklet = enqueue_pai_tasklet,
+        .run_tasklets = run_tasklets,
+#endif
 };