1 files changed, 448 insertions, 2 deletions

diff --git a/litmus/sched_cfifo.c b/litmus/sched_cfifo.c
index f515446f76ed..689b2dbe5fae 100644
--- a/litmus/sched_cfifo.c
+++ b/litmus/sched_cfifo.c
@@ -55,6 +55,10 @@
 #include <litmus/litmus_softirq.h>
 #endif
 
+#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
+#include <linux/interrupt.h>
+#endif
+
 #ifdef CONFIG_LITMUS_NVIDIA
 #include <litmus/nvidia_info.h>
 #endif
@@ -91,6 +95,15 @@ DEFINE_PER_CPU(cpu_entry_t, cfifo_cpu_entries);
 #define test_will_schedule(cpu) \
         (atomic_read(&per_cpu(cfifo_cpu_entries, cpu).will_schedule))
 
+
+#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
+struct tasklet_head
+{
+        struct tasklet_struct *head;
+        struct tasklet_struct **tail;
+};
+#endif
+
 /*
  * In C-FIFO there is a cfifo domain _per_ cluster
  * The number of clusters is dynamically determined accordingly to the
@@ -108,6 +121,12 @@ typedef struct clusterdomain {
         struct bheap      cpu_heap;
         /* lock for this cluster */
 #define cfifo_lock domain.ready_lock
+        
+        
+#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
+        struct tasklet_head pending_tasklets;
+#endif  
+
 } cfifo_domain_t;
 
 /* a cfifo_domain per cluster; allocation is done at init/activation time */
@@ -251,7 +270,7 @@ static void preempt(cpu_entry_t *entry)
         preempt_if_preemptable(entry->scheduled, entry->cpu);
 }
 
-/* requeue - Put an unlinked task into gsn-edf domain.
+/* requeue - Put an unlinked task into c-fifo domain.
  *           Caller must hold cfifo_lock.
  */
 static noinline void requeue(struct task_struct* task)
@@ -395,6 +414,419 @@ static void cfifo_tick(struct task_struct* t)
         }
 }
 
+
+
+
+
+
+
+
+
+
+
+
+#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
+
+
+static void __do_lit_tasklet(struct tasklet_struct* tasklet, unsigned long flushed)
+{
+        if (!atomic_read(&tasklet->count)) {
+                sched_trace_tasklet_begin(tasklet->owner);
+                
+                if (!test_and_clear_bit(TASKLET_STATE_SCHED, &tasklet->state))
+                {
+                        BUG();
+                }
+                TRACE("%s: Invoking tasklet with owner pid = %d (flushed = %d).\n", __FUNCTION__, tasklet->owner->pid, flushed);
+                tasklet->func(tasklet->data);
+                tasklet_unlock(tasklet);
+                
+                sched_trace_tasklet_end(tasklet->owner, flushed);
+        }
+        else {
+                BUG();
+        }
+}
+
+
+static void __extract_tasklets(cfifo_domain_t* cluster, struct task_struct* task, struct tasklet_head* task_tasklets)
+{
+        struct tasklet_struct* step;
+        struct tasklet_struct* tasklet;
+        struct tasklet_struct* prev;
+        
+        task_tasklets->head = NULL;
+        task_tasklets->tail = &(task_tasklets->head);
+        
+        prev = NULL;
+        for(step = cluster->pending_tasklets.head; step != NULL; step = step->next)
+        {
+                if(step->owner == task)
+                {
+                        TRACE("%s: Found tasklet to flush: %d\n", __FUNCTION__, step->owner->pid);
+                        
+                        tasklet = step;
+                        
+                        if(prev) {
+                                prev->next = tasklet->next;
+                        }
+                        else if(cluster->pending_tasklets.head == tasklet) {
+                                // we're at the head.
+                                cluster->pending_tasklets.head = tasklet->next;
+                        }
+                        
+                        if(cluster->pending_tasklets.tail == &tasklet) {
+                                // we're at the tail
+                                if(prev) {
+                                        cluster->pending_tasklets.tail = &prev;
+                                }
+                                else {
+                                        cluster->pending_tasklets.tail = &(cluster->pending_tasklets.head);
+                                }
+                        }
+                        
+                        tasklet->next = NULL;
+                        *(task_tasklets->tail) = tasklet;
+                        task_tasklets->tail = &(tasklet->next);
+                }
+                else {
+                        prev = step;
+                }
+        }
+}
+
+static void flush_tasklets(cfifo_domain_t* cluster, struct task_struct* task)
+{
+        unsigned long flags;
+        struct tasklet_head task_tasklets;
+        struct tasklet_struct* step;
+        
+        raw_spin_lock_irqsave(&cluster->cfifo_lock, flags);
+        __extract_tasklets(cluster, task, &task_tasklets);
+        raw_spin_unlock_irqrestore(&cluster->cfifo_lock, flags);
+        
+        if(cluster->pending_tasklets.head != NULL) {
+                TRACE("%s: Flushing tasklets for %d...\n", __FUNCTION__, task->pid);
+        }
+        
+        // now execute any flushed tasklets.
+        for(step = cluster->pending_tasklets.head; step != NULL; /**/)
+        {
+                struct tasklet_struct* temp = step->next;
+                
+                step->next = NULL;
+                __do_lit_tasklet(step, 1ul);
+                
+                step = temp;
+        }
+}
+
+
+static void do_lit_tasklets(cfifo_domain_t* cluster, struct task_struct* sched_task)
+{
+        int work_to_do = 1;
+        struct tasklet_struct *tasklet = NULL;
+        //struct tasklet_struct *step;
+        unsigned long flags;
+        
+        while(work_to_do) {
+                // remove tasklet at head of list if it has higher priority.
+                raw_spin_lock_irqsave(&cluster->cfifo_lock, flags);     
+                
+                /*
+                step = cluster->pending_tasklets.head;
+                TRACE("%s: (BEFORE) dumping tasklet queue...\n", __FUNCTION__);
+                while(step != NULL){
+                        TRACE("%s: %p (%d)\n", __FUNCTION__, step, step->owner->pid);
+                        step = step->next;
+                }
+                TRACE("%s: tail = %p (%d)\n", __FUNCTION__, *(cluster->pending_tasklets.tail), (*(cluster->pending_tasklets.tail) != NULL) ? (*(cluster->pending_tasklets.tail))->owner->pid : -1);
+                TRACE("%s: done.\n", __FUNCTION__);
+                 */
+                
+                
+                if(cluster->pending_tasklets.head != NULL) {
+                        // remove tasklet at head.
+                        tasklet = cluster->pending_tasklets.head;
+                        
+                        if(fifo_higher_prio(tasklet->owner, sched_task)) {
+                                
+                                if(NULL == tasklet->next) {
+                                        // tasklet is at the head, list only has one element
+                                        TRACE("%s: Tasklet for %d is the last element in tasklet queue.\n", __FUNCTION__, tasklet->owner->pid);
+                                        cluster->pending_tasklets.tail = &(cluster->pending_tasklets.head);
+                                }
+                                
+                                // remove the tasklet from the queue
+                                cluster->pending_tasklets.head = tasklet->next;
+                                
+                                TRACE("%s: Removed tasklet for %d from tasklet queue.\n", __FUNCTION__, tasklet->owner->pid);
+                        }
+                        else {
+                                TRACE("%s: Pending tasklet (%d) does not have priority to run on this CPU (%d).\n", __FUNCTION__, tasklet->owner->pid, smp_processor_id());
+                                tasklet = NULL;
+                        }
+                }
+                else {
+                        TRACE("%s: Tasklet queue is empty.\n", __FUNCTION__);
+                }
+                
+                /*
+                step = cluster->pending_tasklets.head;
+                TRACE("%s: (AFTER) dumping tasklet queue...\n", __FUNCTION__);
+                while(step != NULL){
+                        TRACE("%s: %p (%d)\n", __FUNCTION__, step, step->owner->pid);
+                        step = step->next;
+                }
+                TRACE("%s: tail = %p (%d)\n", __FUNCTION__, *(cluster->pending_tasklets.tail), (*(cluster->pending_tasklets.tail) != NULL) ? (*(cluster->pending_tasklets.tail))->owner->pid : -1);
+                TRACE("%s: done.\n", __FUNCTION__);
+                 */
+                
+                raw_spin_unlock_irqrestore(&cluster->cfifo_lock, flags);
+                
+                if(tasklet) {
+                        __do_lit_tasklet(tasklet, 0ul);
+                        tasklet = NULL; 
+                }
+                else {
+                        work_to_do = 0;
+                }
+        }
+        
+        //TRACE("%s: exited.\n", __FUNCTION__);
+}
+
+
+static void run_tasklets(struct task_struct* sched_task)
+{
+        cfifo_domain_t* cluster;
+        
+#if 0
+        int task_is_rt = is_realtime(sched_task);
+        cfifo_domain_t* cluster;
+        
+        if(is_realtime(sched_task)) {
+                cluster = task_cpu_cluster(sched_task);
+        }
+        else {
+                cluster = remote_cluster(get_cpu());
+        }
+        
+        if(cluster && cluster->pending_tasklets.head != NULL) {
+                TRACE("%s: There are tasklets to process.\n", __FUNCTION__);
+                
+                do_lit_tasklets(cluster, sched_task);
+        }
+        
+        if(!task_is_rt) {
+                put_cpu_no_resched();
+        }
+#else
+        
+        preempt_disable();
+        
+        cluster = (is_realtime(sched_task)) ?
+                task_cpu_cluster(sched_task) :
+                remote_cluster(smp_processor_id());
+        
+        if(cluster && cluster->pending_tasklets.head != NULL) {
+                TRACE("%s: There are tasklets to process.\n", __FUNCTION__);
+                do_lit_tasklets(cluster, sched_task);
+        }
+        
+        preempt_enable_no_resched();
+        
+#endif
+}
+
+
+static void __add_pai_tasklet(struct tasklet_struct* tasklet, cfifo_domain_t* cluster)
+{
+        struct tasklet_struct* step;
+        
+        /*
+        step = cluster->pending_tasklets.head;
+        TRACE("%s: (BEFORE) dumping tasklet queue...\n", __FUNCTION__);
+        while(step != NULL){
+                TRACE("%s: %p (%d)\n", __FUNCTION__, step, step->owner->pid);
+                step = step->next;
+        }
+        TRACE("%s: tail = %p (%d)\n", __FUNCTION__, *(cluster->pending_tasklets.tail), (*(cluster->pending_tasklets.tail) != NULL) ? (*(cluster->pending_tasklets.tail))->owner->pid : -1);
+        TRACE("%s: done.\n", __FUNCTION__);
+         */
+        
+        
+        tasklet->next = NULL;  // make sure there are no old values floating around
+        
+        step = cluster->pending_tasklets.head;
+        if(step == NULL) {
+                TRACE("%s: tasklet queue empty.  inserting tasklet for %d at head.\n", __FUNCTION__, tasklet->owner->pid);
+                // insert at tail.
+                *(cluster->pending_tasklets.tail) = tasklet;
+                cluster->pending_tasklets.tail = &(tasklet->next);              
+        }
+        else if((*(cluster->pending_tasklets.tail) != NULL) &&
+                        fifo_higher_prio((*(cluster->pending_tasklets.tail))->owner, tasklet->owner)) {
+                // insert at tail.
+                TRACE("%s: tasklet belongs at end.  inserting tasklet for %d at tail.\n", __FUNCTION__, tasklet->owner->pid);
+                
+                *(cluster->pending_tasklets.tail) = tasklet;
+                cluster->pending_tasklets.tail = &(tasklet->next);
+        }
+        else {
+                
+        //WARN_ON(1 == 1);
+                
+                // insert the tasklet somewhere in the middle.
+                
+        TRACE("%s: tasklet belongs somewhere in the middle.\n", __FUNCTION__);
+                
+                while(step->next && fifo_higher_prio(step->next->owner, tasklet->owner)) {
+                        step = step->next;
+                }
+                
+                // insert tasklet right before step->next.
+                
+                TRACE("%s: inserting tasklet for %d between %d and %d.\n", __FUNCTION__, tasklet->owner->pid, step->owner->pid, (step->next) ? step->next->owner->pid : -1);
+                
+                tasklet->next = step->next;
+                step->next = tasklet;
+                
+                // patch up the head if needed.
+                if(cluster->pending_tasklets.head == step)
+                {
+                        TRACE("%s: %d is the new tasklet queue head.\n", __FUNCTION__, tasklet->owner->pid);
+                        cluster->pending_tasklets.head = tasklet;
+                }
+        }
+        
+        /*
+        step = cluster->pending_tasklets.head;
+        TRACE("%s: (AFTER) dumping tasklet queue...\n", __FUNCTION__);
+        while(step != NULL){
+                TRACE("%s: %p (%d)\n", __FUNCTION__, step, step->owner->pid);
+                step = step->next;
+        }
+        TRACE("%s: tail = %p (%d)\n", __FUNCTION__, *(cluster->pending_tasklets.tail), (*(cluster->pending_tasklets.tail) != NULL) ? (*(cluster->pending_tasklets.tail))->owner->pid : -1);
+        TRACE("%s: done.\n", __FUNCTION__);
+         */
+        
+        // TODO: Maintain this list in priority order.
+        //      tasklet->next = NULL;
+        //      *(cluster->pending_tasklets.tail) = tasklet;
+        //      cluster->pending_tasklets.tail = &tasklet->next;
+}
+
+static int enqueue_pai_tasklet(struct tasklet_struct* tasklet)
+{
+        cfifo_domain_t *cluster = NULL;
+        cpu_entry_t *targetCPU = NULL;
+        int thisCPU;
+        int runLocal = 0;
+        int runNow = 0;
+        unsigned long flags;
+        
+    if(unlikely((tasklet->owner == NULL) || !is_realtime(tasklet->owner)))
+    {
+        TRACE("%s: No owner associated with this tasklet!\n", __FUNCTION__);
+                return 0;
+    }   
+        
+        cluster = task_cpu_cluster(tasklet->owner);
+        
+        raw_spin_lock_irqsave(&cluster->cfifo_lock, flags);             
+        
+        thisCPU = smp_processor_id();
+        
+#if 1
+#ifdef CONFIG_SCHED_CPU_AFFINITY
+        {
+                cpu_entry_t* affinity = NULL;
+                
+                // use this CPU if it is in our cluster and isn't running any RT work.
+                if(cpu_isset(thisCPU, *cluster->cpu_map) && (__get_cpu_var(cfifo_cpu_entries).linked == NULL)) {
+                        affinity = &(__get_cpu_var(cfifo_cpu_entries));
+                }
+                else {
+                        // this CPU is busy or shouldn't run tasklet in this cluster.
+                        // look for available near by CPUs.
+                        // NOTE: Affinity towards owner and not this CPU.  Is this right?
+                        affinity = 
+                        cfifo_get_nearest_available_cpu(cluster,
+                                                                                   &per_cpu(cfifo_cpu_entries, task_cpu(tasklet->owner)));
+                }
+                
+                targetCPU = affinity;
+        }
+#endif
+#endif
+        
+        if (targetCPU == NULL) {
+                targetCPU = lowest_prio_cpu(cluster);
+        }
+        
+        if (fifo_higher_prio(tasklet->owner, targetCPU->linked)) {
+                if (thisCPU == targetCPU->cpu) {
+                        TRACE("%s: Run tasklet locally (and now).\n", __FUNCTION__);
+                        runLocal = 1;
+                        runNow = 1;
+                }
+                else {
+                        TRACE("%s: Run tasklet remotely (and now).\n", __FUNCTION__);
+                        runLocal = 0;
+                        runNow = 1;
+                }
+        }
+        else {
+                runLocal = 0;
+                runNow = 0;
+        }
+        
+        if(!runLocal) {
+                // enqueue the tasklet
+                __add_pai_tasklet(tasklet, cluster);
+        }
+        
+        raw_spin_unlock_irqrestore(&cluster->cfifo_lock, flags);
+        
+        
+        if (runLocal /*&& runNow */) {  // runNow == 1 is implied
+                TRACE("%s: Running tasklet on CPU where it was received.\n", __FUNCTION__);
+                __do_lit_tasklet(tasklet, 0ul);
+        }
+        else if (runNow /*&& !runLocal */) {  // runLocal == 0 is implied
+                TRACE("%s: Triggering CPU %d to run tasklet.\n", __FUNCTION__, targetCPU->cpu);
+                preempt(targetCPU);  // need to be protected by cfifo_lock?
+        }
+        else {
+                TRACE("%s: Scheduling of tasklet was deferred.\n", __FUNCTION__);
+        }
+        
+        return(1); // success
+}
+
+
+#endif
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 /* Getting schedule() right is a bit tricky. schedule() may not make any
  * assumptions on the state of the current task since it may be called for a
  * number of reasons. The reasons include a scheduler_tick() determined that it
@@ -544,7 +976,7 @@ static void cfifo_task_new(struct task_struct * t, int on_rq, int running)
         cpu_entry_t*            entry;
         cfifo_domain_t*         cluster;
 
-        TRACE("gsn edf: task new %d\n", t->pid);
+        TRACE("cfifo: task new %d\n", t->pid);
 
         /* the cluster doesn't change even if t is running */
         cluster = task_cpu_cluster(t);
@@ -650,6 +1082,10 @@ static void cfifo_task_exit(struct task_struct * t)
         }
         raw_spin_unlock_irqrestore(&cluster->cfifo_lock, flags);
 
+#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
+        flush_tasklets(cluster, t);
+#endif  
+        
         BUG_ON(!is_realtime(t));
         TRACE_TASK(t, "RIP\n");
 }
@@ -1467,6 +1903,12 @@ static long cfifo_activate_plugin(void)
                 bheap_init(&(cfifo[i].cpu_heap));
                 fifo_domain_init(&(cfifo[i].domain), NULL, cfifo_release_jobs);
 
+                
+#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
+                cfifo[i].pending_tasklets.head = NULL;
+                cfifo[i].pending_tasklets.tail = &(cfifo[i].pending_tasklets.head);
+#endif          
+                
                 if(!zalloc_cpumask_var(&cfifo[i].cpu_map, GFP_ATOMIC))
                         return -ENOMEM;
         }
@@ -1578,6 +2020,10 @@ static struct sched_plugin cfifo_plugin __cacheline_aligned_in_smp = {
 #ifdef CONFIG_LITMUS_SOFTIRQD
         .set_prio_inh_klitirqd = set_priority_inheritance_klitirqd,
         .clear_prio_inh_klitirqd = clear_priority_inheritance_klitirqd,
+#endif
+#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
+        .enqueue_pai_tasklet = enqueue_pai_tasklet,
+        .run_tasklets = run_tasklets,
 #endif  
 };