Update PAI to support multiGPUs (todo: klitirqd)

1 files changed, 133 insertions, 70 deletions

diff --git a/litmus/sched_gsn_edf.c b/litmus/sched_gsn_edf.c
index e009b7f34aca..37f7821dca50 100644
--- a/litmus/sched_gsn_edf.c
+++ b/litmus/sched_gsn_edf.c
@@ -436,14 +436,6 @@ static void gsnedf_tick(struct task_struct* t)
 
 
 
-
-
-
-
-
-
-
-
 #ifdef CONFIG_LITMUS_PAI_SOFTIRQD
 
 
@@ -467,96 +459,117 @@ static void __do_lit_tasklet(struct tasklet_struct* tasklet, unsigned long flush
         }
 }
 
-
-static void flush_tasklets(struct task_struct* task)
-{
-        // lazy flushing.
-        // just change ownership to NULL and let idel processor
-        // take care of it. :P
-
-        struct tasklet_struct* step;
-        unsigned long flags;
-
-        raw_spin_lock_irqsave(&gsnedf_lock, flags);
-        for(step = gsnedf_pending_tasklets.head; step != NULL; step = step->next) {
-                if(step->owner == task) {
-                        TRACE("%s: Found tasklet to flush: %d\n", __FUNCTION__, step->owner->pid);
-                        step->owner = NULL;
-                }
-        }
-        raw_spin_unlock_irqrestore(&gsnedf_lock, flags);
-}
-
-
 static void do_lit_tasklets(struct task_struct* sched_task)
 {
         int work_to_do = 1;
         struct tasklet_struct *tasklet = NULL;
-        //struct tasklet_struct *step;
         unsigned long flags;
         
         while(work_to_do) {
                 
                 TS_NV_SCHED_BOTISR_START;
                 
-                // remove tasklet at head of list if it has higher priority.
+                // execute one tasklet that has higher priority
                 raw_spin_lock_irqsave(&gsnedf_lock, flags);     
                 
                 if(gsnedf_pending_tasklets.head != NULL) {
-                        // remove tasklet at head.
-                        tasklet = gsnedf_pending_tasklets.head;
+                        struct tasklet_struct *prev = NULL;
+                        tasklet = gsnedf_pending_tasklets.head;                 
+
+                        while(tasklet && edf_higher_prio(sched_task, tasklet->owner)) {
+                                prev = tasklet;
+                                tasklet = tasklet->next;
+                        }
                         
-                        if(edf_higher_prio(tasklet->owner, sched_task)) {
-                                
-                                if(NULL == tasklet->next) {
-                                        // tasklet is at the head, list only has one element
+                        // remove the tasklet from the queue
+                        if(prev) {
+                                prev->next = tasklet->next;
+                                if(prev->next == NULL) {
                                         TRACE("%s: Tasklet for %d is the last element in tasklet queue.\n", __FUNCTION__, tasklet->owner->pid);
-                                        gsnedf_pending_tasklets.tail = &(gsnedf_pending_tasklets.head);
-                                }
-                                
-                                // remove the tasklet from the queue
-                                gsnedf_pending_tasklets.head = tasklet->next;
-                                
-                                TRACE("%s: Removed tasklet for %d from tasklet queue.\n", __FUNCTION__, tasklet->owner->pid);
+                                        gsnedf_pending_tasklets.tail = &(prev);                                 
+                                }                               
                         }
                         else {
-                                TRACE("%s: Pending tasklet (%d) does not have priority to run on this CPU (%d).\n", __FUNCTION__, tasklet->owner->pid, smp_processor_id());
-                                tasklet = NULL;
+                                gsnedf_pending_tasklets.head = tasklet->next;
+                                if(tasklet->next == NULL) {
+                                        TRACE("%s: Tasklet for %d is the last element in tasklet queue.\n", __FUNCTION__, tasklet->owner->pid);
+                                        gsnedf_pending_tasklets.tail = &(gsnedf_pending_tasklets.head);                                 
+                                }
                         }
                 }
                 else {
                         TRACE("%s: Tasklet queue is empty.\n", __FUNCTION__);
-                }
+                }               
                 
                 raw_spin_unlock_irqrestore(&gsnedf_lock, flags);
                 
-                TS_NV_SCHED_BOTISR_END;
-                
                 if(tasklet) {
                         __do_lit_tasklet(tasklet, 0ul);
-                        tasklet = NULL; 
+                        tasklet = NULL;                         
                 }
                 else {
                         work_to_do = 0;
                 }
+                
+                TS_NV_SCHED_BOTISR_END;         
         }
-        
-        //TRACE("%s: exited.\n", __FUNCTION__);
-}
-
-
-static void run_tasklets(struct task_struct* sched_task)
-{
-        preempt_disable();
-        
-        if(gsnedf_pending_tasklets.head != NULL) {
-                TRACE("%s: There are tasklets to process.\n", __FUNCTION__);
-                do_lit_tasklets(sched_task);
-        }
-        
-        preempt_enable_no_resched();
 }
 
+//static void do_lit_tasklets(struct task_struct* sched_task)
+//{
+//      int work_to_do = 1;
+//      struct tasklet_struct *tasklet = NULL;
+//      //struct tasklet_struct *step;
+//      unsigned long flags;
+//      
+//      while(work_to_do) {
+//              
+//              TS_NV_SCHED_BOTISR_START;
+//              
+//              // remove tasklet at head of list if it has higher priority.
+//              raw_spin_lock_irqsave(&gsnedf_lock, flags);     
+//              
+//              if(gsnedf_pending_tasklets.head != NULL) {
+//                      // remove tasklet at head.
+//                      tasklet = gsnedf_pending_tasklets.head;
+//                      
+//                      if(edf_higher_prio(tasklet->owner, sched_task)) {
+//                              
+//                              if(NULL == tasklet->next) {
+//                                      // tasklet is at the head, list only has one element
+//                                      TRACE("%s: Tasklet for %d is the last element in tasklet queue.\n", __FUNCTION__, tasklet->owner->pid);
+//                                      gsnedf_pending_tasklets.tail = &(gsnedf_pending_tasklets.head);
+//                              }
+//                              
+//                              // remove the tasklet from the queue
+//                              gsnedf_pending_tasklets.head = tasklet->next;
+//                              
+//                              TRACE("%s: Removed tasklet for %d from tasklet queue.\n", __FUNCTION__, tasklet->owner->pid);
+//                      }
+//                      else {
+//                              TRACE("%s: Pending tasklet (%d) does not have priority to run on this CPU (%d).\n", __FUNCTION__, tasklet->owner->pid, smp_processor_id());
+//                              tasklet = NULL;
+//                      }
+//              }
+//              else {
+//                      TRACE("%s: Tasklet queue is empty.\n", __FUNCTION__);
+//              }
+//              
+//              raw_spin_unlock_irqrestore(&gsnedf_lock, flags);
+//              
+//              TS_NV_SCHED_BOTISR_END;
+//              
+//              if(tasklet) {
+//                      __do_lit_tasklet(tasklet, 0ul);
+//                      tasklet = NULL; 
+//              }
+//              else {
+//                      work_to_do = 0;
+//              }
+//      }
+//      
+//      //TRACE("%s: exited.\n", __FUNCTION__);
+//}
 
 static void __add_pai_tasklet(struct tasklet_struct* tasklet)
 {
@@ -604,7 +617,19 @@ static void __add_pai_tasklet(struct tasklet_struct* tasklet)
         }
 }
 
-static int enqueue_pai_tasklet(struct tasklet_struct* tasklet)
+static void gsnedf_run_tasklets(struct task_struct* sched_task)
+{
+        preempt_disable();
+        
+        if(gsnedf_pending_tasklets.head != NULL) {
+                TRACE("%s: There are tasklets to process.\n", __FUNCTION__);
+                do_lit_tasklets(sched_task);
+        }
+        
+        preempt_enable_no_resched();
+}
+
+static int gsnedf_enqueue_pai_tasklet(struct tasklet_struct* tasklet)
 {
         cpu_entry_t *targetCPU = NULL;
         int thisCPU;
@@ -692,6 +717,23 @@ static int enqueue_pai_tasklet(struct tasklet_struct* tasklet)
         return(1); // success
 }
 
+static void gsnedf_change_prio_pai_tasklet(struct task_struct *old_prio,
+                                                                                   struct task_struct *new_prio)
+{
+        struct tasklet_struct* step;
+        unsigned long flags;
+        
+        raw_spin_lock_irqsave(&gsnedf_lock, flags);
+        
+        for(step = gsnedf_pending_tasklets.head; step != NULL; step = step->next) {
+                if(step->owner == old_prio) {
+                        TRACE("%s: Found tasklet to flush: %d\n", __FUNCTION__, step->owner->pid);
+                        step->owner = new_prio;
+                }
+        }
+        raw_spin_unlock_irqrestore(&gsnedf_lock, flags);
+}
+
 #endif  // end PAI
 
 
@@ -954,7 +996,7 @@ static void gsnedf_task_exit(struct task_struct * t)
         unsigned long flags;
 
 #ifdef CONFIG_LITMUS_PAI_SOFTIRQD
-        flush_tasklets(t);
+        gsnedf_change_prio_pai_tasklet(t, NULL);
 #endif          
         
         /* unlink if necessary */
@@ -1072,6 +1114,8 @@ static void __increase_priority_inheritance(struct task_struct* t,
 /* called with IRQs off */
 static void increase_priority_inheritance(struct task_struct* t, struct task_struct* prio_inh)
 {
+        int i = 0;
+        
         raw_spin_lock(&gsnedf_lock);
 
         __increase_priority_inheritance(t, prio_inh);
@@ -1087,6 +1131,14 @@ static void increase_priority_inheritance(struct task_struct* t, struct task_str
 #endif
         
         raw_spin_unlock(&gsnedf_lock);
+        
+#if defined(CONFIG_LITMUS_PAI_SOFTIRQD) && defined(CONFIG_LITMUS_NVIDIA)
+        for(i = find_first_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus));
+                i < NV_DEVICE_NUM;
+                i = find_next_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus), i)) {
+                pai_check_priority_increase(t, i);
+        }
+#endif
 }
 
 
@@ -1147,6 +1199,8 @@ static void __decrease_priority_inheritance(struct task_struct* t,
 static void decrease_priority_inheritance(struct task_struct* t,
                                                                                   struct task_struct* prio_inh)
 {
+        int i;
+        
         raw_spin_lock(&gsnedf_lock);
         __decrease_priority_inheritance(t, prio_inh);
         
@@ -1160,7 +1214,15 @@ static void decrease_priority_inheritance(struct task_struct* t,
         }
 #endif  
         
-        raw_spin_unlock(&gsnedf_lock);  
+        raw_spin_unlock(&gsnedf_lock);
+        
+#if defined(CONFIG_LITMUS_PAI_SOFTIRQD) && defined(CONFIG_LITMUS_NVIDIA)
+        for(i = find_first_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus));
+                i < NV_DEVICE_NUM;
+                i = find_next_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus), i)) {
+                pai_check_priority_decrease(t, i);
+        }
+#endif  
 }
 
 
@@ -1687,8 +1749,9 @@ static struct sched_plugin gsn_edf_plugin __cacheline_aligned_in_smp = {
         .decrease_prio_klitirqd = decrease_priority_inheritance_klitirqd,
 #endif
 #ifdef CONFIG_LITMUS_PAI_SOFTIRQD
-        .enqueue_pai_tasklet = enqueue_pai_tasklet,
-        .run_tasklets = run_tasklets,
+        .enqueue_pai_tasklet = gsnedf_enqueue_pai_tasklet,
+        .change_prio_pai_tasklet = gsnedf_change_prio_pai_tasklet,
+        .run_tasklets = gsnedf_run_tasklets,
 #endif
 };