Final GPUSync implementation.gpusync-rtss12

3 files changed, 76 insertions, 42 deletions

diff --git a/kernel/mutex.c b/kernel/mutex.c
index 2f363b9bfc1f..96bcecd385d3 100644
--- a/kernel/mutex.c
+++ b/kernel/mutex.c
@@ -511,12 +511,12 @@ void mutex_lock_sfx(struct mutex *lock,
         struct task_struct *task = current;
         struct mutex_waiter waiter;
         unsigned long flags;
-        
+
         preempt_disable();
         mutex_acquire(&lock->dep_map, subclass, 0, ip);
 
         spin_lock_mutex(&lock->wait_lock, flags);
-        
+
         if(pre)
         {
                 if(unlikely(pre(pre_arg)))
@@ -530,16 +530,16 @@ void mutex_lock_sfx(struct mutex *lock,
 
         debug_mutex_lock_common(lock, &waiter);
         debug_mutex_add_waiter(lock, &waiter, task_thread_info(task));
-        
+
         /* add waiting tasks to the end of the waitqueue (FIFO): */
         list_add_tail(&waiter.list, &lock->wait_list);
         waiter.task = task;
-        
+
         if (atomic_xchg(&lock->count, -1) == 1)
                 goto done;
-        
+
         lock_contended(&lock->dep_map, ip);
-        
+
         for (;;) {
                 /*
                  * Lets try to take the lock again - this is needed even if
@@ -552,9 +552,9 @@ void mutex_lock_sfx(struct mutex *lock,
                  */
                 if (atomic_xchg(&lock->count, -1) == 1)
                         break;
-                
+
                 __set_task_state(task, state);
-                
+
                 /* didnt get the lock, go to sleep: */
                 spin_unlock_mutex(&lock->wait_lock, flags);
                 preempt_enable_no_resched();
@@ -562,22 +562,22 @@ void mutex_lock_sfx(struct mutex *lock,
                 preempt_disable();
                 spin_lock_mutex(&lock->wait_lock, flags);
         }
-        
+
 done:
         lock_acquired(&lock->dep_map, ip);
         /* got the lock - rejoice! */
         mutex_remove_waiter(lock, &waiter, current_thread_info());
         mutex_set_owner(lock);
-        
+
         /* set it to 0 if there are no waiters left: */
         if (likely(list_empty(&lock->wait_list)))
                 atomic_set(&lock->count, 0);
-        
+
         if(post)
-                post(post_arg); 
-        
+                post(post_arg);
+
         spin_unlock_mutex(&lock->wait_lock, flags);
-        
+
         debug_mutex_free_waiter(&waiter);
         preempt_enable();
 }
@@ -588,16 +588,16 @@ void mutex_unlock_sfx(struct mutex *lock,
                                         side_effect_t post, unsigned long post_arg)
 {
         unsigned long flags;
-        
+
         spin_lock_mutex(&lock->wait_lock, flags);
-        
+
         if(pre)
                 pre(pre_arg);
-        
+
         //mutex_release(&lock->dep_map, nested, _RET_IP_);
         mutex_release(&lock->dep_map, 1, _RET_IP_);
         debug_mutex_unlock(lock);
-        
+
         /*
          * some architectures leave the lock unlocked in the fastpath failure
          * case, others need to leave it locked. In the later case we have to
@@ -605,21 +605,21 @@ void mutex_unlock_sfx(struct mutex *lock,
          */
         if (__mutex_slowpath_needs_to_unlock())
                 atomic_set(&lock->count, 1);
-        
+
         if (!list_empty(&lock->wait_list)) {
                 /* get the first entry from the wait-list: */
                 struct mutex_waiter *waiter =
                 list_entry(lock->wait_list.next,
                                    struct mutex_waiter, list);
-                
+
                 debug_mutex_wake_waiter(lock, waiter);
-                
+
                 wake_up_process(waiter->task);
         }
-        
+
         if(post)
                 post(post_arg);
-        
-        spin_unlock_mutex(&lock->wait_lock, flags);     
+
+        spin_unlock_mutex(&lock->wait_lock, flags);
 }
 EXPORT_SYMBOL(mutex_unlock_sfx);
diff --git a/kernel/sched.c b/kernel/sched.c
index f3d9a69a3777..2f990b4b24f9 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -4430,8 +4430,8 @@ litmus_need_resched_nonpreemptible:
 
 #ifdef CONFIG_LITMUS_PAI_SOFTIRQD
         litmus->run_tasklets(prev);
-#endif  
-        
+#endif
+
         srp_ceiling_block();
 }
 EXPORT_SYMBOL(schedule);
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 1c42e08fdfaa..4d7b1a3e4d01 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -216,7 +216,7 @@ EXPORT_SYMBOL(local_bh_enable_ip);
 
 asmlinkage void __do_softirq(void)
 {
-    struct softirq_action *h; 
+    struct softirq_action *h;
     __u32 pending;
     int max_restart = MAX_SOFTIRQ_RESTART;
     int cpu;
@@ -254,10 +254,10 @@ restart:
                        softirq_to_name[vec_nr], h->action,
                        prev_count, preempt_count());
                 preempt_count() = prev_count;
-            }   
+            }
 
             rcu_bh_qs(cpu);
-        }   
+        }
         h++;
         pending >>= 1;
     } while (pending);
@@ -412,13 +412,45 @@ struct tasklet_head
 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 
+#ifdef CONFIG_LITMUS_NVIDIA
+static int __do_nv_now(struct tasklet_struct* tasklet)
+{
+        int success = 1;
+
+        if(tasklet_trylock(tasklet)) {
+                if (!atomic_read(&tasklet->count)) {
+                        if (!test_and_clear_bit(TASKLET_STATE_SCHED, &tasklet->state)) {
+                                BUG();
+                        }
+                        tasklet->func(tasklet->data);
+                        tasklet_unlock(tasklet);
+                }
+                else {
+                        success = 0;
+                }
+
+                tasklet_unlock(tasklet);
+        }
+        else {
+                success = 0;
+        }
+
+        return success;
+}
+#endif
+
 
 void __tasklet_schedule(struct tasklet_struct *t)
 {
 #ifdef CONFIG_LITMUS_NVIDIA
         if(is_nvidia_func(t->func))
         {
-                u32 nvidia_device = get_tasklet_nv_device_num(t);       
+#if 0
+                // do nvidia tasklets right away and return
+                if(__do_nv_now(t))
+                        return;
+#else
+                u32 nvidia_device = get_tasklet_nv_device_num(t);
                 //              TRACE("%s: Handling NVIDIA tasklet for device\t%u\tat\t%llu\n",
                 //                        __FUNCTION__, nvidia_device,litmus_clock());
 
@@ -438,7 +470,7 @@ void __tasklet_schedule(struct tasklet_struct *t)
                         if(is_realtime(device_owner))
                         {
                                 TRACE("%s: Handling NVIDIA tasklet for device %u at %llu\n",
-                                          __FUNCTION__, nvidia_device,litmus_clock());                          
+                                          __FUNCTION__, nvidia_device,litmus_clock());
                                 TRACE("%s: the owner task %d of NVIDIA Device %u is RT-task\n",
                                           __FUNCTION__,device_owner->pid,nvidia_device);
 
@@ -461,7 +493,9 @@ void __tasklet_schedule(struct tasklet_struct *t)
                         }
                 }
                 unlock_nv_registry(nvidia_device, &flags);
+#endif
         }
+
 #endif
 
         ___tasklet_schedule(t);
@@ -487,19 +521,19 @@ void __tasklet_hi_schedule(struct tasklet_struct *t)
 {
 #ifdef CONFIG_LITMUS_NVIDIA
         if(is_nvidia_func(t->func))
-        {       
+        {
                 u32 nvidia_device = get_tasklet_nv_device_num(t);
                 //              TRACE("%s: Handling NVIDIA tasklet for device\t%u\tat\t%llu\n",
                 //                        __FUNCTION__, nvidia_device,litmus_clock());
 
                 unsigned long flags;
                 struct task_struct* device_owner;
-                
+
                 lock_nv_registry(nvidia_device, &flags);
-                
+
                 device_owner = get_nv_max_device_owner(nvidia_device);
 
-                if(device_owner==NULL) 
+                if(device_owner==NULL)
                 {
                         t->owner = NULL;
                 }
@@ -508,10 +542,10 @@ void __tasklet_hi_schedule(struct tasklet_struct *t)
                         if( is_realtime(device_owner))
                         {
                                 TRACE("%s: Handling NVIDIA tasklet for device %u\tat %llu\n",
-                                          __FUNCTION__, nvidia_device,litmus_clock());                          
+                                          __FUNCTION__, nvidia_device,litmus_clock());
                                 TRACE("%s: the owner task %d of NVIDIA Device %u is RT-task\n",
                                           __FUNCTION__,device_owner->pid,nvidia_device);
-                                
+
                                 t->owner = device_owner;
                                 sched_trace_tasklet_release(t->owner);
                                 if(likely(_litmus_tasklet_hi_schedule(t,nvidia_device)))
@@ -553,15 +587,15 @@ EXPORT_SYMBOL(___tasklet_hi_schedule);
 void __tasklet_hi_schedule_first(struct tasklet_struct *t)
 {
         BUG_ON(!irqs_disabled());
-#ifdef CONFIG_LITMUS_NVIDIA     
+#ifdef CONFIG_LITMUS_NVIDIA
         if(is_nvidia_func(t->func))
-        {       
+        {
                 u32 nvidia_device = get_tasklet_nv_device_num(t);
                 //              TRACE("%s: Handling NVIDIA tasklet for device\t%u\tat\t%llu\n",
                 //                        __FUNCTION__, nvidia_device,litmus_clock());
                 unsigned long flags;
                 struct task_struct* device_owner;
-                
+
                 lock_nv_registry(nvidia_device, &flags);
 
                 device_owner = get_nv_max_device_owner(nvidia_device);
@@ -576,10 +610,10 @@ void __tasklet_hi_schedule_first(struct tasklet_struct *t)
                         {
                                 TRACE("%s: Handling NVIDIA tasklet for device %u at %llu\n",
                                           __FUNCTION__, nvidia_device,litmus_clock());
-                                
+
                                 TRACE("%s: the owner task %d of NVIDIA Device %u is RT-task\n",
                                           __FUNCTION__,device_owner->pid,nvidia_device);
-                                
+
                                 t->owner = device_owner;
                                 sched_trace_tasklet_release(t->owner);
                                 if(likely(_litmus_tasklet_hi_schedule_first(t,nvidia_device)))