5 files changed, 65 insertions, 46 deletions

diff --git a/include/litmus/litmus.h b/include/litmus/litmus.h
index 512efef341..8fc9f630c8 100644
--- a/include/litmus/litmus.h
+++ b/include/litmus/litmus.h
@@ -200,5 +200,6 @@ static inline lt_t litmus_clock(void)
 #define make_np(t) do {t->rt_param.kernel_np++;} while(0);
 #define take_np(t) do {t->rt_param.kernel_np--;} while(0);
 
+void srp_ceiling_block(void);
 
 #endif
diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h
index 118e8590fd..933c1961cb 100644
--- a/include/litmus/rt_param.h
+++ b/include/litmus/rt_param.h
@@ -73,6 +73,9 @@ struct rt_param {
         /* is the task sleeping? */
         unsigned int            flags:8;
 
+        /* do we need to check for srp blocking? */
+        unsigned int            srp_non_recurse:1;
+
         /* user controlled parameters */
         struct rt_task          task_params;
 
diff --git a/kernel/sched.c b/kernel/sched.c
index 5d31bc9d56..00097dea89 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -3688,6 +3688,8 @@ need_resched_nonpreemptible:
         preempt_enable_no_resched();
         if (unlikely(test_thread_flag(TIF_NEED_RESCHED)))
                 goto need_resched;
+        if (srp_active())
+                srp_ceiling_block();
 }
 EXPORT_SYMBOL(schedule);
 
diff --git a/litmus/litmus.c b/litmus/litmus.c
index 2909d5c04e..60aed9dd30 100644
--- a/litmus/litmus.c
+++ b/litmus/litmus.c
@@ -153,8 +153,6 @@ asmlinkage long sys_complete_job(void)
          * appropriate queue and call schedule
          */
         retval = litmus->complete_job();
-        if (!retval)
-                srp_ceiling_block();
       out:
         return retval;
 }
@@ -200,15 +198,10 @@ asmlinkage long sys_wait_for_job_release(unsigned int job)
          * FIXME: At least problem 2 should be taken care of eventually.
          */
         while (!retval && job > current->rt_param.job_params.job_no)
-          /* If the last job overran then job <= job_no and we
-           * don't send the task to sleep.
-           */
-          retval = litmus->complete_job();
-
-        /* We still have to honor the SRP after the actual release.
-         */
-        if (!retval)
-                srp_ceiling_block();
+                /* If the last job overran then job <= job_no and we
+                 * don't send the task to sleep.
+                 */
+                retval = litmus->complete_job();
       out:
         return retval;
 }
diff --git a/litmus/litmus_sem.c b/litmus/litmus_sem.c
index 60e82a2494..5d51d337d9 100644
--- a/litmus/litmus_sem.c
+++ b/litmus/litmus_sem.c
@@ -317,6 +317,24 @@ atomic_t srp_objects_in_use = ATOMIC_INIT(0);
 
 DEFINE_PER_CPU(struct srp, srp);
 
+
+/* Initialize SRP semaphores at boot time. */
+static int __init srp_init(void)
+{
+        int i;
+
+        printk("Initializing SRP per-CPU ceilings...");
+        for (i = 0; i < NR_CPUS; i++) {
+                init_waitqueue_head(&per_cpu(srp, i).ceiling_blocked);
+                INIT_LIST_HEAD(&per_cpu(srp, i).ceiling);
+        }
+        printk(" done!\n");
+
+        return 0;
+}
+module_init(srp_init);
+
+
 #define system_ceiling(srp) list2prio(srp->ceiling.next)
 
 
@@ -347,7 +365,7 @@ static void srp_add_prio(struct srp* srp, struct srp_priority* prio)
         struct list_head *pos;
         if (in_list(&prio->list)) {
                 printk(KERN_CRIT "WARNING: SRP violation detected, prio is already in "
-                       "ceiling list!\n");
+                       "ceiling list! cpu=%d, srp=%p\n", smp_processor_id(), ceiling2sem(prio));
                 return;
         }
         list_for_each(pos, &srp->ceiling)
@@ -384,6 +402,8 @@ static noinline int open_srp_semaphore(struct od_table_entry* entry, void* __use
         struct srp_priority t_prio;
 
         TRACE("opening SRP semaphore %p, cpu=%d\n", sem, sem->cpu);
+        if (!srp_active())
+                return -EBUSY;
 
         if (sem->cpu == UNDEF_SEM)
                 sem->cpu = get_partition(t);
@@ -415,22 +435,6 @@ struct fdso_ops srp_sem_ops = {
         .destroy = destroy_srp_semaphore
 };
 
-/* Initialize SRP semaphores at boot time. */
-static int __init srp_sema_boot_init(void)
-{
-        int i;
-
-        printk("Initializing SRP per-CPU ceilings...");
-        for (i = 0; i < NR_CPUS; i++) {
-                init_waitqueue_head(&per_cpu(srp, i).ceiling_blocked);
-                INIT_LIST_HEAD(&per_cpu(srp, i).ceiling);
-        }
-        printk(" done!\n");
-
-        return 0;
-}
-__initcall(srp_sema_boot_init);
-
 
 void do_srp_down(struct srp_semaphore* sem)
 {
@@ -438,18 +442,20 @@ void do_srp_down(struct srp_semaphore* sem)
         srp_add_prio(&__get_cpu_var(srp), &sem->ceiling);
         WARN_ON(sem->owner != NULL);
         sem->owner = current;
+        TRACE_CUR("acquired srp 0x%p\n", sem);
 }
 
 void do_srp_up(struct srp_semaphore* sem)
 {       
         /* Determine new system priority ceiling for this CPU. */
         WARN_ON(!in_list(&sem->ceiling.list));
-        if (!in_list(&sem->ceiling.list))
+        if (in_list(&sem->ceiling.list))
                 list_del(&sem->ceiling.list);
 
         sem->owner = NULL;
 
         /* Wake tasks on this CPU, if they exceed current ceiling. */
+        TRACE_CUR("released srp 0x%p\n", sem);
         wake_up_all(&__get_cpu_var(srp).ceiling_blocked);
 }
 
@@ -522,36 +528,50 @@ static int srp_wake_up(wait_queue_t *wait, unsigned mode, int sync,
 }
 
 
-/* Wait for current task priority to exceed system-wide priority ceiling.
- * Can be used to determine when it is safe to run a job after its release.
- */
-void srp_ceiling_block(void)
+
+static void do_ceiling_block(struct task_struct *tsk)
 {
-        struct task_struct *tsk = current;
         wait_queue_t wait = {
                 .private   = tsk,
                 .func      = srp_wake_up,
                 .task_list = {NULL, NULL}
         };
 
-        /* bail out early if there aren't any SRP resources around */
-        if (!atomic_read(&srp_objects_in_use))
+        tsk->state = TASK_UNINTERRUPTIBLE;
+        add_wait_queue(&__get_cpu_var(srp).ceiling_blocked, &wait);
+        tsk->rt_param.srp_non_recurse = 1;
+        preempt_enable_no_resched();
+        schedule();
+        preempt_disable();
+        tsk->rt_param.srp_non_recurse = 0;
+        remove_wait_queue(&__get_cpu_var(srp).ceiling_blocked, &wait);
+}
+
+/* Wait for current task priority to exceed system-wide priority ceiling.
+ */
+void srp_ceiling_block(void)
+{
+        struct task_struct *tsk = current;
+
+        /* Only applies to real-time tasks, but optimize for RT tasks. */
+        if (unlikely(!is_realtime(tsk)))
+                return;
+
+        /* Avoid recursive ceiling blocking. */
+        if (unlikely(tsk->rt_param.srp_non_recurse))
+                return;
+
+        /* Bail out early if there aren't any SRP resources around. */
+        if (likely(!atomic_read(&srp_objects_in_use)))
                 return;
 
         preempt_disable();
+        tsk->rt_param.srp_block = 0;
         if (!srp_exceeds_ceiling(tsk, &__get_cpu_var(srp))) {
-                tsk->state = TASK_UNINTERRUPTIBLE;
-                add_wait_queue(&__get_cpu_var(srp).ceiling_blocked, &wait);
                 TRACE_CUR("is priority ceiling blocked.\n");
-                preempt_enable_no_resched();
-                schedule();
-                /* Access to CPU var must occur with preemptions disabled,
-                 * otherwise Linux debug code complains loudly, even if it is
-                 * ok here.
-                 */
-                preempt_disable();
+                while (!srp_exceeds_ceiling(tsk, &__get_cpu_var(srp)))
+                        do_ceiling_block(tsk);
                 TRACE_CUR("finally exceeds system ceiling.\n");
-                remove_wait_queue(&__get_cpu_var(srp).ceiling_blocked, &wait);
         } else
                 TRACE_CUR("is not priority ceiling blocked\n"); 
         preempt_enable();