Litmus core: simplify np-section protocol

User a 32-bit word for all non-preemptive section flags.
Set the "please yield soon" flag atomically when
accessing it on remotely-scheduled tasks.

4 files changed, 57 insertions, 31 deletions

diff --git a/include/litmus/litmus.h b/include/litmus/litmus.h
index e7769ca36ec..12af2226633 100644
--- a/include/litmus/litmus.h
+++ b/include/litmus/litmus.h
@@ -137,7 +137,7 @@ static inline int is_kernel_np(struct task_struct *t)
 
 static inline int is_user_np(struct task_struct *t)
 {
-        return tsk_rt(t)->ctrl_page ? tsk_rt(t)->ctrl_page->np_flag : 0;
+        return tsk_rt(t)->ctrl_page ? tsk_rt(t)->ctrl_page->sched.np.flag : 0;
 }
 
 static inline void request_exit_np(struct task_struct *t)
@@ -147,17 +147,11 @@ static inline void request_exit_np(struct task_struct *t)
                  * into the kernel at the end of a critical section. */
                 if (likely(tsk_rt(t)->ctrl_page)) {
                         TRACE_TASK(t, "setting delayed_preemption flag\n");
-                        tsk_rt(t)->ctrl_page->delayed_preemption = 1;
+                        tsk_rt(t)->ctrl_page->sched.np.preempt = 1;
                 }
         }
 }
 
-static inline void clear_exit_np(struct task_struct *t)
-{
-        if (likely(tsk_rt(t)->ctrl_page))
-                tsk_rt(t)->ctrl_page->delayed_preemption = 0;
-}
-
 static inline void make_np(struct task_struct *t)
 {
         tsk_rt(t)->kernel_np++;
@@ -171,6 +165,34 @@ static inline int take_np(struct task_struct *t)
         return --tsk_rt(t)->kernel_np;
 }
 
+/* returns 0 if remote CPU needs an IPI to preempt, 1 if no IPI is required */
+static inline int request_exit_np_atomic(struct task_struct *t)
+{
+        union np_flag old, new;
+
+        if (tsk_rt(t)->ctrl_page) {
+                old.raw = tsk_rt(t)->ctrl_page->sched.raw;
+                if (old.np.flag == 0) {
+                        /* no longer non-preemptive */
+                        return 0;
+                } else if (old.np.preempt) {
+                        /* already set, nothing for us to do */
+                        return 1;
+                } else {
+                        /* non preemptive and flag not set */
+                        new.raw = old.raw;
+                        new.np.preempt = 1;
+                        /* if we get old back, then we atomically set the flag */
+                        return cmpxchg(&tsk_rt(t)->ctrl_page->sched.raw, old.raw, new.raw) == old.raw;
+                        /* If we raced with a concurrent change, then so be
+                         * it. Deliver it by IPI.  We don't want an unbounded
+                         * retry loop here since tasks might exploit that to
+                         * keep the kernel busy indefinitely. */
+                }
+        } else
+                return 0;
+}
+
 #else
 
 static inline int is_kernel_np(struct task_struct* t)
@@ -189,12 +211,19 @@ static inline void request_exit_np(struct task_struct *t)
         BUG();
 }
 
-static inline void clear_exit_np(struct task_struct* t)
+static inline int request_exist_np_atomic(struct task_struct *t)
 {
+        return 0;
 }
 
 #endif
 
+static inline void clear_exit_np(struct task_struct *t)
+{
+        if (likely(tsk_rt(t)->ctrl_page))
+                tsk_rt(t)->ctrl_page->sched.np.preempt = 0;
+}
+
 static inline int is_np(struct task_struct *t)
 {
 #ifdef CONFIG_SCHED_DEBUG_TRACE
diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h
index 389be077586..d6d79917416 100644
--- a/include/litmus/rt_param.h
+++ b/include/litmus/rt_param.h
@@ -42,6 +42,16 @@ struct rt_task {
         budget_policy_t budget_policy; /* ignored by pfair */
 };
 
+union np_flag {
+        uint32_t raw;
+        struct {
+                /* Is the task currently in a non-preemptive section? */
+                uint32_t flag:31;
+                /* Should the task call into the scheduler? */
+                uint32_t preempt:1;
+        } np;
+};
+
 /* The definition of the data that is shared between the kernel and real-time
  * tasks via a shared page (see litmus/ctrldev.c).
  *
@@ -57,11 +67,7 @@ struct rt_task {
  * determining preemption/migration overheads).
  */
 struct control_page {
-        /* Is the task currently in a non-preemptive section? */
-        int np_flag;
-        /* Should the task call into the kernel when it leaves
-         * its non-preemptive section? */
-        int delayed_preemption;
+        volatile union np_flag sched;
 
         /* to be extended */
 };
diff --git a/litmus/litmus.c b/litmus/litmus.c
index 73af6c3010d..301390148d0 100644
--- a/litmus/litmus.c
+++ b/litmus/litmus.c
@@ -529,6 +529,8 @@ static int __init _init_litmus(void)
          */
         printk("Starting LITMUS^RT kernel\n");
 
+        BUILD_BUG_ON(sizeof(union np_flag) != sizeof(uint32_t));
+
         register_sched_plugin(&linux_sched_plugin);
 
         bheap_node_cache    = KMEM_CACHE(bheap_node, SLAB_PANIC);
diff --git a/litmus/sched_plugin.c b/litmus/sched_plugin.c
index d54886df1f5..00a1900d645 100644
--- a/litmus/sched_plugin.c
+++ b/litmus/sched_plugin.c
@@ -35,29 +35,18 @@ void preempt_if_preemptable(struct task_struct* t, int cpu)
                         /* local CPU case */
                         /* check if we need to poke userspace */
                         if (is_user_np(t))
-                                /* yes, poke it */
+                                /* Yes, poke it. This doesn't have to be atomic since
+                                 * the task is definitely not executing. */
                                 request_exit_np(t);
                         else if (!is_kernel_np(t))
                                 /* only if we are allowed to preempt the
                                  * currently-executing task */
                                 reschedule = 1;
                 } else {
-                        /* remote CPU case */
-                        if (is_user_np(t)) {
-                                /* need to notify user space of delayed
-                                 * preemption */
-
-                                /* to avoid a race, set the flag, then test
-                                 * again */
-                                request_exit_np(t);
-                                /* make sure it got written */
-                                mb();
-                        }
-                        /* Only send an ipi if remote task might have raced our
-                         * request, i.e., send an IPI to make sure in case it
-                         * exited its critical section.
-                         */
-                        reschedule = !is_np(t) && !is_kernel_np(t);
+                        /* Remote CPU case.  Only notify if it's not a kernel
+                         * NP section and if we didn't set the userspace
+                         * flag. */
+                        reschedule = !(is_kernel_np(t) || request_exit_np_atomic(t));
                 }
         }
         if (likely(reschedule))