CONFIG: Allow ready queue lock to be recurisve.

Adds a configuration option to allow the ready queue
to be a recursive (raw) spinlock.  This is useful
in implementing inheritance from nested locking
and budget enforcement actions.

5 files changed, 69 insertions, 25 deletions

diff --git a/include/litmus/rt_domain.h b/include/litmus/rt_domain.h
index 6d3c4672dca9..182314afac7c 100644
--- a/include/litmus/rt_domain.h
+++ b/include/litmus/rt_domain.h
@@ -7,6 +7,10 @@
 
 #include <litmus/bheap.h>
 
+#ifdef CONFIG_RECURSIVE_READYQ_LOCK
+#include <litmus/rspinlock.h>
+#endif
+
 #define RELEASE_QUEUE_SLOTS 127 /* prime */
 
 struct _rt_domain;
@@ -20,9 +24,25 @@ struct release_queue {
         struct list_head                slot[RELEASE_QUEUE_SLOTS];
 };
 
+#ifdef CONFIG_RECURSIVE_READYQ_LOCK
+#define raw_readyq_spinlock_t   raw_rspinlock_t
+#define raw_readyq_lock_irqsave raw_rspin_lock_irqsave
+#define raw_readyq_lock                 raw_rspin_lock
+#define raw_readyq_unlock_irqrestore    raw_rspin_unlock_irqrestore
+#define raw_readyq_unlock               raw_rspin_unlock
+#define raw_readyq_lock_init    raw_rspin_lock_init
+#else
+#define raw_readyq_spinlock_t   raw_spinlock_t
+#define raw_readyq_lock_irqsave raw_spin_lock_irqsave
+#define raw_readyq_lock                 raw_spin_lock
+#define raw_readyq_unlock_irqrestore    raw_spin_unlock_irqrestore
+#define raw_readyq_unlock               raw_spin_unlock
+#define raw_readyq_lock_init    raw_spin_lock_init
+#endif
+
 typedef struct _rt_domain {
         /* runnable rt tasks are in here */
-        raw_spinlock_t                  ready_lock;
+        raw_readyq_spinlock_t   ready_lock;
         struct bheap                    ready_queue;
 
         /* real-time tasks waiting for release are in here */
@@ -65,6 +85,7 @@ struct release_heap {
 };
 
 
+
 static inline struct task_struct* __next_ready(rt_domain_t* rt)
 {
         struct bheap_node *hn = bheap_peek(rt->order, &rt->ready_queue);
@@ -125,17 +146,17 @@ static inline void add_ready(rt_domain_t* rt, struct task_struct *new)
 {
         unsigned long flags;
         /* first we need the write lock for rt_ready_queue */
-        raw_spin_lock_irqsave(&rt->ready_lock, flags);
+        raw_readyq_lock_irqsave(&rt->ready_lock, flags);
         __add_ready(rt, new);
-        raw_spin_unlock_irqrestore(&rt->ready_lock, flags);
+        raw_readyq_unlock_irqrestore(&rt->ready_lock, flags);
 }
 
 static inline void merge_ready(rt_domain_t* rt, struct bheap* tasks)
 {
         unsigned long flags;
-        raw_spin_lock_irqsave(&rt->ready_lock, flags);
+        raw_readyq_lock_irqsave(&rt->ready_lock, flags);
         __merge_ready(rt, tasks);
-        raw_spin_unlock_irqrestore(&rt->ready_lock, flags);
+        raw_readyq_unlock_irqrestore(&rt->ready_lock, flags);
 }
 
 static inline struct task_struct* take_ready(rt_domain_t* rt)
@@ -143,9 +164,9 @@ static inline struct task_struct* take_ready(rt_domain_t* rt)
         unsigned long flags;
         struct task_struct* ret;
         /* first we need the write lock for rt_ready_queue */
-        raw_spin_lock_irqsave(&rt->ready_lock, flags);
+        raw_readyq_lock_irqsave(&rt->ready_lock, flags);
         ret = __take_ready(rt);
-        raw_spin_unlock_irqrestore(&rt->ready_lock, flags);
+        raw_readyq_unlock_irqrestore(&rt->ready_lock, flags);
         return ret;
 }
 
@@ -183,9 +204,9 @@ static inline int jobs_pending(rt_domain_t* rt)
         unsigned long flags;
         int ret;
         /* first we need the write lock for rt_ready_queue */
-        raw_spin_lock_irqsave(&rt->ready_lock, flags);
+        raw_readyq_lock_irqsave(&rt->ready_lock, flags);
         ret = !bheap_empty(&rt->ready_queue);
-        raw_spin_unlock_irqrestore(&rt->ready_lock, flags);
+        raw_readyq_unlock_irqrestore(&rt->ready_lock, flags);
         return ret;
 }
 
diff --git a/litmus/Kconfig b/litmus/Kconfig
index 437ce82f97e9..a1a6cc699348 100644
--- a/litmus/Kconfig
+++ b/litmus/Kconfig
@@ -12,6 +12,15 @@ config PLUGIN_CEDF
           On smaller platforms (e.g., ARM PB11MPCore), using C-EDF
           makes little sense since there aren't any shared caches.
 
+config RECURSIVE_READYQ_LOCK
+        bool "Recursive Ready Queue Lock"
+        default n
+        help
+          Protects ready queues with a raw recursive spinlock instead
+          of a normal raw spinlock.
+
+          If unsure, say No.
+
 config PLUGIN_PFAIR
         bool "PFAIR"
         depends on HIGH_RES_TIMERS && !NO_HZ
diff --git a/litmus/rt_domain.c b/litmus/rt_domain.c
index 4528c4362ecf..abf54ad5c2f2 100644
--- a/litmus/rt_domain.c
+++ b/litmus/rt_domain.c
@@ -286,7 +286,7 @@ void rt_domain_init(rt_domain_t *rt,
         for (i = 0; i < RELEASE_QUEUE_SLOTS; i++)
                 INIT_LIST_HEAD(&rt->release_queue.slot[i]);
 
-        raw_spin_lock_init(&rt->ready_lock);
+        raw_readyq_lock_init(&rt->ready_lock);
         raw_spin_lock_init(&rt->release_lock);
         raw_spin_lock_init(&rt->tobe_lock);
 
diff --git a/litmus/sched_cedf.c b/litmus/sched_cedf.c
index 2837f12953ea..e8de5dd18b7f 100644
--- a/litmus/sched_cedf.c
+++ b/litmus/sched_cedf.c
@@ -2288,7 +2288,10 @@ UNSUPPORTED_AFF_OBS:
         return err;
 }
 #endif
-#endif // CONFIG_LITMUS_AFFINITY_LOCKING
+
+
+
+#endif // CONFIG_LITMUS_NESTED_LOCKING
 
 
 #ifdef VERBOSE_INIT
diff --git a/litmus/sched_pfair.c b/litmus/sched_pfair.c
index c06326faf9ce..707e6b6f2483 100644
--- a/litmus/sched_pfair.c
+++ b/litmus/sched_pfair.c
@@ -120,13 +120,13 @@ static inline struct pfair_cluster* from_domain(rt_domain_t* rt)
         return container_of(rt, struct pfair_cluster, pfair);
 }
 
-static inline raw_spinlock_t* cluster_lock(struct pfair_cluster* cluster)
+static inline raw_readyq_spinlock_t* cluster_lock(struct pfair_cluster* cluster)
 {
         /* The ready_lock is used to serialize all scheduling events. */
         return &cluster->pfair.ready_lock;
 }
 
-static inline raw_spinlock_t* cpu_lock(struct pfair_state* state)
+static inline raw_readyq_spinlock_t* cpu_lock(struct pfair_state* state)
 {
         return cluster_lock(cpu_cluster(state));
 }
@@ -452,11 +452,12 @@ static void schedule_next_quantum(struct pfair_cluster *cluster, quanta_t time)
 {
         struct pfair_state *cpu;
         struct list_head* pos;
+        raw_readyq_spinlock_t* readyq_lock = cluster_lock(cluster);
 
         /* called with interrupts disabled */
         PTRACE("--- Q %lu at %llu PRE-SPIN\n",
                time, litmus_clock());
-        raw_spin_lock(cluster_lock(cluster));
+        raw_readyq_lock(readyq_lock);
         PTRACE("<<< Q %lu at %llu\n",
                time, litmus_clock());
 
@@ -491,7 +492,7 @@ static void schedule_next_quantum(struct pfair_cluster *cluster, quanta_t time)
         }
         PTRACE(">>> Q %lu at %llu\n",
                time, litmus_clock());
-        raw_spin_unlock(cluster_lock(cluster));
+        raw_readyq_unlock(readyq_lock);
 }
 
 static noinline void wait_for_quantum(quanta_t q, struct pfair_state* state)
@@ -607,6 +608,7 @@ static struct task_struct* pfair_schedule(struct task_struct * prev)
         struct pfair_cluster* cluster = cpu_cluster(state);
         int blocks, completion, out_of_time;
         struct task_struct* next = NULL;
+        raw_readyq_spinlock_t* readyq_lock;
 
 #ifdef CONFIG_RELEASE_MASTER
         /* Bail out early if we are the release master.
@@ -618,7 +620,8 @@ static struct task_struct* pfair_schedule(struct task_struct * prev)
         }
 #endif
 
-        raw_spin_lock(cpu_lock(state));
+        readyq_lock = cpu_lock(state);
+        raw_readyq_lock(readyq_lock);
 
         blocks      = is_realtime(prev) && !is_running(prev);
         completion  = is_realtime(prev) && is_completed(prev);
@@ -650,7 +653,7 @@ static struct task_struct* pfair_schedule(struct task_struct * prev)
                         tsk_rt(next)->scheduled_on = cpu_id(state);
         }
         sched_state_task_picked();
-        raw_spin_unlock(cpu_lock(state));
+        raw_readyq_unlock(readyq_lock);
 
         if (next)
                 TRACE_TASK(next, "scheduled rel=%lu at %lu (%llu)\n",
@@ -665,12 +668,14 @@ static void pfair_task_new(struct task_struct * t, int on_rq, int running)
 {
         unsigned long flags;
         struct pfair_cluster* cluster;
+        raw_readyq_spinlock_t* readyq_lock;
 
         TRACE("pfair: task new %d state:%d\n", t->pid, t->state);
 
         cluster = tsk_pfair(t)->cluster;
+        readyq_lock = cluster_lock(cluster);
 
-        raw_spin_lock_irqsave(cluster_lock(cluster), flags);
+        raw_readyq_lock_irqsave(readyq_lock, flags);
 
         prepare_release(t, cluster->pfair_time + 1);
 
@@ -686,7 +691,7 @@ static void pfair_task_new(struct task_struct * t, int on_rq, int running)
 
         check_preempt(t);
 
-        raw_spin_unlock_irqrestore(cluster_lock(cluster), flags);
+        raw_readyq_unlock_irqrestore(readyq_lock, flags);
 }
 
 static void pfair_task_wake_up(struct task_struct *t)
@@ -695,13 +700,15 @@ static void pfair_task_wake_up(struct task_struct *t)
         lt_t now;
         int requeue = 0;
         struct pfair_cluster* cluster;
+        raw_readyq_spinlock_t* readyq_lock;
 
         cluster = tsk_pfair(t)->cluster;
+        readyq_lock = cluster_lock(cluster);
 
         TRACE_TASK(t, "wakes at %llu, release=%lu, pfair_time:%lu\n",
                    litmus_clock(), cur_release(t), cluster->pfair_time);
 
-        raw_spin_lock_irqsave(cluster_lock(cluster), flags);
+        raw_readyq_lock_irqsave(readyq_lock, flags);
 
         /* If a task blocks and wakes before its next job release,
          * then it may resume if it is currently linked somewhere
@@ -726,7 +733,7 @@ static void pfair_task_wake_up(struct task_struct *t)
 
         check_preempt(t);
 
-        raw_spin_unlock_irqrestore(cluster_lock(cluster), flags);
+        raw_readyq_unlock_irqrestore(readyq_lock, flags);
         TRACE_TASK(t, "wake up done at %llu\n", litmus_clock());
 }
 
@@ -741,10 +748,12 @@ static void pfair_task_exit(struct task_struct * t)
 {
         unsigned long flags;
         struct pfair_cluster *cluster;
+        raw_readyq_spinlock_t* readyq_lock;
 
         BUG_ON(!is_realtime(t));
 
         cluster = tsk_pfair(t)->cluster;
+        readyq_lock = cluster_lock(cluster);
 
         /* Remote task from release or ready queue, and ensure
          * that it is not the scheduled task for ANY CPU. We
@@ -753,12 +762,12 @@ static void pfair_task_exit(struct task_struct * t)
          * might not be the same as the CPU that the PFAIR scheduler
          * has chosen for it.
          */
-        raw_spin_lock_irqsave(cluster_lock(cluster), flags);
+        raw_readyq_lock_irqsave(readyq_lock, flags);
 
         TRACE_TASK(t, "RIP, state:%d\n", t->state);
         drop_all_references(t);
 
-        raw_spin_unlock_irqrestore(cluster_lock(cluster), flags);
+        raw_readyq_unlock_irqrestore(readyq_lock, flags);
 
         kfree(t->rt_param.pfair);
         t->rt_param.pfair = NULL;
@@ -771,12 +780,14 @@ static void pfair_release_at(struct task_struct* task, lt_t start)
         quanta_t release;
 
         struct pfair_cluster *cluster;
+        raw_readyq_spinlock_t* readyq_lock;
 
         cluster = tsk_pfair(task)->cluster;
+        readyq_lock = cluster_lock(cluster);
 
         BUG_ON(!is_realtime(task));
 
-        raw_spin_lock_irqsave(cluster_lock(cluster), flags);
+        raw_readyq_lock_irqsave(readyq_lock, flags);
         release_at(task, start);
         release = time2quanta(start, CEIL);
 
@@ -786,7 +797,7 @@ static void pfair_release_at(struct task_struct* task, lt_t start)
         prepare_release(task, release);
         add_release(&cluster->pfair, task);
 
-        raw_spin_unlock_irqrestore(cluster_lock(cluster), flags);
+        raw_readyq_unlock_irqrestore(readyq_lock, flags);
 }
 
 static void init_subtask(struct subtask* sub, unsigned long i,