Allow for concurrent blocking / user-space job completions.

4 files changed, 51 insertions, 32 deletions

diff --git a/include/litmus/litmus.h b/include/litmus/litmus.h
index 93961d08efd..a78f7f94344 100644
--- a/include/litmus/litmus.h
+++ b/include/litmus/litmus.h
@@ -51,8 +51,9 @@ void litmus_exit_task(struct task_struct *tsk);
 #define get_user_deadline(t)    (tsk_rt(t)->user_job.deadline)
 
 /*      Realtime utility macros */
-#define get_rt_flags(t)         (tsk_rt(t)->flags)
-#define set_rt_flags(t,f)       (tsk_rt(t)->flags=(f))
+#define get_rt_flags(t)         (tsk_rt(t)->flags)
+#define set_rt_flags(t,f)       (tsk_rt(t)->flags|=(f))
+#define clear_rt_flags(t,f)     (tsk_rt(t)->flags&=(~f))
 #define is_priority_boosted(t)  (tsk_rt(t)->priority_boosted)
 #define get_boost_start(t)  (tsk_rt(t)->boost_start_time)
 
diff --git a/include/litmus/sched_mc.h b/include/litmus/sched_mc.h
index 5473ee30cb5..327f476aad7 100644
--- a/include/litmus/sched_mc.h
+++ b/include/litmus/sched_mc.h
@@ -28,14 +28,14 @@ struct mc_data {
 #define tsk_mc_data(t)   (tsk_rt(t)->mc_data)
 #define tsk_mc_crit(t) \
         (tsk_mc_data(t) ? tsk_mc_data(t)->mc_task.crit : CRIT_LEVEL_C)
-#define is_ghost(t)                                             \
-        ((get_rt_flags(t) & RT_F_BLOCKED) ||                            \
-         (get_rt_job(t) < get_user_job(t) && (get_rt_flags(t) & RT_F_SLEEP)))
+#define is_cblocked(t) (get_rt_flags(t) & RT_F_BLOCKED)
+#define is_ghost(t)                                                     \
+        (get_rt_job(t) < get_user_job(t) && (get_rt_flags(t) & RT_F_SLEEP))
 #define ce_sid(ce) \
         (-((ce)->level * num_online_cpus() + crit_cpu(ce)->cpu + 1))
 
 #define TS "(%s/%d:%d:%d:%s)"
-#define TA(t) (t) ? tsk_mc_data(t) ? is_ghost(t) ? "ghost" : t->comm \
+#define TA(t) (t) ? tsk_mc_data(t) ? is_ghost(t) ? "ghost" : is_cblocked(t) ? "blocked" : t->comm \
                                                  : t->comm : "NULL",    \
                 (t) ? t->pid : 1,                                       \
                 (t) ? get_rt_job(t) : 1,                                \
diff --git a/litmus/rt_domain.c b/litmus/rt_domain.c
index 687dc129bc2..6e6a1236bb2 100644
--- a/litmus/rt_domain.c
+++ b/litmus/rt_domain.c
@@ -25,7 +25,7 @@
 #define TRACE(fmt, args...) STRACE(fmt, ## args)
 
 /* Uncomment when debugging timer races... */
-#if 0
+#if 1
 #define VTRACE_TASK TRACE_TASK
 #define VTRACE TRACE
 #else
@@ -440,6 +440,7 @@ static void pd_requeue(domain_t *dom, struct task_struct *task)
                       task->comm, task->pid, get_exec_cost(task), get_rt_period(task),
                       get_release(task), litmus_clock());
         } else {
+                tsk_rt(task)->flags = RT_F_RUNNING;
                 /* task has to wait for next release */
                 VTRACE_TASK(task, "add release(), rel=%llu\n", get_release(task));
                 add_release(domain, task);
diff --git a/litmus/sched_mc.c b/litmus/sched_mc.c
index 33adc860b6b..0ee7be46f1e 100644
--- a/litmus/sched_mc.c
+++ b/litmus/sched_mc.c
@@ -123,6 +123,10 @@ static void release_resources(struct task_struct *t)
                 TRACE_MC_TASK(t, "Releasing resources\n");
 
                 raw_spin_lock(&dgl_lock);
+                if (get_rt_flags(t) & RT_F_BLOCKED) {
+                        TRACE_MC_TASK(t, "Forced off before we got our stuff\n");
+                        clear_rt_flags(t, RT_F_BLOCKED);
+                }
                 remove_group_req(&group_lock, tsk_rt(t)->req);
                 raw_spin_unlock(&dgl_lock);
                 take_np(t);
@@ -312,6 +316,8 @@ static void link_task_to_crit(struct crit_entry *ce,
         /* Unlink last task */
         if (ce->linked) {
                 ce->domain->release_resources(ce->linked);
+                if (ce->state == CS_BLOCKED)
+                        ce->state == CS_ACTIVE;
 
                 TRACE_MC_TASK(ce->linked, "Unlinking\n");
                 ce->linked->rt_param.linked_on = NO_CPU;
@@ -439,7 +445,7 @@ static void link_task_to_cpu(struct cpu_entry *entry, struct task_struct *task)
         struct crit_entry *ce;
         TRACE_MC_TASK(task, "Linking to P%d\n", entry->cpu);
         BUG_ON(task && tsk_rt(task)->linked_on != entry->cpu);
-        BUG_ON(task && is_ghost(task));
+        BUG_ON(task && (is_ghost(task) || is_cblocked(task)));
 
         if (entry->linked) {
                 sched_trace_server_switch_away(-entry->linked->pid,
@@ -449,7 +455,7 @@ static void link_task_to_cpu(struct cpu_entry *entry, struct task_struct *task)
         }
 
         if (task) {
-                set_rt_flags(task, RT_F_RUNNING);
+                tsk_rt(task)->flags = RT_F_RUNNING;
                 sched_trace_server_switch_to(-task->pid,
                                              get_rt_job(task),
                                              task->pid,
@@ -500,7 +506,7 @@ static void preempt_crit(struct domain *dom, struct crit_entry *ce)
          * running on this CPU as otherwise the crit_entry would have
          * been disabled and a preemption could not have occurred
          */
-        if (!is_ghost(task)) {
+        if (!is_ghost(task) && !is_cblocked(task)) {
                 preempt_cpu(entry, task);
         } else if (old && old == entry->linked) {
                 /* Preempted running task with ghost job. Nothing should run */
@@ -563,7 +569,7 @@ static void update_crit_levels(struct cpu_entry *entry)
  */
 static void check_for_preempt(struct domain *dom)
 {
-        int recheck = 1, higher_prio, was_ghost;
+        int recheck = 1, higher_prio, was_inelig, update = 0;
         struct cpu_entry *entry;
         struct crit_entry *ce;
 
@@ -599,22 +605,24 @@ static void check_for_preempt(struct domain *dom)
                 raw_spin_lock(&entry->lock);
 
                 if (can_use(ce)) {
-                        was_ghost = ce->linked && !is_ghost(ce->linked) &&
-                                ce->linked != entry->linked;
+                        was_inelig = ce->linked && !is_ghost(ce->linked) &&
+                                ce->linked != entry->linked && !is_cblocked(ce->linked);
                         higher_prio = mc_preempt_needed(dom, ce->linked);
 
-                        if (was_ghost) {
+                        if (was_inelig) {
                                 preempt_cpu(entry, ce->linked);
                                 start_crit(ce);
                         } else if (higher_prio)
                                 preempt_crit(dom, ce);
+                        update = was_inelig || higher_prio;
 
-                        if (was_ghost || higher_prio) {
-                                update_crit_levels(entry);
-                                return;
-                        }
+                } else {
+                        TRACE_CRIT_ENTRY(ce, "Can't use!\n");
                 }
-                raw_spin_unlock(&entry->lock);
+                if (update)
+                        update_crit_levels(entry);
+                else
+                        raw_spin_unlock(&entry->lock);
         }
 }
 
@@ -695,22 +703,25 @@ static void job_completion(struct task_struct *task, int forced)
         release_server = budget_exhausted(task);
 
         if (release_server || forced) {
+                if (release_server)
+                        sched_trace_server_completion(-task->pid,
+                                                      get_rt_job(task));
                 /* Only unlink (and release resources) if the current server job
                  * must stop logically running
                  */
                 remove_from_all(task);
         }
 
+        BUG_ON(get_rt_flags(task) & RT_F_BLOCKED);
+
         if (lt_before(get_user_release(task), litmus_clock())) {
                 TRACE_TASK(task, "Executable task going back to running\n");
-                set_rt_flags(task, RT_F_RUNNING);
+                clear_rt_flags(task, RT_F_SLEEP);
         }
 
         if (release_server || forced) {
                 /* TODO: Level A does this independently and should not */
                 if (release_server && CRIT_LEVEL_A != tsk_mc_crit(task)) {
-                        sched_trace_server_completion(-task->pid,
-                                                      get_rt_job(task));
                         prepare_for_next_period(task);
                 }
 
@@ -902,7 +913,7 @@ static void mc_task_new(struct task_struct *t, int on_rq, int running)
         tsk_rt(t)->task_params.budget_policy = PRECISE_ENFORCEMENT;
 
         /* Apply chunking */
-        if (level == CRIT_LEVEL_B && color_chunk
+        if (level == CRIT_LEVEL_B && color_chunk &&
             lt_after(get_exec_cost(t), color_chunk)) {
                 tsk_rt(t)->orig_cost = get_exec_cost(t);
         }
@@ -923,6 +934,7 @@ static void mc_task_new(struct task_struct *t, int on_rq, int running)
         release_at(t, litmus_clock());
         if (running) {
                 BUG_ON(entry->scheduled);
+                TRACE_MC_TASK(t, "Was already running\n");
                 entry->scheduled = t;
                 tsk_rt(t)->scheduled_on = entry->cpu;
                 tsk_rt(t)->last_exec_time = litmus_clock();
@@ -951,7 +963,11 @@ static void mc_task_wake_up(struct task_struct *task)
                 release_at(task, now);
                 sched_trace_task_release(task);
         }
-        if (!is_ghost(task))
+
+        if (budget_exhausted(task))
+                /* Rare, but possible, race condition */
+                job_completion(task, 1);
+        else
                 job_arrival(task);
 
         local_irq_restore(flags);
@@ -1064,7 +1080,8 @@ void pick_next_task(struct cpu_entry *entry)
                                 update_crit_position(ce);
                         }
                 }
-                if (ready_task && !is_ghost(ready_task)) {
+                if (ready_task && !is_ghost(ready_task) &&
+                    !is_cblocked(ready_task)) {
                         link_task_to_cpu(entry, ready_task);
                         raw_spin_unlock(dom->lock);
                         update_crit_levels(entry);
@@ -1112,7 +1129,7 @@ static struct task_struct* mc_schedule(struct task_struct* prev)
         exists      = entry->scheduled != NULL;
         blocks      = exists && !is_running(entry->scheduled);
         out_of_time = exists && budget_exhausted(entry->scheduled);
-        sleep       = exists && get_rt_flags(entry->scheduled) == RT_F_SLEEP;
+        sleep       = exists && get_rt_flags(entry->scheduled) & RT_F_SLEEP;
         global      = exists && is_global_task(entry->scheduled);
         preempt     = entry->scheduled != entry->linked;
         lower       = exists && preempt && entry->linked &&
@@ -1155,9 +1172,10 @@ static struct task_struct* mc_schedule(struct task_struct* prev)
         /* A remote processor unblocked one of our crit levels */
         if (entry->crit_signal != NUM_CRIT_LEVELS) {
                 ce = &entry->crit_entries[entry->crit_signal];
-                raw_spin_lock(&ce->domain->lock);
+                TRACE_CRIT_ENTRY(ce, "Processing signal for %d\n", entry->crit_signal);
+                raw_spin_lock(ce->domain->lock);
                 check_for_preempt(ce->domain);
-                raw_spin_unlock(&ce->domain->lock);
+                raw_spin_unlock(ce->domain->lock);
                 entry->crit_signal = NUM_CRIT_LEVELS;
         }
 
@@ -1414,15 +1432,14 @@ static void cpu_acquired(int cpu)
         TRACE_CRIT_ENTRY(ce, "Acquired lock\n");
 
         BUG_ON(!ce->linked);
-        BUG_ON(get_rt_flags(ce->linked) & RT_F_SLEEP);
 
-        set_rt_flags(ce->linked, RT_F_RUNNING);
+        clear_rt_flags(ce->linked, RT_F_BLOCKED);
         sched_trace_task_resume(ce->linked);
 
         if (ce->state == CS_BLOCKED) {
                 entry->crit_signal = CRIT_LEVEL_B;
-                /* Yes this is ok for race conditions, but only because no other
-                 * state will ever apply to a partitioned crit entry
+                /* Yes this is ok for race conditions, but only in the system
+                 * for the MC-COLOR paper
                  */
                 ce->state = CS_ACTIVE;
                 litmus_reschedule(cpu);