Removed ARM-specific hacks which disabled less common mixed-criticality features.wip-mc

1 files changed, 154 insertions, 91 deletions

diff --git a/litmus/sched_mc.c b/litmus/sched_mc.c
index 6edf86935a29..c8e50d30a483 100644
--- a/litmus/sched_mc.c
+++ b/litmus/sched_mc.c
@@ -30,7 +30,7 @@
 #include <litmus/dgl.h>
 #include <litmus/color.h>
 #include <litmus/way_tracker.h>
-#warning "MUST ADD CHECK FOR MAX WAYS"
+
 struct mc_signal {
         int                     update:1;
         int                     preempt:1;
@@ -207,6 +207,7 @@ static int cpu_lower_prio(struct bheap_node *a, struct bheap_node *b)
 static int mc_preempt_needed(struct domain *dom, struct task_struct* curr)
 {
         struct task_struct *next = dom->peek_ready(dom);
+
         if (!next || !curr) {
                 return next && !curr;
         } else {
@@ -223,10 +224,13 @@ static int mc_preempt_needed(struct domain *dom, struct task_struct* curr)
 static void update_crit_position(struct crit_entry *ce)
 {
         struct bheap *heap;
+
         if (is_global(ce->domain)) {
                 heap = domain_data(ce->domain)->heap;
+
                 BUG_ON(!heap);
                 BUG_ON(!bheap_node_in_heap(ce->node));
+
                 bheap_delete(cpu_lower_prio, heap, ce->node);
                 bheap_insert(cpu_lower_prio, heap, ce->node);
         }
@@ -239,6 +243,7 @@ static void update_crit_position(struct crit_entry *ce)
 static void fix_crit_position(struct crit_entry *ce)
 {
         struct server *server = &ce->server;
+
         if (is_global(ce->domain) && server->in_transit) {
                 server_state_change(server, server->state, 0);
                 update_crit_position(ce);
@@ -368,7 +373,7 @@ static void link_task_to_crit(struct crit_entry *ce,
                         server_state_change(ce_server, SS_ACTIVE, 0);
                 }
 
-                TRACE_MC_TASK(ce->server.linked, "Unlinking\n");
+                /* TRACE_MC_TASK(ce->server.linked, "Unlinking\n"); */
 
                 stop_crit(ce);
                 tsk_rt(ce->server.linked)->server.parent = 0;
@@ -552,9 +557,7 @@ static struct task_struct* preempt_crit(struct domain *dom, struct crit_entry *c
 
         /* Per-domain preemption */
         link_task_to_crit(ce, task);
-        /* if (old && can_requeue(old)) { */
-        /*      dom->requeue(dom, old); */
-        /* } */
+
         update_crit_position(ce);
 
         /* Preempt actual execution if this is a running task.
@@ -574,6 +577,7 @@ static struct task_struct* preempt_crit(struct domain *dom, struct crit_entry *c
 
 /**
  * update_crit_levels() - Update criticality entries for the new cpu state.
+ * Disables criticality levels lower than @entry's currenly linked task.
  * This should be called after a new task has been linked to @entry.
  * The caller must hold the @entry->lock, but this method will release it.
  */
@@ -585,6 +589,8 @@ static void update_crit_levels(struct cpu_entry *entry)
         struct task_struct *readmit[NUM_CRIT_LEVELS];
         enum crit_level level = entry_level(entry);
 
+        STRACE("Updating crit levels for cpu %d\n", entry->cpu);
+
         /* Remove lower priority tasks from the entry */
         for (i = level + 1; i < NUM_CRIT_LEVELS; i++) {
                 ce = &entry->crit_entries[i];
@@ -609,8 +615,10 @@ static void update_crit_levels(struct cpu_entry *entry)
                                 link_task_to_crit(ce, NULL);
                         }
                         TRACE_CRIT_ENTRY(ce, "Removing lower crit\n");
-                        server_state_change(server, SS_REMOVED, 1);
-
+                        server_state_change(server, SS_REMOVED,
+                                            is_global(ce->domain)?1:0);
+                } else {
+                        TRACE_CRIT_ENTRY(ce, "Already removed!\n");
                 }
         }
         /* Need to unlock so we can access domains */
@@ -669,21 +677,26 @@ static void check_global_preempt(struct domain *dom)
         }
 }
 
-static void check_partitioned_preempt(struct domain *dom)
+static void check_partitioned_preempt(struct cpu_entry *entry,
+                                      struct domain *dom)
 {
-        struct cpu_entry *entry;
-        struct crit_entry *ce;
+        struct crit_entry *ce = domain_data(dom)->crit_entry;
 
-        ce = domain_data(dom)->crit_entry;
-        entry = crit_cpu(ce);
+        /* Cache next task */
+        dom->peek_ready(dom);
+
+        raw_spin_lock(&entry->lock);
 
         if (ce->server.state == SS_REMOVED ||
             !mc_preempt_needed(dom, ce->server.linked)) {
-                return;
+                goto out_unlock;
         }
 
         entry->signal.preempt = 1;
         litmus_reschedule(entry->cpu);
+
+ out_unlock:
+        raw_spin_unlock(&entry->lock);
 }
 
 /**
@@ -701,12 +714,7 @@ static void check_for_preempt(struct domain *dom)
                 ce = domain_data(dom)->crit_entry;
                 entry = crit_cpu(ce);
 
-                /* Cache next task */
-                dom->peek_ready(dom);
-
-                raw_spin_lock(&entry->lock);
-                check_partitioned_preempt(dom);
-                raw_spin_unlock(&entry->lock);
+                check_partitioned_preempt(entry, dom);
         }
 }
 
@@ -798,25 +806,24 @@ static void job_completion(struct task_struct *task, int forced)
 
         if (lt_before(get_user_release(task), litmus_clock()) ||
             (release_server && tsk_rt(task)->completed)){
-                TRACE_TASK(task, "Executable task going back to running\n");
+                TRACE_MC_TASK(task, "Executable task going back to running\n");
                 tsk_rt(task)->completed = 0;
         }
 
         if (release_server || forced) {
-                /* TODO: Level A does this independently and should not */
-                if (release_server && CRIT_LEVEL_A != tsk_mc_crit(task)) {
-                        prepare_for_next_period(task);
-                }
+                prepare_for_next_period(task);
 
-                TRACE_TASK(task, "Is released: %d, now: %llu, rel: %llu\n",
-                           is_released(task, litmus_clock()), litmus_clock(),
-                           get_release(task));
+                TRACE_MC_TASK(task, "Is released: %d, now: %llu, rel: %llu\n",
+                              is_released(task, litmus_clock()), litmus_clock(),
+                              get_release(task));
 
                 /* Requeue non-blocking tasks */
                 if (is_running(task)) {
                         job_arrival(task);
                 }
         } else if (is_ghost(task)) {
+                BUG_ON(tsk_rt(task)->linked_on == NO_CPU);
+
                 entry = &per_cpu(cpus, tsk_rt(task)->linked_on);
                 ce = &entry->crit_entries[tsk_mc_crit(task)];
 
@@ -847,24 +854,27 @@ static enum hrtimer_restart mc_ghost_exhausted(struct hrtimer *timer)
 #endif
         struct task_struct *tmp = NULL;
         struct cpu_entry *entry = crit_cpu(ce);
-        TRACE("Firing here at %llu\n", litmus_clock());
-        TRACE_CRIT_ENTRY(ce, "For this\n");
+        int resched = 0;
+
+        TRACE_CRIT_ENTRY(ce, "Firing here at %llu\n", litmus_clock());
 
         raw_spin_lock(&entry->lock);
 
-        if (is_ghost(ce->server.linked)) {
+        if (ce->server.linked && is_ghost(ce->server.linked)) {
                 update_server_time(ce->server.linked);
                 if (budget_exhausted(ce->server.linked)) {
                         tmp = ce->server.linked;
                 }
         } else {
-                litmus_reschedule(crit_cpu(ce)->cpu);
+                resched = 1;
         }
 
         raw_spin_unlock(&entry->lock);
 
         if (tmp)
                 job_completion(tmp, 1);
+        else if (resched)
+                litmus_reschedule(entry->cpu);
 
 #ifndef CONFIG_MERGE_TIMERS
         return HRTIMER_NORESTART;
@@ -891,8 +901,6 @@ static lt_t __ce_timer_function(struct ce_dom_data *ce_data)
             ce->server.linked == ce_data->should_schedule)
         {
                 old_link = ce->server.linked;
-                link_task_to_crit(ce, NULL);
-                mc_ce_job_completion(dom, old_link);
         }
         raw_spin_unlock(&crit_cpu(ce)->lock);
 
@@ -900,7 +908,7 @@ static lt_t __ce_timer_function(struct ce_dom_data *ce_data)
 
         /* Job completion will check for preemptions by means of calling job
          * arrival if the task is not blocked */
-        if (NULL != old_link) {
+        if (old_link) {
                 STRACE("old_link " TS " so will call job completion\n", TA(old_link));
                 raw_spin_unlock(dom->lock);
                 job_completion(old_link, 1);
@@ -993,8 +1001,10 @@ static void mc_task_new(struct task_struct *t, int on_rq, int running)
                 entry = &per_cpu(cpus, task_cpu(t));
         t->rt_param._domain = entry->crit_entries[level].domain;
 
+#ifdef CONFIG_SCHED_TASK_TRACE
         tsk_rt(t)->flush = 0;
         tsk_rt(t)->load  = 0;
+#endif
 
         /* Userspace and kernelspace view of task state may differ.
          * Model kernel state as a budget enforced container
@@ -1098,16 +1108,17 @@ static void mc_task_exit(struct task_struct *task)
                 color_sched_out_task(task);
         }
 
+        /* TODO: restore. This was geting triggered by race conditions even when
+         * no level-A task was executing */
+        if (CRIT_LEVEL_A == tsk_mc_crit(task))
+                mc_ce_task_exit_common(task);
+
         remove_from_all(task);
         if (tsk_rt(task)->scheduled_on != NO_CPU) {
                 per_cpu(cpus, tsk_rt(task)->scheduled_on).scheduled = NULL;
                 tsk_rt(task)->scheduled_on = NO_CPU;
         }
 
-        /* TODO: restore. This was geting triggered by race conditions even when
-         * no level-A task was executing */
-        /* if (CRIT_LEVEL_A == tsk_mc_crit(task)) */
-        /*      mc_ce_task_exit_common(task); */
 
         local_irq_restore(flags);
 }
@@ -1259,30 +1270,30 @@ static void process_update_signal(struct cpu_entry *entry)
                 stop_crit(ce);
                 server_state_change(crit_server, SS_BLOCKED, 0);
         }
-
-
 }
 
-static void process_signals(struct cpu_entry *entry)
+static void process_preempt_signal(struct cpu_entry *entry)
 {
+        int i;
         struct domain *dom;
         struct crit_entry *ce;
-        struct mc_signal signal;
-        struct task_struct *preempted;
+        struct task_struct *preempted = NULL;
+        struct server *server;
 
-        ce  = &entry->crit_entries[CRIT_LEVEL_B];
-        dom = ce->domain;
+        STRACE("Reading preempt signal\n");
 
-        /* Load signals */
-        raw_spin_lock(&entry->signal_lock);
-        signal = entry->signal;
-        clear_signal(&entry->signal);
-        raw_spin_unlock(&entry->signal_lock);
+        for (i = 0; i < NUM_CRIT_LEVELS; i++) {
+                ce = &entry->crit_entries[i];
+                dom = ce->domain;
+                server = &ce->server;
+                preempted = NULL;
 
-        if (signal.preempt) {
+                /* Swap locks. We cannot acquire a domain lock while
+                 * holding an entry lock or deadlocks will happen
+                 */
                 raw_spin_lock(dom->lock);
-                /* A higher-priority task may exist */
-                STRACE("Reading preempt signal\n");
+
+                /* Do domain stuff before grabbing CPU locks */
                 dom->peek_ready(dom);
 
                 raw_spin_lock(&entry->lock);
@@ -1308,26 +1319,106 @@ static void process_signals(struct cpu_entry *entry)
                                 raw_spin_unlock(dom->lock);
                         }
 
-                        raw_spin_lock(&entry->lock);
+                        break;
                 } else {
+                        raw_spin_unlock(&entry->lock);
                         raw_spin_unlock(dom->lock);
                 }
-        } else {
-                raw_spin_lock(&entry->lock);
+        }
+}
+
+static void process_signals(struct cpu_entry *entry)
+{
+        struct mc_signal signal;
+
+        /* Load signals */
+        raw_spin_lock(&entry->signal_lock);
+        signal = entry->signal;
+        clear_signal(&entry->signal);
+        raw_spin_unlock(&entry->signal_lock);
+
+        if (signal.preempt) {
+                process_preempt_signal(entry);
         }
 
+        raw_spin_lock(&entry->lock);
+
         if (signal.update) {
                 process_update_signal(entry);
         }
 }
 
+static void reschedule_if_signaled(struct cpu_entry *entry)
+{
+        struct mc_signal signal;
+
+        raw_spin_lock(&entry->signal_lock);
+        signal = entry->signal;
+        raw_spin_unlock(&entry->signal_lock);
+
+        if (signal.update || signal.preempt) {
+                litmus_reschedule_local();
+        }
+}
+
+static void pre_schedule(struct task_struct *prev)
+{
+        lt_t exec, start = litmus_clock();
+
+        /* Update userspace exec time */
+        if (prev && tsk_rt(prev)->last_exec_time) {
+                exec = start - tsk_rt(prev)->last_exec_time;
+                tsk_rt(prev)->user_job.exec_time += exec;
+        }
+
+        /* Flush task pages */
+        if (prev && tsk_mc_crit(prev) == CRIT_LEVEL_B &&
+            is_realtime(prev) && get_rt_job(prev) > 1 && lock_cache) {
+                color_sched_out_task(prev);
+
+#ifdef CONFIG_SCHED_TASK_TRACE
+                tsk_rt(prev)->load += litmus_clock() - start;
+#endif
+        }
+
+        TS_LVLA_SCHED_START;
+        TS_LVLB_SCHED_START;
+        TS_LVLC_SCHED_START;
+}
+
+static void post_schedule(struct task_struct *next)
+{
+        lt_t start;
+
+        switch (tsk_mc_crit(next)) {
+        case CRIT_LEVEL_A: TS_LVLA_SCHED_END(next); break;
+        case CRIT_LEVEL_B: TS_LVLB_SCHED_END(next); break;
+        case CRIT_LEVEL_C: TS_LVLC_SCHED_END(next); break;
+        }
+
+        /* Cache in task pages */
+        if (tsk_mc_crit(next) == CRIT_LEVEL_B && lock_cache &&
+            get_rt_job(next) > 1) {
+                start = litmus_clock();
+
+                color_sched_in_task(next);
+
+#ifdef CONFIG_SCHED_TASK_TRACE
+                BUG_ON(tsk_rt(next)->load);
+                tsk_rt(next)->load = litmus_clock() - start;
+#endif
+        }
+
+        tsk_rt(next)->last_exec_time = litmus_clock();
+}
+
 /**
  * mc_schedule() - Return next task which should be scheduled.
  */
 static struct task_struct* mc_schedule(struct task_struct* prev)
 {
-        lt_t start, exec;
-        int out_of_time, sleep, preempt, exists, blocks, global, lower, work;
+
+        int out_of_time, sleep, preempt, exists, blocks, global, lower;
         struct cpu_entry* entry = &__get_cpu_var(cpus);
         struct task_struct *next = NULL;
 
@@ -1340,22 +1431,7 @@ static struct task_struct* mc_schedule(struct task_struct* prev)
                 low_prio_arrival(entry->will_schedule);
         }
 
-        if (prev && tsk_rt(prev)->last_exec_time) {
-                exec = litmus_clock() - tsk_rt(prev)->last_exec_time;
-                tsk_rt(prev)->user_job.exec_time += exec;
-        }
-
-        if (prev && tsk_mc_crit(prev) == CRIT_LEVEL_B &&
-            is_realtime(prev) && get_rt_job(prev) > 1 && lock_cache) {
-                start = litmus_clock();
-                work = color_sched_out_task(prev);
-                tsk_rt(prev)->flush = litmus_clock() - start;
-                ++tsk_rt(prev)->flush_work;
-        }
-
-        TS_LVLA_SCHED_START;
-        TS_LVLB_SCHED_START;
-        TS_LVLC_SCHED_START;
+        pre_schedule(prev);
 
         raw_spin_lock(&entry->lock);
 
@@ -1406,9 +1482,7 @@ static struct task_struct* mc_schedule(struct task_struct* prev)
                         job_arrival(entry->scheduled);
         }
 
-        /* TODO: move this down somehow */
-        sched_state_task_picked();
-
+        /* Acquires the entry lock */
         process_signals(entry);
 
         /* Pick next task if none is linked */
@@ -1424,23 +1498,12 @@ static struct task_struct* mc_schedule(struct task_struct* prev)
 
         raw_spin_unlock(&entry->lock);
 
-        if (next) {
-                switch (tsk_mc_crit(next)) {
-                case CRIT_LEVEL_A: TS_LVLA_SCHED_END(next); break;
-                case CRIT_LEVEL_B: TS_LVLB_SCHED_END(next); break;
-                case CRIT_LEVEL_C: TS_LVLC_SCHED_END(next); break;
-                }
-        }
+        sched_state_task_picked();
 
-        if (next && tsk_mc_crit(next) == CRIT_LEVEL_B && lock_cache && get_rt_job(next) > 1) {
-                start = litmus_clock();
-                work = color_sched_in_task(next);
-                tsk_rt(next)->load = litmus_clock() - start;
-                tsk_rt(next)->load_work = work;
-        }
+        reschedule_if_signaled(entry);
 
         if (next) {
-                tsk_rt(next)->last_exec_time = litmus_clock();
+                post_schedule(next);
                 TRACE_MC_TASK(next, "Picked this task\n");
         } else {
                 STRACE("CPU %d idles at %llu\n", entry->cpu, litmus_clock());