Merged with ce domains

1 files changed, 153 insertions, 71 deletions

diff --git a/litmus/sched_mc.c b/litmus/sched_mc.c
index 30898246ea38..7b74958d1f4f 100644
--- a/litmus/sched_mc.c
+++ b/litmus/sched_mc.c
@@ -25,28 +25,7 @@
 #include <litmus/event_group.h>
 
 #include <litmus/sched_mc.h>
-
-/**
- * crit_entry_t - State of a CPU within each criticality level system.
- * @level       Criticality level of this entry
- * @linked      Logically running task, ghost or regular
- * @domain      Domain from which to draw tasks
- * @usable      False if a higher criticality task is running
- * @timer       For ghost task budget enforcement
- * @node        Used to sort crit_entries by preemptability in global domains
- */
-struct crit_entry {
-        enum crit_level         level;
-        struct task_struct*     linked;
-        struct domain*          domain;
-        int                     usable;
-#ifdef CONFIG_MERGE_TIMERS
-        struct rt_event         event;
-#else
-        struct hrtimer          timer;
-#endif
-        struct bheap_node*      node;
-};
+#include <litmus/ce_domain.h>
 
 /**
  * struct cpu_entry - State of a CPU for the entire MC system
@@ -71,18 +50,6 @@ struct cpu_entry {
 #endif
 };
 
-/**
- * struct domain_data - Wrap domains with related CPU state
- * @domain      A domain for a criticality level
- * @heap        The preemptable heap of crit entries (for global domains)
- * @crit_entry  The crit entry for this domain (for partitioned domains)
- */
-struct domain_data {
-        struct domain           domain;
-        struct bheap*           heap;
-        struct crit_entry*      crit_entry;
-};
-
 DEFINE_PER_CPU(struct cpu_entry, cpus);
 #ifdef CONFIG_RELEASE_MASTER
 static int interrupt_cpu;
@@ -101,9 +68,9 @@ static struct event_group* global_group;
 #define crit_cpu(ce) \
         (container_of((void*)((ce) - (ce)->level), struct cpu_entry, crit_entries))
 #define TRACE_ENTRY(e, fmt, args...)                            \
-        TRACE("P%d, linked=" TS " " fmt "\n", e->cpu, TA(e->linked), ##args)
+        STRACE("P%d, linked=" TS " " fmt "\n", e->cpu, TA(e->linked), ##args)
 #define TRACE_CRIT_ENTRY(ce, fmt, args...)                      \
-        TRACE("%s P%d, linked=" TS " " fmt "\n",                \
+        STRACE("%s P%d, linked=" TS " " fmt "\n",                       \
               (ce)->domain->name, crit_cpu(ce)->cpu, TA((ce)->linked), ##args)
 
 /*
@@ -162,7 +129,7 @@ static inline struct crit_entry* lowest_prio_cpu(struct domain *dom)
 static inline void cancel_ghost(struct crit_entry *ce)
 {
 #ifdef CONFIG_MERGE_TIMERS
-        cancel_event(&ce->event);
+        cancel_event(crit_cpu(ce)->event_group, &ce->event);
 #else
         hrtimer_try_to_cancel(&ce->timer);
 #endif
@@ -174,9 +141,7 @@ static inline void cancel_ghost(struct crit_entry *ce)
 static inline void arm_ghost(struct crit_entry *ce, lt_t fire)
 {
 #ifdef CONFIG_MERGE_TIMERS
-        struct event_group* group = (is_global(ce->domain)) ?
-                global_group : crit_cpu(ce)->event_group;
-        add_event(group, &ce->event, fire);
+        add_event(crit_cpu(ce)->event_group, &ce->event, fire);
 #else
         __hrtimer_start_range_ns(&ce->timer,
                                  ns_to_ktime(when_to_fire),
@@ -243,10 +208,14 @@ static void link_task_to_crit(struct crit_entry *ce,
         ce->linked = task;
         if (task) {
                 task->rt_param.linked_on = crit_cpu(ce)->cpu;
-                if (is_ghost(task)) {
+                if (is_ghost(task) && CRIT_LEVEL_A != tsk_mc_crit(task)) {
+                        /* There is a level-A timer that will force a
+                         * preemption, so we don't set this for level-A
+                         * tasks.
+                         */
                         /* Reset budget timer */
                         task->se.exec_start = litmus_clock();
-                        when_to_fire = litmus_clock() +
+                        when_to_fire = task->se.exec_start +
                                 tsk_mc_data(task)->mc_job.ghost_budget;
                         arm_ghost(ce, when_to_fire);
                 }
@@ -261,6 +230,7 @@ static void link_task_to_crit(struct crit_entry *ce,
 }
 
 static void check_for_preempt(struct domain*);
+
 /**
  * job_arrival() - Called when a task re-enters the system.
  * Caller must hold no locks.
@@ -330,7 +300,7 @@ static void fix_global_levels(void)
         struct list_head *pos, *safe;
         struct task_struct *t;
 
-        TRACE("Fixing global levels\n");
+        STRACE("Fixing global levels");
         for_each_online_cpu(c) {
                 e = &per_cpu(cpus, c);
                 raw_spin_lock(&e->redir_lock);
@@ -527,6 +497,7 @@ static void remove_from_all(struct task_struct* task)
  */
 static void job_completion(struct task_struct *task, int forced)
 {
+        lt_t now;
         TRACE_MC_TASK(task, "Completed");
         sched_trace_task_completion(task, forced);
         BUG_ON(!task);
@@ -535,8 +506,11 @@ static void job_completion(struct task_struct *task, int forced)
         set_rt_flags(task, RT_F_SLEEP);
         remove_from_all(task);
 
+        now = litmus_clock();
+
         /* If it's not a ghost job, do ghost job conversion */
         if (!is_ghost(task)) {
+                TRACE_MC_TASK(task, "is not a ghost task");
                 tsk_mc_data(task)->mc_job.ghost_budget = budget_remaining(task);
                 tsk_mc_data(task)->mc_job.is_ghost = 1;
         }
@@ -546,6 +520,7 @@ static void job_completion(struct task_struct *task, int forced)
          * conversion. Revert back to a normal task and complete the period.
          */
         if (tsk_mc_data(task)->mc_job.ghost_budget == 0) {
+                TRACE_MC_TASK(task, "has zero ghost budget");
                 tsk_mc_data(task)->mc_job.is_ghost = 0;
                 prepare_for_next_period(task);
                 if (is_released(task, litmus_clock()))
@@ -573,9 +548,9 @@ static enum hrtimer_restart mc_ghost_exhausted(struct hrtimer *timer)
         unsigned long flags;
         struct task_struct *tmp = NULL;
 
-
         local_irq_save(flags);
-        TRACE_CRIT_ENTRY(ce, "Ghost exhausted firing");
+        TRACE("Ghost exhausted\n");
+        TRACE_CRIT_ENTRY(ce, "Firing here");
 
         /* Due to race conditions, we cannot just set the linked
          * task's budget to 0 as it may no longer be the task
@@ -601,6 +576,52 @@ static enum hrtimer_restart mc_ghost_exhausted(struct hrtimer *timer)
 #endif
 }
 
+static enum hrtimer_restart ce_timer_function(struct hrtimer *timer)
+{
+        struct ce_dom_data *ce_data =
+                container_of(timer, struct ce_dom_data, timer);
+        struct crit_entry *ce = &per_cpu(cpus, ce_data->cpu).crit_entries[CRIT_LEVEL_A];
+        struct domain *dom = ce->domain;
+        struct task_struct *old_link = NULL;
+        unsigned long flags;
+
+        TRACE("MC level-A timer callback for CPU %d\n", ce_data->cpu);
+
+        local_irq_save(flags);
+
+        raw_spin_lock(dom->lock);
+
+        raw_spin_lock(&crit_cpu(ce)->lock);
+        if (ce->linked &&
+            ce->linked == ce_data->should_schedule &&
+            is_ghost(ce->linked))
+        {
+                old_link = ce->linked;
+                tsk_mc_data(ce->linked)->mc_job.ghost_budget = 0;
+                link_task_to_crit(ce, NULL);
+        }
+        raw_spin_unlock(&crit_cpu(ce)->lock);
+
+        mc_ce_timer_callback_common(dom, timer);
+
+        /* job completion will check for preemptions by means of calling job
+         * arrival if the task is not blocked */
+        if (NULL != old_link) {
+                STRACE("old_link " TS " so will call job completion\n", TA(old_link));
+                raw_spin_unlock(dom->lock);
+                job_completion(old_link, 0);
+        } else {
+                STRACE("old_link was null, so will call check for preempt\n");
+                raw_spin_unlock(dom->lock);
+                check_for_preempt(dom);
+        }
+
+        local_irq_restore(flags);
+
+        return HRTIMER_RESTART;
+}
+
+
 /**
  * mc_release_jobs() - Add heap of tasks to the system, check for preemptions.
  */
@@ -611,7 +632,7 @@ static void mc_release_jobs(rt_domain_t* rt, struct bheap* tasks)
         struct domain *dom = get_task_domain(first);
 
         raw_spin_lock_irqsave(dom->lock, flags);
-        TRACE_MC_TASK(first, "Jobs released");
+        TRACE(TS "Jobs released\n", TA(first));
         __merge_ready(rt, tasks);
         check_for_preempt(dom);
         raw_spin_unlock_irqrestore(dom->lock, flags);
@@ -664,7 +685,7 @@ static void mc_task_wake_up(struct task_struct *task)
         lt_t now = litmus_clock();
         local_irq_save(flags);
 
-        TRACE_MC_TASK(task, "Wakes up");
+        TRACE(TS " wakes up\n", TA(task));
         if (is_tardy(task, now)) {
                 /* Task missed its last release */
                 release_at(task, now);
@@ -683,7 +704,7 @@ static void mc_task_block(struct task_struct *task)
 {
         unsigned long flags;
         local_irq_save(flags);
-        TRACE_MC_TASK(task, "Block at %llu", litmus_clock());
+        TRACE(TS " blocks\n", TA(task));
         remove_from_all(task);
         local_irq_restore(flags);
 }
@@ -696,7 +717,7 @@ static void mc_task_exit(struct task_struct *task)
         unsigned long flags;
         local_irq_save(flags);
         BUG_ON(!is_realtime(task));
-        TRACE_MC_TASK(task, "RIP");
+        TRACE(TS " RIP\n", TA(task));
 
         remove_from_all(task);
         if (tsk_rt(task)->scheduled_on != NO_CPU) {
@@ -704,6 +725,9 @@ static void mc_task_exit(struct task_struct *task)
                 tsk_rt(task)->scheduled_on = NO_CPU;
         }
 
+        if (CRIT_LEVEL_A == tsk_mc_crit(task))
+                mc_ce_task_exit_common(task);
+
         local_irq_restore(flags);
 }
 
@@ -713,19 +737,30 @@ static void mc_task_exit(struct task_struct *task)
  */
 static long mc_admit_task(struct task_struct* task)
 {
+        const enum crit_level crit = tsk_mc_crit(task);
+        long ret;
         if (!tsk_mc_data(task)) {
                 printk(KERN_WARNING "Tried to admit task with no criticality "
                         "level\n");
-                return -EINVAL;
+                ret = -EINVAL;
+                goto out;
         }
-        if (tsk_mc_crit(task) < CRIT_LEVEL_C && get_partition(task) == NO_CPU) {
+        if (crit < CRIT_LEVEL_C && get_partition(task) == NO_CPU) {
                 printk(KERN_WARNING "Tried to admit partitioned task with no "
                        "partition\n");
-                return -EINVAL;
+                ret = -EINVAL;
+                goto out;
+        }
+        if (crit == CRIT_LEVEL_A) {
+                ret = mc_ce_admit_task_common(task);
+                if (ret)
+                        goto out;
         }
         printk(KERN_INFO "Admitted task with criticality level %d\n",
                 tsk_mc_crit(task));
-        return 0;
+        ret = 0;
+out:
+        return ret;
 }
 
 /**
@@ -761,11 +796,11 @@ static struct task_struct* mc_schedule(struct task_struct * prev)
 
         if (exists) {
                 entry->scheduled->rt_param.scheduled_on = NO_CPU;
-                TRACE_MC_TASK(prev,
-                           "blocks:%d out_of_time:%d sleep:%d preempt:%d "
-                           "state:%d sig:%d global:%d",
-                           blocks, out_of_time, sleep, preempt,
-                           prev->state, signal_pending(prev), global);
+                TRACE(TS
+                      " blocks:%d out_of_time:%d sleep:%d preempt:%d "
+                      "state:%d sig:%d global:%d\n", TA(prev),
+                      blocks, out_of_time, sleep, preempt,
+                      prev->state, signal_pending(prev), global);
         }
         raw_spin_unlock(&entry->lock);
 
@@ -781,7 +816,7 @@ static struct task_struct* mc_schedule(struct task_struct * prev)
         /* Any task which exhausts its budget or sleeps waiting for its next
          * period completes unless its execution has been forcibly stopped.
          */
-        if ((out_of_time || sleep) && !blocks && !preempt)
+        if ((out_of_time || sleep) && !blocks)/* && !preempt)*/
                 job_completion(entry->scheduled, !sleep);
         /* Global scheduled tasks must wait for a deschedule before they
          * can rejoin the global state. Rejoin them here.
@@ -836,10 +871,29 @@ static struct task_struct* mc_schedule(struct task_struct * prev)
         if (next)
                 TRACE_MC_TASK(next, "Scheduled at %llu", litmus_clock());
         else if (exists && !next)
-                TRACE("Becomes idle at %llu\n", litmus_clock());
+                TRACE_ENTRY(entry, "Becomes idle at %llu", litmus_clock());
         return next;
 }
 
+/*
+ * This is the plugin's release at function, called by the release task-set
+ * system call. Other places in the file use the generic LITMUS release_at(),
+ * which is not this.
+ */
+void mc_release_at(struct task_struct *ts, lt_t start)
+{
+        /* hack so that we can have CE timers start at the right time */
+        if (CRIT_LEVEL_A == tsk_mc_crit(ts))
+                mc_ce_release_at_common(ts, start);
+        else
+                release_at(ts, start);
+}
+
+long mc_deactivate_plugin(void)
+{
+        return mc_ce_deactivate_plugin_common();
+}
+
 /* **************************************************************************
  * Initialization
  * ************************************************************************** */
@@ -850,7 +904,8 @@ static struct task_struct* mc_schedule(struct task_struct * prev)
 
 /* LVL-A */
 DEFINE_PER_CPU(struct domain_data, _mc_crit_a);
-DEFINE_PER_CPU(rt_domain_t, _mc_crit_a_rt);
+DEFINE_PER_CPU(raw_spinlock_t, _mc_crit_a_lock);
+DEFINE_PER_CPU(struct ce_dom_data, _mc_crit_a_ce_data);
 /* LVL-B */
 DEFINE_PER_CPU(struct domain_data, _mc_crit_b);
 DEFINE_PER_CPU(rt_domain_t, _mc_crit_b_rt);
@@ -870,12 +925,31 @@ DEFINE_PER_CPU(struct event_group, _mc_groups);
 
 static long mc_activate_plugin(void)
 {
+        struct domain_data *dom_data;
+        struct domain *dom;
+        struct domain_data *our_domains[NR_CPUS];
+        int cpu, n = 0;
+        long ret;
+
 #ifdef CONFIG_RELEASE_MASTER
         interrupt_cpu = atomic_read(&release_master_cpu);
         if (interrupt_cpu == NO_CPU)
                 interrupt_cpu = 0;
 #endif
-        return 0;
+
+        for_each_online_cpu(cpu) {
+                BUG_ON(NR_CPUS <= n);
+                dom = per_cpu(cpus, cpu).crit_entries[CRIT_LEVEL_A].domain;
+                dom_data = domain_data(dom);
+                our_domains[cpu] = dom_data;
+                n++;
+        }
+        ret = mc_ce_set_domains(n, our_domains);
+        if (ret)
+                goto out;
+        ret = mc_ce_activate_plugin_common();
+out:
+        return ret;
 }
 
 static struct sched_plugin mc_plugin __cacheline_aligned_in_smp = {
@@ -888,6 +962,8 @@ static struct sched_plugin mc_plugin __cacheline_aligned_in_smp = {
         .task_block             = mc_task_block,
         .admit_task             = mc_admit_task,
         .activate_plugin        = mc_activate_plugin,
+        .release_at             = mc_release_at,
+        .deactivate_plugin      = mc_deactivate_plugin,
 };
 
 static void init_crit_entry(struct crit_entry *ce, enum crit_level level,
@@ -958,13 +1034,15 @@ static inline void init_edf_domain(struct domain *dom, rt_domain_t *rt,
 #endif
 }
 
+struct domain_data *ce_domain_for(int);
 static int __init init_mc(void)
 {
         int cpu;
         struct cpu_entry *entry;
         struct domain_data *dom_data;
         rt_domain_t *rt;
-        raw_spinlock_t *a_dom, *b_dom, *c_dom; /* For lock debugger */
+        raw_spinlock_t *a_dom_lock, *b_dom_lock, *c_dom_lock; /* For lock debugger */
+        struct ce_dom_data *ce_data;
 
         for_each_online_cpu(cpu) {
                 entry = &per_cpu(cpus, cpu);
@@ -992,11 +1070,15 @@ static int __init init_mc(void)
 
                 /* CRIT_LEVEL_A */
                 dom_data = &per_cpu(_mc_crit_a, cpu);
-                rt = &per_cpu(_mc_crit_a_rt, cpu);
+                ce_data = &per_cpu(_mc_crit_a_ce_data, cpu);
+                a_dom_lock = &per_cpu(_mc_crit_a_lock, cpu);
+                raw_spin_lock_init(a_dom_lock);
+                ce_domain_init(&dom_data->domain,
+                                a_dom_lock, ce_requeue, ce_peek_and_take_ready,
+                                ce_peek_and_take_ready, mc_preempt_needed,
+                                ce_higher_prio, ce_data, cpu,
+                                ce_timer_function);
                 init_local_domain(entry, dom_data, CRIT_LEVEL_A);
-                init_edf_domain(&dom_data->domain, rt, cpu, CRIT_LEVEL_A);
-                a_dom = dom_data->domain.lock;
-                raw_spin_lock_init(a_dom);
                 dom_data->domain.name = "LVL-A";
 
                 /* CRIT_LEVEL_B */
@@ -1004,8 +1086,8 @@ static int __init init_mc(void)
                 rt = &per_cpu(_mc_crit_b_rt, cpu);
                 init_local_domain(entry, dom_data, CRIT_LEVEL_B);
                 init_edf_domain(&dom_data->domain, rt, cpu, CRIT_LEVEL_B);
-                b_dom = dom_data->domain.lock;
-                raw_spin_lock_init(b_dom);
+                b_dom_lock = dom_data->domain.lock;
+                raw_spin_lock_init(b_dom_lock);
                 dom_data->domain.name = "LVL-B";
         }
 
@@ -1022,8 +1104,8 @@ static int __init init_mc(void)
         init_global_domain(&_mc_crit_c, CRIT_LEVEL_C,
                            &_mc_heap_c, _mc_nodes_c);
         init_edf_domain(&_mc_crit_c.domain, &_mc_crit_c_rt, 0, CRIT_LEVEL_C);
-        c_dom = _mc_crit_c.domain.lock;
-        raw_spin_lock_init(c_dom);
+        c_dom_lock = _mc_crit_c.domain.lock;
+        raw_spin_lock_init(c_dom_lock);
         _mc_crit_c.domain.name = "LVL-C";
 
         return register_sched_plugin(&mc_plugin);