1 files changed, 75 insertions, 60 deletions

diff --git a/litmus/sched_mc.c b/litmus/sched_mc.c
index 7b74958d1f4f..17f84c9eba79 100644
--- a/litmus/sched_mc.c
+++ b/litmus/sched_mc.c
@@ -27,6 +27,8 @@
 #include <litmus/sched_mc.h>
 #include <litmus/ce_domain.h>
 
+/* XXX TODO Do we ever want to move level-A timers? */
+
 /**
  * struct cpu_entry - State of a CPU for the entire MC system
  * @cpu           CPU id
@@ -34,6 +36,9 @@
  * @linked        Task that should be running / is logically running
  * @lock          For serialization
  * @crit_entries  Array of CPU state per criticality level
+ * @redir         List of redirected work for this CPU.
+ * @redir_lock    Lock for @redir.
+ * @event_group   Event group for timer merging.
  */
 struct cpu_entry {
         int                     cpu;
@@ -45,18 +50,12 @@ struct cpu_entry {
         struct list_head        redir;
         raw_spinlock_t          redir_lock;
 #endif
-#ifdef CONFIG_MERGE_TIMERS
-        struct event_group*     event_group;
-#endif
 };
 
 DEFINE_PER_CPU(struct cpu_entry, cpus);
 #ifdef CONFIG_RELEASE_MASTER
 static int interrupt_cpu;
 #endif
-#ifdef CONFIG_MERGE_TIMERS
-static struct event_group* global_group;
-#endif
 
 #define domain_data(dom)  (container_of(dom, struct domain_data, domain))
 #define is_global(dom)    (domain_data(dom)->heap)
@@ -67,6 +66,7 @@ static struct event_group* global_group;
         (((e)->linked) ? tsk_mc_crit((e)->linked) : NUM_CRIT_LEVELS - 1)
 #define crit_cpu(ce) \
         (container_of((void*)((ce) - (ce)->level), struct cpu_entry, crit_entries))
+#define get_crit_entry_for(cpu, level) (&per_cpu(cpus, cpu).crit_entries[level])
 #define TRACE_ENTRY(e, fmt, args...)                            \
         STRACE("P%d, linked=" TS " " fmt "\n", e->cpu, TA(e->linked), ##args)
 #define TRACE_CRIT_ENTRY(ce, fmt, args...)                      \
@@ -129,7 +129,7 @@ static inline struct crit_entry* lowest_prio_cpu(struct domain *dom)
 static inline void cancel_ghost(struct crit_entry *ce)
 {
 #ifdef CONFIG_MERGE_TIMERS
-        cancel_event(crit_cpu(ce)->event_group, &ce->event);
+        cancel_event(&ce->event);
 #else
         hrtimer_try_to_cancel(&ce->timer);
 #endif
@@ -141,10 +141,10 @@ static inline void cancel_ghost(struct crit_entry *ce)
 static inline void arm_ghost(struct crit_entry *ce, lt_t fire)
 {
 #ifdef CONFIG_MERGE_TIMERS
-        add_event(crit_cpu(ce)->event_group, &ce->event, fire);
+        add_event(get_event_group_for(crit_cpu(ce)->cpu), &ce->event, fire);
 #else
         __hrtimer_start_range_ns(&ce->timer,
-                                 ns_to_ktime(when_to_fire),
+                                 ns_to_ktime(fire),
                                  0 /* delta */,
                                  HRTIMER_MODE_ABS_PINNED,
                                  0 /* no wakeup */);
@@ -270,10 +270,8 @@ static void low_prio_arrival(struct task_struct *task)
         if (!can_requeue(task)) return;
 
 #ifdef  CONFIG_PLUGIN_MC_REDIRECT
-#ifndef CONFIG_PLUGIN_MC_REDIRECT_ALL
         if (!is_global_task(task))
-            goto arrive;
-#endif
+                goto arrive;
         if (smp_processor_id() != interrupt_cpu) {
                 entry = &__get_cpu_var(cpus);
                 raw_spin_lock(&entry->redir_lock);
@@ -284,7 +282,7 @@ static void low_prio_arrival(struct task_struct *task)
         } else
 #endif
         {
-        arrive:
+arrive:
                 job_arrival(task);
         }
 }
@@ -576,19 +574,19 @@ static enum hrtimer_restart mc_ghost_exhausted(struct hrtimer *timer)
 #endif
 }
 
-static enum hrtimer_restart ce_timer_function(struct hrtimer *timer)
+/*
+ * The MC-CE common timer callback code for merged and non-merged timers.
+ * Returns the next time the timer should fire.
+ */
+static lt_t __ce_timer_function(struct ce_dom_data *ce_data)
 {
-        struct ce_dom_data *ce_data =
-                container_of(timer, struct ce_dom_data, timer);
-        struct crit_entry *ce = &per_cpu(cpus, ce_data->cpu).crit_entries[CRIT_LEVEL_A];
+        struct crit_entry *ce = get_crit_entry_for(ce_data->cpu, CRIT_LEVEL_A);
         struct domain *dom = ce->domain;
         struct task_struct *old_link = NULL;
-        unsigned long flags;
+        lt_t next_timer_abs;
 
         TRACE("MC level-A timer callback for CPU %d\n", ce_data->cpu);
 
-        local_irq_save(flags);
-
         raw_spin_lock(dom->lock);
 
         raw_spin_lock(&crit_cpu(ce)->lock);
@@ -602,7 +600,7 @@ static enum hrtimer_restart ce_timer_function(struct hrtimer *timer)
         }
         raw_spin_unlock(&crit_cpu(ce)->lock);
 
-        mc_ce_timer_callback_common(dom, timer);
+        next_timer_abs = mc_ce_timer_callback_common(dom);
 
         /* job completion will check for preemptions by means of calling job
          * arrival if the task is not blocked */
@@ -615,11 +613,38 @@ static enum hrtimer_restart ce_timer_function(struct hrtimer *timer)
                 raw_spin_unlock(dom->lock);
                 check_for_preempt(dom);
         }
+        return next_timer_abs;
+}
 
+#ifdef CONFIG_MERGE_TIMERS
+static void ce_timer_function(struct rt_event *e)
+{
+        struct ce_dom_data *ce_data =
+                container_of(e, struct ce_dom_data, event);
+        struct event_group *event_group = get_event_group_for(ce_data->cpu);
+        unsigned long flags;
+        lt_t next_timer_abs;
+
+        local_irq_save(flags);
+        next_timer_abs = __ce_timer_function(ce_data);
+        add_event(event_group, e, next_timer_abs);
         local_irq_restore(flags);
+}
+#else /* else to CONFIG_MERGE_TIMERS */
+static enum hrtimer_restart ce_timer_function(struct hrtimer *timer)
+{
+        struct ce_dom_data *ce_data =
+                container_of(timer, struct ce_dom_data, timer);
+        unsigned long flags;
+        lt_t next_timer_abs;
 
+        local_irq_save(flags);
+        next_timer_abs = __ce_timer_function(ce_data);
+        hrtimer_set_expires(timer, ns_to_ktime(next_timer_abs));
+        local_irq_restore(flags);
         return HRTIMER_RESTART;
 }
+#endif /* CONFIG_MERGE_TIMERS */
 
 
 /**
@@ -915,14 +940,6 @@ static rt_domain_t _mc_crit_c_rt;
 struct bheap _mc_heap_c;
 struct bheap_node _mc_nodes_c[NR_CPUS];
 
-#ifdef CONFIG_MERGE_TIMERS
-#ifdef CONFIG_PLUGIN_MC_RELEASE_MASTER
-struct event_group _mc_group;
-#else
-DEFINE_PER_CPU(struct event_group, _mc_groups);
-#endif
-#endif
-
 static long mc_activate_plugin(void)
 {
         struct domain_data *dom_data;
@@ -933,8 +950,14 @@ static long mc_activate_plugin(void)
 
 #ifdef CONFIG_RELEASE_MASTER
         interrupt_cpu = atomic_read(&release_master_cpu);
-        if (interrupt_cpu == NO_CPU)
-                interrupt_cpu = 0;
+#if defined(CONFIG_PLUGIN_MC_REDIRECT) || \
+    (defined(CONFIG_PLUGIN_MC_RELEASE_MASTER) && defined(CONFIG_MERGE_TIMERS))
+        if (NO_CPU == interrupt_cpu) {
+                printk(KERN_ERR "LITMUS-MC: need a release master\n");
+                ret = -EINVAL;
+                goto out;
+        }
+#endif
 #endif
 
         for_each_online_cpu(cpu) {
@@ -1016,20 +1039,30 @@ static void init_global_domain(struct domain_data *dom_data, enum crit_level lev
 }
 
 static inline void init_edf_domain(struct domain *dom, rt_domain_t *rt,
-                                   int timer_cpu, int prio)
+                                   int prio, int is_partitioned, int cpu)
 {
         pd_domain_init(dom, rt, edf_ready_order, NULL,
                        mc_release_jobs, mc_preempt_needed,
                        edf_higher_prio);
-#ifdef CONFIG_PLUGIN_MC_RELEASE_MASTER
-#ifdef CONFIG_MERGE_TIMERS
-        rt->event_group = &_mc_group;
+#if defined(CONFIG_PLUGIN_MC_RELEASE_MASTER) && defined(CONFIG_MERGE_TIMERS)
+        /* All timers are on one CPU and release-master is using the event
+         * merging interface as well. */
+        BUG_ON(NO_CPU == interrupt_cpu);
+        rt->event_group = get_event_group_for(interrupt_cpu);
         rt->prio = prio;
-#else
+#elif defined(CONFIG_PLUGIN_MC_RELEASE_MASTER) && !defined(CONFIG_MERGE_TIMERS)
+        /* Using release master, but not merging timers. */
         rt->release_master = interrupt_cpu;
-#endif
-#elif CONFIG_MERGE_TIMERS
-        rt->event_group = &_mc_groups[timer_cpu];
+#elif !defined(CONFIG_PLUGIN_MC_RELEASE_MASTER) && defined(CONFIG_MERGE_TIMERS)
+        /* Merge the timers, but don't move them to the release master. */
+        if (is_partitioned) {
+                rt->event_group = get_event_group_for(cpu);
+        } else {
+                /* Global timers will be added to the event groups that code is
+                 * executing on when add_event() is called.
+                 */
+                rt->event_group = NULL;
+        }
         rt->prio = prio;
 #endif
 }
@@ -1058,16 +1091,6 @@ static int __init init_mc(void)
                 INIT_LIST_HEAD(&entry->redir);
 #endif
 
-#ifdef CONFIG_MERGE_TIMERS
-#ifdef CONFIG_PLUGIN_MC_RELEASE_MASTER
-                entry->event_group = &_mc_group;
-#else
-                init_event_group(&_mc_groups[cpu],
-                                 CONFIG_MERGE_TIMERS_WINDOW, cpu);
-                entry->event_group = &_mc_groups[cpu];
-#endif
-#endif
-
                 /* CRIT_LEVEL_A */
                 dom_data = &per_cpu(_mc_crit_a, cpu);
                 ce_data = &per_cpu(_mc_crit_a_ce_data, cpu);
@@ -1085,25 +1108,17 @@ static int __init init_mc(void)
                 dom_data = &per_cpu(_mc_crit_b, cpu);
                 rt = &per_cpu(_mc_crit_b_rt, cpu);
                 init_local_domain(entry, dom_data, CRIT_LEVEL_B);
-                init_edf_domain(&dom_data->domain, rt, cpu, CRIT_LEVEL_B);
+                init_edf_domain(&dom_data->domain, rt, CRIT_LEVEL_B, 1, cpu);
                 b_dom_lock = dom_data->domain.lock;
                 raw_spin_lock_init(b_dom_lock);
                 dom_data->domain.name = "LVL-B";
         }
 
-#ifdef CONFIG_MERGE_TIMERS
-#ifdef CONFIG_PLUGIN_MC_RELEASE_MASTER
-        init_event_group(&_mc_group, CONFIG_MERGE_TIMERS_WINDOW, interrupt_cpu);
-        global_group = &_mc_group;
-#else
-        global_group = &_mc_groups[0];
-#endif
-#endif
-
         /* CRIT_LEVEL_C */
         init_global_domain(&_mc_crit_c, CRIT_LEVEL_C,
                            &_mc_heap_c, _mc_nodes_c);
-        init_edf_domain(&_mc_crit_c.domain, &_mc_crit_c_rt, 0, CRIT_LEVEL_C);
+        init_edf_domain(&_mc_crit_c.domain, &_mc_crit_c_rt, CRIT_LEVEL_C,
+                        0, NO_CPU);
         c_dom_lock = _mc_crit_c.domain.lock;
         raw_spin_lock_init(c_dom_lock);
         _mc_crit_c.domain.name = "LVL-C";