Refactor timer merging and add it to CE plugin.

THIS CODE IS UNTESTED

We now initialize one event group for each cpu on system start. We can
get the event group for a CPU via a function in event_group.c

Another change is that an event now stores what group it is in when it
add_event() is called on it. This lets us cancel it without knowing what
event group it is in.

The above is important because Level-C events (like releases) have a
NULL event group. When calling add_event(), it will get the event group
of the current CPU. If the event needs to be canceled later, we need
that saved group in the event so we know where to remove it from.

1 files changed, 75 insertions, 60 deletions

diff --git a/litmus/sched_mc.c b/litmus/sched_mc.c
index 7b74958d1f4f..17f84c9eba79 100644
--- a/litmus/sched_mc.c
+++ b/litmus/sched_mc.c
@@ -27,6 +27,8 @@
 #include <litmus/sched_mc.h>
 #include <litmus/ce_domain.h>
 
+/* XXX TODO Do we ever want to move level-A timers? */
+
 /**
  * struct cpu_entry - State of a CPU for the entire MC system
  * @cpu           CPU id
@@ -34,6 +36,9 @@
  * @linked        Task that should be running / is logically running
  * @lock          For serialization
  * @crit_entries  Array of CPU state per criticality level
+ * @redir         List of redirected work for this CPU.
+ * @redir_lock    Lock for @redir.
+ * @event_group   Event group for timer merging.
  */
 struct cpu_entry {
         int                     cpu;
@@ -45,18 +50,12 @@ struct cpu_entry {
         struct list_head        redir;
         raw_spinlock_t          redir_lock;
 #endif
-#ifdef CONFIG_MERGE_TIMERS
-        struct event_group*     event_group;
-#endif
 };
 
 DEFINE_PER_CPU(struct cpu_entry, cpus);
 #ifdef CONFIG_RELEASE_MASTER
 static int interrupt_cpu;
 #endif
-#ifdef CONFIG_MERGE_TIMERS
-static struct event_group* global_group;
-#endif
 
 #define domain_data(dom)  (container_of(dom, struct domain_data, domain))
 #define is_global(dom)    (domain_data(dom)->heap)
@@ -67,6 +66,7 @@ static struct event_group* global_group;
         (((e)->linked) ? tsk_mc_crit((e)->linked) : NUM_CRIT_LEVELS - 1)
 #define crit_cpu(ce) \
         (container_of((void*)((ce) - (ce)->level), struct cpu_entry, crit_entries))
+#define get_crit_entry_for(cpu, level) (&per_cpu(cpus, cpu).crit_entries[level])
 #define TRACE_ENTRY(e, fmt, args...)                            \
         STRACE("P%d, linked=" TS " " fmt "\n", e->cpu, TA(e->linked), ##args)
 #define TRACE_CRIT_ENTRY(ce, fmt, args...)                      \
@@ -129,7 +129,7 @@ static inline struct crit_entry* lowest_prio_cpu(struct domain *dom)
 static inline void cancel_ghost(struct crit_entry *ce)
 {
 #ifdef CONFIG_MERGE_TIMERS
-        cancel_event(crit_cpu(ce)->event_group, &ce->event);
+        cancel_event(&ce->event);
 #else
         hrtimer_try_to_cancel(&ce->timer);
 #endif
@@ -141,10 +141,10 @@ static inline void cancel_ghost(struct crit_entry *ce)
 static inline void arm_ghost(struct crit_entry *ce, lt_t fire)
 {
 #ifdef CONFIG_MERGE_TIMERS
-        add_event(crit_cpu(ce)->event_group, &ce->event, fire);
+        add_event(get_event_group_for(crit_cpu(ce)->cpu), &ce->event, fire);
 #else
         __hrtimer_start_range_ns(&ce->timer,
-                                 ns_to_ktime(when_to_fire),
+                                 ns_to_ktime(fire),
                                  0 /* delta */,
                                  HRTIMER_MODE_ABS_PINNED,
                                  0 /* no wakeup */);
@@ -270,10 +270,8 @@ static void low_prio_arrival(struct task_struct *task)
         if (!can_requeue(task)) return;
 
 #ifdef  CONFIG_PLUGIN_MC_REDIRECT
-#ifndef CONFIG_PLUGIN_MC_REDIRECT_ALL
         if (!is_global_task(task))
-            goto arrive;
-#endif
+                goto arrive;
         if (smp_processor_id() != interrupt_cpu) {
                 entry = &__get_cpu_var(cpus);
                 raw_spin_lock(&entry->redir_lock);
@@ -284,7 +282,7 @@ static void low_prio_arrival(struct task_struct *task)
         } else
 #endif
         {
-        arrive:
+arrive:
                 job_arrival(task);
         }
 }
@@ -576,19 +574,19 @@ static enum hrtimer_restart mc_ghost_exhausted(struct hrtimer *timer)
 #endif
 }
 
-static enum hrtimer_restart ce_timer_function(struct hrtimer *timer)
+/*
+ * The MC-CE common timer callback code for merged and non-merged timers.
+ * Returns the next time the timer should fire.
+ */
+static lt_t __ce_timer_function(struct ce_dom_data *ce_data)
 {
-        struct ce_dom_data *ce_data =
-                container_of(timer, struct ce_dom_data, timer);
-        struct crit_entry *ce = &per_cpu(cpus, ce_data->cpu).crit_entries[CRIT_LEVEL_A];
+        struct crit_entry *ce = get_crit_entry_for(ce_data->cpu, CRIT_LEVEL_A);
         struct domain *dom = ce->domain;
         struct task_struct *old_link = NULL;
-        unsigned long flags;
+        lt_t next_timer_abs;
 
         TRACE("MC level-A timer callback for CPU %d\n", ce_data->cpu);
 
-        local_irq_save(flags);
-
         raw_spin_lock(dom->lock);
 
         raw_spin_lock(&crit_cpu(ce)->lock);
@@ -602,7 +600,7 @@ static enum hrtimer_restart ce_timer_function(struct hrtimer *timer)
         }
         raw_spin_unlock(&crit_cpu(ce)->lock);
 
-        mc_ce_timer_callback_common(dom, timer);
+        next_timer_abs = mc_ce_timer_callback_common(dom);
 
         /* job completion will check for preemptions by means of calling job
          * arrival if the task is not blocked */
@@ -615,11 +613,38 @@ static enum hrtimer_restart ce_timer_function(struct hrtimer *timer)
                 raw_spin_unlock(dom->lock);
                 check_for_preempt(dom);
         }
+        return next_timer_abs;
+}
 
+#ifdef CONFIG_MERGE_TIMERS
+static void ce_timer_function(struct rt_event *e)
+{
+        struct ce_dom_data *ce_data =
+                container_of(e, struct ce_dom_data, event);
+        struct event_group *event_group = get_event_group_for(ce_data->cpu);
+        unsigned long flags;
+        lt_t next_timer_abs;
+
+        local_irq_save(flags);
+        next_timer_abs = __ce_timer_function(ce_data);
+        add_event(event_group, e, next_timer_abs);
         local_irq_restore(flags);
+}
+#else /* else to CONFIG_MERGE_TIMERS */
+static enum hrtimer_restart ce_timer_function(struct hrtimer *timer)
+{
+        struct ce_dom_data *ce_data =
+                container_of(timer, struct ce_dom_data, timer);
+        unsigned long flags;
+        lt_t next_timer_abs;
 
+        local_irq_save(flags);
+        next_timer_abs = __ce_timer_function(ce_data);
+        hrtimer_set_expires(timer, ns_to_ktime(next_timer_abs));
+        local_irq_restore(flags);
         return HRTIMER_RESTART;
 }
+#endif /* CONFIG_MERGE_TIMERS */
 
 
 /**
@@ -915,14 +940,6 @@ static rt_domain_t _mc_crit_c_rt;
 struct bheap _mc_heap_c;
 struct bheap_node _mc_nodes_c[NR_CPUS];
 
-#ifdef CONFIG_MERGE_TIMERS
-#ifdef CONFIG_PLUGIN_MC_RELEASE_MASTER
-struct event_group _mc_group;
-#else
-DEFINE_PER_CPU(struct event_group, _mc_groups);
-#endif
-#endif
-
 static long mc_activate_plugin(void)
 {
         struct domain_data *dom_data;
@@ -933,8 +950,14 @@ static long mc_activate_plugin(void)
 
 #ifdef CONFIG_RELEASE_MASTER
         interrupt_cpu = atomic_read(&release_master_cpu);
-        if (interrupt_cpu == NO_CPU)
-                interrupt_cpu = 0;
+#if defined(CONFIG_PLUGIN_MC_REDIRECT) || \
+    (defined(CONFIG_PLUGIN_MC_RELEASE_MASTER) && defined(CONFIG_MERGE_TIMERS))
+        if (NO_CPU == interrupt_cpu) {
+                printk(KERN_ERR "LITMUS-MC: need a release master\n");
+                ret = -EINVAL;
+                goto out;
+        }
+#endif
 #endif
 
         for_each_online_cpu(cpu) {
@@ -1016,20 +1039,30 @@ static void init_global_domain(struct domain_data *dom_data, enum crit_level lev
 }
 
 static inline void init_edf_domain(struct domain *dom, rt_domain_t *rt,
-                                   int timer_cpu, int prio)
+                                   int prio, int is_partitioned, int cpu)
 {
         pd_domain_init(dom, rt, edf_ready_order, NULL,
                        mc_release_jobs, mc_preempt_needed,
                        edf_higher_prio);
-#ifdef CONFIG_PLUGIN_MC_RELEASE_MASTER
-#ifdef CONFIG_MERGE_TIMERS
-        rt->event_group = &_mc_group;
+#if defined(CONFIG_PLUGIN_MC_RELEASE_MASTER) && defined(CONFIG_MERGE_TIMERS)
+        /* All timers are on one CPU and release-master is using the event
+         * merging interface as well. */
+        BUG_ON(NO_CPU == interrupt_cpu);
+        rt->event_group = get_event_group_for(interrupt_cpu);
         rt->prio = prio;
-#else
+#elif defined(CONFIG_PLUGIN_MC_RELEASE_MASTER) && !defined(CONFIG_MERGE_TIMERS)
+        /* Using release master, but not merging timers. */
         rt->release_master = interrupt_cpu;
-#endif
-#elif CONFIG_MERGE_TIMERS
-        rt->event_group = &_mc_groups[timer_cpu];
+#elif !defined(CONFIG_PLUGIN_MC_RELEASE_MASTER) && defined(CONFIG_MERGE_TIMERS)
+        /* Merge the timers, but don't move them to the release master. */
+        if (is_partitioned) {
+                rt->event_group = get_event_group_for(cpu);
+        } else {
+                /* Global timers will be added to the event groups that code is
+                 * executing on when add_event() is called.
+                 */
+                rt->event_group = NULL;
+        }
         rt->prio = prio;
 #endif
 }
@@ -1058,16 +1091,6 @@ static int __init init_mc(void)
                 INIT_LIST_HEAD(&entry->redir);
 #endif
 
-#ifdef CONFIG_MERGE_TIMERS
-#ifdef CONFIG_PLUGIN_MC_RELEASE_MASTER
-                entry->event_group = &_mc_group;
-#else
-                init_event_group(&_mc_groups[cpu],
-                                 CONFIG_MERGE_TIMERS_WINDOW, cpu);
-                entry->event_group = &_mc_groups[cpu];
-#endif
-#endif
-
                 /* CRIT_LEVEL_A */
                 dom_data = &per_cpu(_mc_crit_a, cpu);
                 ce_data = &per_cpu(_mc_crit_a_ce_data, cpu);
@@ -1085,25 +1108,17 @@ static int __init init_mc(void)
                 dom_data = &per_cpu(_mc_crit_b, cpu);
                 rt = &per_cpu(_mc_crit_b_rt, cpu);
                 init_local_domain(entry, dom_data, CRIT_LEVEL_B);
-                init_edf_domain(&dom_data->domain, rt, cpu, CRIT_LEVEL_B);
+                init_edf_domain(&dom_data->domain, rt, CRIT_LEVEL_B, 1, cpu);
                 b_dom_lock = dom_data->domain.lock;
                 raw_spin_lock_init(b_dom_lock);
                 dom_data->domain.name = "LVL-B";
         }
 
-#ifdef CONFIG_MERGE_TIMERS
-#ifdef CONFIG_PLUGIN_MC_RELEASE_MASTER
-        init_event_group(&_mc_group, CONFIG_MERGE_TIMERS_WINDOW, interrupt_cpu);
-        global_group = &_mc_group;
-#else
-        global_group = &_mc_groups[0];
-#endif
-#endif
-
         /* CRIT_LEVEL_C */
         init_global_domain(&_mc_crit_c, CRIT_LEVEL_C,
                            &_mc_heap_c, _mc_nodes_c);
-        init_edf_domain(&_mc_crit_c.domain, &_mc_crit_c_rt, 0, CRIT_LEVEL_C);
+        init_edf_domain(&_mc_crit_c.domain, &_mc_crit_c_rt, CRIT_LEVEL_C,
+                        0, NO_CPU);
         c_dom_lock = _mc_crit_c.domain.lock;
         raw_spin_lock_init(c_dom_lock);
         _mc_crit_c.domain.name = "LVL-C";