Implementation of Adaptive-EDZL

This patch introduces the Adaptive-EDZL (AEDZL) scheduler.  AEDZL
uses feedback-control to estimate job execution time.  This improves
the detection of zero-laxity points if WCETs are not tight.

8 files changed, 1131 insertions, 1 deletions

diff --git a/include/litmus/edzl_common.h b/include/litmus/edzl_common.h
index d9ecfa1a70f9..8b2fbcd455ab 100644
--- a/include/litmus/edzl_common.h
+++ b/include/litmus/edzl_common.h
@@ -21,4 +21,9 @@ int edzl_higher_prio(struct task_struct* first,
 int edzl_ready_order(struct bheap_node* a, struct bheap_node* b);
 
 int edzl_preemption_needed(rt_domain_t* rt, struct task_struct *t);
+
+#ifdef CONFIG_PLUGIN_AEDZL
+int aedzl_preemption_needed(rt_domain_t* rt, struct task_struct *t);
+#endif
+
 #endif
diff --git a/include/litmus/fpmath.h b/include/litmus/fpmath.h
new file mode 100644
index 000000000000..0ad1927e2261
--- /dev/null
+++ b/include/litmus/fpmath.h
@@ -0,0 +1,124 @@
+#ifndef __FP_MATH_H__
+#define __FP_MATH_H__
+
+#define FP_SHIFT 10
+#define ROUND_BIT (FP_SHIFT - 1)
+#define ONE FP(1)
+
+#define _fp(x) ((fp_t) {x})
+
+static const fp_t FP_ZERO = {.val = 0};
+static const fp_t FP_ONE = {.val = (1 << FP_SHIFT)};
+
+static inline fpbuf_t _point(fp_t x) 
+{       
+        return (x.val % (1 << FP_SHIFT));
+        
+}       
+
+#define fp2str(x) x.val
+/*(x.val >> FP_SHIFT), (x.val % (1 << FP_SHIFT)) */
+#define _FP_  "%ld/1024"
+
+static inline fp_t FP(fpbuf_t x)
+{
+        return _fp(((fpbuf_t) x) << FP_SHIFT);
+}
+
+static inline fpbuf_t _floor(fp_t x)
+{
+        return x.val >> FP_SHIFT;
+}
+
+/* FIXME: negative rounding */
+static inline fpbuf_t _round(fp_t x)
+{
+        return _floor(x) + ((x.val >> ROUND_BIT) & 1);
+}       
+
+/* divide two integers to obtain a fixed point value  */
+static inline fp_t _frac(fpbuf_t a, fpbuf_t b)
+{
+        return _fp(FP(a).val / (b));
+}
+
+/* multiply two fixed point values */
+static inline fp_t _mul(fp_t a, fp_t b)
+{       
+        return _fp((a.val * b.val) >> FP_SHIFT);
+}
+
+static inline fp_t _div(fp_t a, fp_t b)
+{
+        /* try not to overflow */
+        if (unlikely(  a.val > (2l << (BITS_PER_LONG - FP_SHIFT)) ))
+                return _fp((a.val / b.val) << FP_SHIFT);
+        else
+                return _fp((a.val << FP_SHIFT) / b.val);
+}
+
+static inline fp_t _add(fp_t a, fp_t b)
+{
+        return _fp(a.val + b.val);
+}
+
+static inline fp_t _sub(fp_t a, fp_t b)
+{
+        return _fp(a.val - b.val);
+}
+
+static inline fp_t _neg(fp_t x)
+{
+        return _fp(-x.val);
+}
+
+static inline fp_t _abs(fp_t x)
+{
+        return _fp(abs(x.val));
+}
+
+/* equiv. to casting float/double to integer */
+static inline fpbuf_t _fp_to_integer(fp_t x)
+{
+        return _floor(_abs(x)) * ((x.val > 0) ? 1 : -1);        
+}
+
+static inline fp_t _integer_to_fp(fpbuf_t x)
+{
+        return _frac(x,1);
+}
+
+static inline int _leq(fp_t a, fp_t b)
+{
+        return a.val <= b.val;
+}
+
+static inline int _geq(fp_t a, fp_t b)
+{
+        return a.val >= b.val;
+}
+
+static inline int _lt(fp_t a, fp_t b)
+{
+        return a.val < b.val;
+}
+
+static inline int _gt(fp_t a, fp_t b)
+{
+        return a.val > b.val;
+}
+
+static inline int _eq(fp_t a, fp_t b)
+{
+        return a.val == b.val;
+}
+
+static inline fp_t _max(fp_t a, fp_t b)
+{
+        if (a.val < b.val)
+                return b;
+        else
+                return a;
+}
+
+#endif
diff --git a/include/litmus/litmus.h b/include/litmus/litmus.h
index 26cddf5c9c84..95125d23c458 100644
--- a/include/litmus/litmus.h
+++ b/include/litmus/litmus.h
@@ -9,6 +9,10 @@
 #include <linux/jiffies.h>
 #include <litmus/sched_trace.h>
 
+#ifdef CONFIG_PLUGIN_AEDZL
+#include <litmus/fpmath.h>
+#endif
+
 #ifdef CONFIG_RELEASE_MASTER
 extern atomic_t release_master_cpu;
 #endif
@@ -96,6 +100,35 @@ inline static lt_t budget_remaining(struct task_struct* t)
 #ifdef CONFIG_PLUGIN_EDZL
 #define get_zerolaxity(t)   (tsk_rt(t)->job_params.zero_laxity)
 #define set_zerolaxity(t,s) (tsk_rt(t)->job_params.zero_laxity=(s))
+
+#ifdef CONFIG_PLUGIN_AEDZL
+inline static lt_t get_exec_cost_est(struct task_struct* t)
+{
+        return (lt_t)( /* express cost in terms of lt_t */
+                                abs( /* fp_t sometimes has both num and denom < 0,
+                                                assume fraction is not negative and take abs()*/
+                                        _fp_to_integer( /* truncate off fractional part */
+                                                        _mul( /* exe = util * period */
+                                                                        tsk_rt(t)->task_params.util_est, _frac(get_rt_period(t), 1)
+                                                                )
+                                                        )
+                                        )
+                                );
+}
+
+inline static int budget_exhausted_est(struct task_struct* t)
+{
+        return get_exec_time(t) >= get_exec_cost_est(t);
+}
+
+inline static lt_t budget_remaining_est(struct task_struct* t)
+{
+        if (!budget_exhausted_est(t))
+                return get_exec_cost_est(t) - get_exec_time(t);
+        else
+                return 0; /* avoid overflow */
+}
+#endif
 #endif
 
 #define budget_enforced(t) (tsk_rt(t)->task_params.budget_policy != NO_ENFORCEMENT)
@@ -157,6 +190,20 @@ inline static lt_t laxity_remaining(struct task_struct* t)
         else
                 return 0;
 }
+
+#ifdef CONFIG_PLUGIN_AEDZL
+inline static lt_t laxity_remaining_est(struct task_struct* t)
+{
+        lt_t now = litmus_clock();
+        lt_t remaining = budget_remaining_est(t);
+        lt_t deadline = get_deadline(t);
+
+        if(now + remaining < deadline)
+                return (deadline - (now + remaining));
+        else
+                return 0;
+}
+#endif
 #endif
 
 
diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h
index 53741727d5d0..d1a1f6ab523a 100644
--- a/include/litmus/rt_param.h
+++ b/include/litmus/rt_param.h
@@ -33,6 +33,14 @@ typedef enum {
         PRECISE_ENFORCEMENT  /* NOT IMPLEMENTED - enforced with hrtimers */
 } budget_policy_t;
 
+#ifdef CONFIG_PLUGIN_AEDZL
+typedef long fpbuf_t;
+typedef struct
+{
+        fpbuf_t val;
+} fp_t;
+#endif
+
 struct rt_task {
         lt_t            exec_cost;
         lt_t            period;
@@ -40,6 +48,11 @@ struct rt_task {
         unsigned int    cpu;
         task_class_t    cls;
         budget_policy_t budget_policy; /* ignored by pfair */
+
+#ifdef CONFIG_PLUGIN_AEDZL
+        fp_t            util_est;
+        fp_t            accum_err;
+#endif
 };
 
 /* The definition of the data that is shared between the kernel and real-time
diff --git a/litmus/Kconfig b/litmus/Kconfig
index cb0ff9785152..55cfb5de4637 100644
--- a/litmus/Kconfig
+++ b/litmus/Kconfig
@@ -34,6 +34,18 @@ config PLUGIN_EDZL
 
           if unsure, say Yes.
 
+config PLUGIN_AEDZL
+        bool "AEDZL"
+        depends on PLUGIN_EDZL
+        default y
+        help
+          Include the AEDZL (Adpative Earliest Deadline, Zero Laxity) plugin in
+          the kernel. AEDZL functions like EDZL, except that it uses feedback-
+          control methods to estimate actual job execution time.  This improves
+          the accuracy of determined zero-laxity points.
+
+          If unsure, say Yes.
+
 config RELEASE_MASTER
         bool "Release-master Support"
         depends on ARCH_HAS_SEND_PULL_TIMERS
diff --git a/litmus/Makefile b/litmus/Makefile
index 5ec42ac7cafc..6a3b5274f2c0 100644
--- a/litmus/Makefile
+++ b/litmus/Makefile
@@ -19,6 +19,7 @@ obj-y     = sched_plugin.o litmus.o \
 obj-$(CONFIG_PLUGIN_CEDF) += sched_cedf.o
 obj-$(CONFIG_PLUGIN_PFAIR) += sched_pfair.o
 obj-$(CONFIG_PLUGIN_EDZL) += sched_edzl.o edzl_common.o
+obj-$(CONFIG_PLUGIN_AEDZL) += sched_aedzl.o
 
 obj-$(CONFIG_FEATHER_TRACE) += ft_event.o ftdev.o
 obj-$(CONFIG_SCHED_TASK_TRACE) += sched_task_trace.o
diff --git a/litmus/edzl_common.c b/litmus/edzl_common.c
index f925901703ab..aa035ecdd8f9 100644
--- a/litmus/edzl_common.c
+++ b/litmus/edzl_common.c
@@ -72,7 +72,7 @@ void edzl_domain_init(rt_domain_t* rt, check_resched_needed_t resched,
  */
 int edzl_preemption_needed(rt_domain_t* rt, struct task_struct *t)
 {
-        /* we need the read lock for edf_ready_queue */
+        /* we need the read lock for edzl_ready_queue */
         /* no need to preempt if there is nothing pending */
         if (!__jobs_pending(rt))
                 return 0;
@@ -96,3 +96,29 @@ int edzl_preemption_needed(rt_domain_t* rt, struct task_struct *t)
 
         return edzl_higher_prio(__next_ready(rt), t);
 }
+
+
+#ifdef CONFIG_PLUGIN_AEDZL
+int aedzl_preemption_needed(rt_domain_t* rt, struct task_struct *t)
+{
+        /* we need the read lock for edzl_ready_queue */
+        /* no need to preempt if there is nothing pending */
+        if (!__jobs_pending(rt))
+                return 0;
+        /* we need to reschedule if t doesn't exist */
+        if (!t)
+                return 1;
+        /* make sure to get non-rt stuff out of the way */
+        if (!is_realtime(t))
+                return 1;
+
+        /* Detect zero-laxity as needed.  Easier to do it here than in tick.
+           (No timer is used to detect zero-laxity while a job is running.) */
+        if(unlikely(!get_zerolaxity(t) && laxity_remaining_est(t) == 0))
+        {
+                set_zerolaxity(t, 1);
+        }
+
+        return edzl_higher_prio(__next_ready(rt), t);
+}
+#endif
diff --git a/litmus/sched_aedzl.c b/litmus/sched_aedzl.c
new file mode 100644
index 000000000000..d521a6497677
--- /dev/null
+++ b/litmus/sched_aedzl.c
@@ -0,0 +1,902 @@
+/*
+ * litmus/sched_aedzl.c
+ *
+ * Implementation of the Adaptive-EDZL scheduling algorithm.
+ *
+ * This version uses the simple approach and serializes all scheduling
+ * decisions by the use of a queue lock. This is probably not the
+ * best way to do it, but it should suffice for now.
+ */
+
+#include <linux/spinlock.h>
+#include <linux/percpu.h>
+#include <linux/sched.h>
+#include <linux/hrtimer.h>
+
+#include <litmus/litmus.h>
+#include <litmus/jobs.h>
+#include <litmus/sched_plugin.h>
+#include <litmus/edzl_common.h>
+#include <litmus/sched_trace.h>
+
+#include <litmus/bheap.h>
+
+#include <linux/module.h>
+
+/* Overview of AEDZL operations.
+ *
+ * link_task_to_cpu(T, cpu)     - Low-level operation to update the linkage
+ *                                structure (NOT the actually scheduled
+ *                                task). If there is another linked task To
+ *                                already it will set To->linked_on = NO_CPU
+ *                                (thereby removing its association with this
+ *                                CPU). However, it will not requeue the
+ *                                previously linked task (if any). It will set
+ *                                T's state to RT_F_RUNNING and check whether
+ *                                it is already running somewhere else. If T
+ *                                is scheduled somewhere else it will link
+ *                                it to that CPU instead (and pull the linked
+ *                                task to cpu). T may be NULL.
+ *
+ * unlink(T)                    - Unlink removes T from all scheduler data
+ *                                structures. If it is linked to some CPU it
+ *                                will link NULL to that CPU. If it is
+ *                                currently queued in the edzl queue it will
+ *                                be removed from the rt_domain. It is safe to
+ *                                call unlink(T) if T is not linked. T may not
+ *                                be NULL.
+ *
+ * requeue(T)                   - Requeue will insert T into the appropriate
+ *                                queue. If the system is in real-time mode and
+ *                                the T is released already, it will go into the
+ *                                ready queue. If the system is not in
+ *                                real-time mode is T, then T will go into the
+ *                                release queue. If T's release time is in the
+ *                                future, it will go into the release
+ *                                queue. That means that T's release time/job
+ *                                no/etc. has to be updated before requeu(T) is
+ *                                called. It is not safe to call requeue(T)
+ *                                when T is already queued. T may not be NULL.
+ *
+ * aedzl_job_arrival(T) - This is the catch all function when T enters
+ *                                the system after either a suspension or at a
+ *                                job release. It will queue T (which means it
+ *                                is not safe to call aedzl_job_arrival(T) if
+ *                                T is already queued) and then check whether a
+ *                                preemption is necessary. If a preemption is
+ *                                necessary it will update the linkage
+ *                                accordingly and cause scheduled to be called
+ *                                (either with an IPI or need_resched). It is
+ *                                safe to call aedzl_job_arrival(T) if T's
+ *                                next job has not been actually released yet
+ *                                (releast time in the future). T will be put
+ *                                on the release queue in that case.
+ *
+ * job_completion(T)            - Take care of everything that needs to be done
+ *                                to prepare T for its next release and place
+ *                                it in the right queue with
+ *                                aedzl_job_arrival().
+ *
+ *
+ * When we now that T is linked to CPU then link_task_to_cpu(NULL, CPU) is
+ * equivalent to unlink(T). Note that if you unlink a task from a CPU none of
+ * the functions will automatically propagate pending task from the ready queue
+ * to a linked task. This is the job of the calling function ( by means of
+ * __take_ready).
+ */
+
+
+/* cpu_entry_t - maintain the linked and scheduled state
+ */
+typedef struct  {
+        int                     cpu;
+        struct task_struct*     linked;         /* only RT tasks */
+        struct task_struct*     scheduled;      /* only RT tasks */
+        atomic_t                will_schedule;  /* prevent unneeded IPIs */
+        struct bheap_node*      hn;
+} cpu_entry_t;
+DEFINE_PER_CPU(cpu_entry_t, aedzl_cpu_entries);
+
+cpu_entry_t* aedzl_cpus[NR_CPUS];
+
+#define set_will_schedule() \
+        (atomic_set(&__get_cpu_var(aedzl_cpu_entries).will_schedule, 1))
+#define clear_will_schedule() \
+        (atomic_set(&__get_cpu_var(aedzl_cpu_entries).will_schedule, 0))
+#define test_will_schedule(cpu) \
+        (atomic_read(&per_cpu(aedzl_cpu_entries, cpu).will_schedule))
+
+
+/* the cpus queue themselves according to priority in here */
+static struct bheap_node aedzl_heap_node[NR_CPUS];
+static struct bheap      aedzl_cpu_heap;
+
+/* Design choice: use one RT domain for both zero-laxity and regular jobs. */
+
+static rt_domain_t aedzl;
+#define aedzl_lock (aedzl.ready_lock)
+
+static const fp_t a_fp = {.val = 104}; /* 0.102 -- 102/1000 */
+static const fp_t b_fp = {.val = 310}; /* 0.303 -- 303/1000 */
+
+/* Uncomment this if you want to see all scheduling decisions in the
+ * TRACE() log.
+#define WANT_ALL_SCHED_EVENTS
+ */
+
+static int cpu_lower_prio(struct bheap_node *_a, struct bheap_node *_b)
+{
+        cpu_entry_t *a, *b;
+        a = _a->value;
+        b = _b->value;
+        /* Note that a and b are inverted: we want the lowest-priority CPU at
+         * the top of the heap.
+         */
+        return edzl_higher_prio(b->linked, a->linked);
+}
+
+/* update_cpu_position - Move the cpu entry to the correct place to maintain
+ *                       order in the cpu queue. Caller must hold edzl lock.
+ */
+static void update_cpu_position(cpu_entry_t *entry)
+{
+        if (likely(bheap_node_in_heap(entry->hn)))
+                bheap_delete(cpu_lower_prio, &aedzl_cpu_heap, entry->hn);
+        bheap_insert(cpu_lower_prio, &aedzl_cpu_heap, entry->hn);
+}
+
+/* caller must hold edzl lock */
+static cpu_entry_t* lowest_prio_cpu(void)
+{
+        struct bheap_node* hn;
+        hn = bheap_peek(cpu_lower_prio, &aedzl_cpu_heap);
+        return hn->value;
+}
+
+
+/* link_task_to_cpu - Update the link of a CPU.
+ *                    Handles the case where the to-be-linked task is already
+ *                    scheduled on a different CPU.
+ */
+static noinline void link_task_to_cpu(struct task_struct* linked,
+                                      cpu_entry_t *entry)
+{
+        cpu_entry_t *sched;
+        struct task_struct* tmp;
+        int on_cpu;
+
+        BUG_ON(linked && !is_realtime(linked));
+
+        /* Currently linked task is set to be unlinked. */
+        if (entry->linked) {
+                entry->linked->rt_param.linked_on = NO_CPU;
+        }
+
+        /* Link new task to CPU. */
+        if (linked) {
+                set_rt_flags(linked, RT_F_RUNNING);
+                /* handle task is already scheduled somewhere! */
+                on_cpu = linked->rt_param.scheduled_on;
+                if (on_cpu != NO_CPU) {
+                        sched = &per_cpu(aedzl_cpu_entries, on_cpu);
+                        /* this should only happen if not linked already */
+                        BUG_ON(sched->linked == linked);
+
+                        /* If we are already scheduled on the CPU to which we
+                         * wanted to link, we don't need to do the swap --
+                         * we just link ourselves to the CPU and depend on
+                         * the caller to get things right.
+                         */
+                        if (entry != sched) {
+                                TRACE_TASK(linked,
+                                           "already scheduled on %d, updating link.\n",
+                                           sched->cpu);
+                                tmp = sched->linked;
+                                linked->rt_param.linked_on = sched->cpu;
+                                sched->linked = linked;
+                                update_cpu_position(sched);
+                                linked = tmp;
+                        }
+                }
+                if (linked) /* might be NULL due to swap */
+                        linked->rt_param.linked_on = entry->cpu;
+        }
+        entry->linked = linked;
+#ifdef WANT_ALL_SCHED_EVENTS
+        if (linked)
+                TRACE_TASK(linked, "linked to %d.\n", entry->cpu);
+        else
+                TRACE("NULL linked to %d.\n", entry->cpu);
+#endif
+        update_cpu_position(entry);
+}
+
+/* unlink - Make sure a task is not linked any longer to an entry
+ *          where it was linked before. Must hold aedzl_lock.
+ */
+static noinline void unlink(struct task_struct* t)
+{
+        cpu_entry_t *entry;
+
+        if (unlikely(!t)) {
+                TRACE_BUG_ON(!t);
+                return;
+        }
+
+        if (t->rt_param.linked_on != NO_CPU) {
+                /* unlink */
+                entry = &per_cpu(aedzl_cpu_entries, t->rt_param.linked_on);
+                t->rt_param.linked_on = NO_CPU;
+                link_task_to_cpu(NULL, entry);
+        } else if (is_queued(t)) {
+                /* This is an interesting situation: t is scheduled,
+                 * but was just recently unlinked.  It cannot be
+                 * linked anywhere else (because then it would have
+                 * been relinked to this CPU), thus it must be in some
+                 * queue. We must remove it from the list in this
+                 * case.
+                 */
+                remove(&aedzl, t);
+        }
+}
+
+static inline void update_exe_estimate(struct task_struct *t, lt_t actual_cost)
+{
+        fp_t err, new;
+        fp_t actual_util = _frac(actual_cost, get_rt_period(t));
+
+        TRACE_TASK(t, "OLD cost: %llu, est cost: %llu, est util: %d.%d, est err: %d.%d\n",
+                                        tsk_rt(t)->task_params.exec_cost,
+                                        get_exec_cost_est(t),
+                                        _fp_to_integer(tsk_rt(t)->task_params.util_est), _point(tsk_rt(t)->task_params.util_est),
+                                        _fp_to_integer(tsk_rt(t)->task_params.accum_err), _point(tsk_rt(t)->task_params.accum_err));
+
+        err = _sub(actual_util, tsk_rt(t)->task_params.util_est);
+        /*
+        TRACE_TASK(t, "delta err = actual - est\n"
+                                  "\t%d.%d = %d.%d - %d.%d\n",
+                                  _fp_to_integer(err), _point(err),
+                                  _fp_to_integer(actual_util), _point(actual_util),
+                                  _fp_to_integer(tsk_rt(t)->task_params.util_est), _point(tsk_rt(t)->task_params.util_est));
+        */
+
+        new = _add(_mul(a_fp, err),
+                           _mul(b_fp, tsk_rt(t)->task_params.accum_err));
+        /*
+        TRACE_TASK(t, "util est = a*(delta error) + b*(accum error)\n"
+                                  "\t%d.%d = %d.%d * %d.%d + %d.%d * %d.%d\n",
+                                  _fp_to_integer(new), _point(new),
+                                  _fp_to_integer(a_fp), _point(a_fp),
+                                  _fp_to_integer(err), _point(err),
+                                  _fp_to_integer(b_fp), _point(b_fp),
+                                  _fp_to_integer(tsk_rt(t)->task_params.accum_err), _point(tsk_rt(t)->task_params.accum_err));
+        */
+
+        tsk_rt(t)->task_params.util_est = new;
+        tsk_rt(t)->task_params.accum_err = _add(tsk_rt(t)->task_params.accum_err, err);
+
+        TRACE_TASK(t, "cost: %llu, est cost: %llu, est util: %d.%d, est err: %d.%d, (delta cost: %d.%d, delta err: %d.%d)\n",
+                                        tsk_rt(t)->task_params.exec_cost,
+                                        get_exec_cost_est(t),
+                                        _fp_to_integer(tsk_rt(t)->task_params.util_est), _point(tsk_rt(t)->task_params.util_est),
+                                        _fp_to_integer(tsk_rt(t)->task_params.accum_err), _point(tsk_rt(t)->task_params.accum_err),
+                                        _fp_to_integer(new), _point(new),
+                                        _fp_to_integer(err), _point(err));
+}
+
+
+static void update_queue_position(struct task_struct *t);
+
+static enum hrtimer_restart on_zero_laxity(struct hrtimer *timer)
+{
+        unsigned long flags;
+        struct task_struct* t;
+
+        lt_t now = litmus_clock();
+
+        TRACE("Zero-laxity timer went off!\n");
+
+        raw_spin_lock_irqsave(&aedzl_lock, flags);
+
+        t = container_of(container_of(timer, struct rt_param, zl_timer),
+                                        struct task_struct,
+                                        rt_param);
+
+        TRACE_TASK(t, "Reached zero-laxity. (now: %llu, zl-pt: %lld, time remaining (now): %lld)\n",
+                                        now,
+                                        get_deadline(t) - budget_remaining_est(t),
+                                        get_deadline(t) - now);
+
+        tsk_rt(t)->zl_timer_armed = 0;
+        set_zerolaxity(t, 1);
+        update_queue_position(t);
+
+        raw_spin_unlock_irqrestore(&aedzl_lock, flags);
+
+        return HRTIMER_NORESTART;
+}
+
+/* __aedzl_take_ready - call's __take_ready with EDZL timer cancelation side-effect. */
+static inline struct task_struct* __aedzl_take_ready(rt_domain_t* rt)
+{
+        struct task_struct* t = __take_ready(rt);
+
+        if(t)
+        {
+                if(get_zerolaxity(t) == 0)
+                {
+                        if(tsk_rt(t)->zl_timer_armed)
+                        {
+                                int cancel_ret;
+
+                                TRACE_TASK(t, "Canceling zero-laxity timer.\n");
+                                cancel_ret = hrtimer_try_to_cancel(&tsk_rt(t)->zl_timer);
+                                WARN_ON(cancel_ret == 0); /* should never be inactive. */
+                                tsk_rt(t)->zl_timer_armed = 0;
+                        }
+                }
+                else
+                {
+                        TRACE_TASK(t, "Task already has zero-laxity flagged.\n");
+                }
+        }
+
+        return t;
+}
+
+/* __aedzl_add_ready - call's __add_ready with EDZL setting timer side-effect. */
+static inline void __aedzl_add_ready(rt_domain_t* rt, struct task_struct *new)
+{
+        __add_ready(rt, new);
+
+        if(get_zerolaxity(new) == 0)
+        {
+                lt_t when_to_fire;
+
+                when_to_fire = get_deadline(new) - budget_remaining_est(new);
+
+                TRACE_TASK(new, "Setting zero-laxity timer for %llu. (deadline: %llu, est remaining: %llu)\n",
+                                                when_to_fire,
+                                                get_deadline(new),
+                                                budget_remaining_est(new));
+
+                __hrtimer_start_range_ns(&tsk_rt(new)->zl_timer,
+                                                ns_to_ktime(when_to_fire),
+                                                0,
+                                                HRTIMER_MODE_ABS_PINNED,
+                                                0);
+
+                tsk_rt(new)->zl_timer_armed = 1;
+        }
+        else
+        {
+                TRACE_TASK(new, "Already has zero-laxity when added to ready queue. (deadline: %llu, est remaining: %llu))\n",
+                                get_deadline(new),
+                                budget_remaining_est(new));
+        }
+}
+
+static void check_for_preemptions(void);
+
+/* Glenn: Originally written for FMLP prio-inheritance, but
+   we can use it to upgrade the priority of a zero-laxity task. */
+static void update_queue_position(struct task_struct *t)
+{
+        /* Assumption: caller holds aedzl_lock */
+
+        int check_preempt = 0;
+
+        if (tsk_rt(t)->linked_on != NO_CPU) {
+                TRACE_TASK(t, "%s: linked  on %d\n",
+                           __FUNCTION__, tsk_rt(t)->linked_on);
+                /* Holder is scheduled; need to re-order CPUs.
+                 * We can't use heap_decrease() here since
+                 * the cpu_heap is ordered in reverse direction, so
+                 * it is actually an increase. */
+                bheap_delete(cpu_lower_prio, &aedzl_cpu_heap,
+                            aedzl_cpus[tsk_rt(t)->linked_on]->hn);
+                bheap_insert(cpu_lower_prio, &aedzl_cpu_heap,
+                            aedzl_cpus[tsk_rt(t)->linked_on]->hn);
+        } else {
+                /* holder may be queued: first stop queue changes */
+                raw_spin_lock(&aedzl.release_lock);
+                if (is_queued(t)) {
+                        TRACE_TASK(t, "%s: is queued\n",
+                                   __FUNCTION__);
+                        /* We need to update the position
+                         * of holder in some heap. Note that this
+                         * may be a release heap. */
+                        check_preempt =
+                                !bheap_decrease(edzl_ready_order,
+                                               tsk_rt(t)->heap_node);
+                } else {
+                        /* Nothing to do: if it is not queued and not linked
+                         * then it is currently being moved by other code
+                         * (e.g., a timer interrupt handler) that will use the
+                         * correct priority when enqueuing the task. */
+                        TRACE_TASK(t, "%s: is NOT queued => Done.\n",
+                                   __FUNCTION__);
+                }
+                raw_spin_unlock(&aedzl.release_lock);
+
+                /* If holder was enqueued in a release heap, then the following
+                 * preemption check is pointless, but we can't easily detect
+                 * that case. If you want to fix this, then consider that
+                 * simply adding a state flag requires O(n) time to update when
+                 * releasing n tasks, which conflicts with the goal to have
+                 * O(log n) merges. */
+                if (check_preempt) {
+                        /* heap_decrease() hit the top level of the heap: make
+                         * sure preemption checks get the right task, not the
+                         * potentially stale cache. */
+                        bheap_uncache_min(edzl_ready_order,
+                                         &aedzl.ready_queue);
+                        check_for_preemptions();
+                }
+        }
+}
+
+
+
+/* preempt - force a CPU to reschedule
+ */
+static void preempt(cpu_entry_t *entry)
+{
+        preempt_if_preemptable(entry->scheduled, entry->cpu);
+}
+
+/* requeue - Put an unlinked task into gsn-edf domain.
+ *           Caller must hold aedzl_lock.
+ */
+static noinline void requeue(struct task_struct* task)
+{
+        BUG_ON(!task);
+        /* sanity check before insertion */
+        BUG_ON(is_queued(task));
+
+        if (is_released(task, litmus_clock())) {
+                __aedzl_add_ready(&aedzl, task);
+        }
+        else {
+                /* it has got to wait */
+                add_release(&aedzl, task);
+        }
+}
+
+/* check for any necessary preemptions */
+static void check_for_preemptions(void)
+{
+        struct task_struct *task;
+        cpu_entry_t* last;
+
+        for(last = lowest_prio_cpu();
+            aedzl_preemption_needed(&aedzl, last->linked);
+            last = lowest_prio_cpu()) {
+                /* preemption necessary */
+                task = __aedzl_take_ready(&aedzl);
+
+                TRACE("check_for_preemptions: attempting to link task %d to %d\n",
+                      task->pid, last->cpu);
+                if (last->linked)
+                        requeue(last->linked);
+                link_task_to_cpu(task, last);
+                preempt(last);
+        }
+}
+
+/* aedzl_job_arrival: task is either resumed or released */
+static noinline void aedzl_job_arrival(struct task_struct* task)
+{
+        BUG_ON(!task);
+
+        /* tag zero-laxity at release time */
+        set_zerolaxity(task, laxity_remaining_est(task) == 0);
+
+        requeue(task);
+        check_for_preemptions();
+}
+
+static void aedzl_release_jobs(rt_domain_t* rt, struct bheap* tasks)
+{
+        unsigned long flags;
+
+        raw_spin_lock_irqsave(&aedzl_lock, flags);
+
+        __merge_ready(rt, tasks);
+        check_for_preemptions();
+
+        raw_spin_unlock_irqrestore(&aedzl_lock, flags);
+}
+
+/* caller holds aedzl_lock */
+static noinline void job_completion(struct task_struct *t, int forced)
+{
+        BUG_ON(!t);
+
+        sched_trace_task_completion(t, forced);
+
+        TRACE_TASK(t, "job_completion().\n");
+
+        /* set flags */
+        set_rt_flags(t, RT_F_SLEEP);
+
+        /* update exe estimate */
+        update_exe_estimate(t, get_exec_time(t));
+
+        /* prepare for next period */
+        prepare_for_next_period(t);
+
+        if (is_released(t, litmus_clock()))
+                sched_trace_task_release(t);
+        /* unlink */
+        unlink(t);
+        /* requeue
+         * But don't requeue a blocking task. */
+        if (is_running(t))
+                aedzl_job_arrival(t);
+}
+
+/* aedzl_tick - this function is called for every local timer
+ *                         interrupt.
+ *
+ *                   checks whether the current task has expired and checks
+ *                   whether we need to preempt it if it has not expired
+ */
+static void aedzl_tick(struct task_struct* t)
+{
+        if (is_realtime(t) && budget_enforced(t) && budget_exhausted(t)) {
+                if (!is_np(t)) {
+                        /* np tasks will be preempted when they become
+                         * preemptable again
+                         */
+                        set_tsk_need_resched(t);
+                        set_will_schedule();
+                        TRACE("aedzl_scheduler_tick: "
+                              "%d is preemptable "
+                              " => FORCE_RESCHED\n", t->pid);
+                } else if (is_user_np(t)) {
+                        TRACE("aedzl_scheduler_tick: "
+                              "%d is non-preemptable, "
+                              "preemption delayed.\n", t->pid);
+                        request_exit_np(t);
+                }
+        }
+}
+
+/* Getting schedule() right is a bit tricky. schedule() may not make any
+ * assumptions on the state of the current task since it may be called for a
+ * number of reasons. The reasons include a scheduler_tick() determined that it
+ * was necessary, because sys_exit_np() was called, because some Linux
+ * subsystem determined so, or even (in the worst case) because there is a bug
+ * hidden somewhere. Thus, we must take extreme care to determine what the
+ * current state is.
+ *
+ * The CPU could currently be scheduling a task (or not), be linked (or not).
+ *
+ * The following assertions for the scheduled task could hold:
+ *
+ *      - !is_running(scheduled)        // the job blocks
+ *      - scheduled->timeslice == 0     // the job completed (forcefully)
+ *      - get_rt_flag() == RT_F_SLEEP   // the job completed (by syscall)
+ *      - linked != scheduled           // we need to reschedule (for any reason)
+ *      - is_np(scheduled)              // rescheduling must be delayed,
+ *                                         sys_exit_np must be requested
+ *
+ * Any of these can occur together.
+ */
+static struct task_struct* aedzl_schedule(struct task_struct * prev)
+{
+        cpu_entry_t* entry = &__get_cpu_var(aedzl_cpu_entries);
+        int out_of_time, sleep, preempt, np, exists, blocks;
+        struct task_struct* next = NULL;
+
+#ifdef CONFIG_RELEASE_MASTER
+        /* Bail out early if we are the release master.
+         * The release master never schedules any real-time tasks.
+         */
+        if (aedzl.release_master == entry->cpu)
+                return NULL;
+#endif
+
+        raw_spin_lock(&aedzl_lock);
+        clear_will_schedule();
+
+        /* sanity checking */
+        BUG_ON(entry->scheduled && entry->scheduled != prev);
+        BUG_ON(entry->scheduled && !is_realtime(prev));
+        BUG_ON(is_realtime(prev) && !entry->scheduled);
+
+        /* (0) Determine state */
+        exists      = entry->scheduled != NULL;
+        blocks      = exists && !is_running(entry->scheduled);
+        out_of_time = exists &&
+                                  budget_enforced(entry->scheduled) &&
+                                  budget_exhausted(entry->scheduled);
+        np          = exists && is_np(entry->scheduled);
+        sleep       = exists && get_rt_flags(entry->scheduled) == RT_F_SLEEP;
+        preempt     = entry->scheduled != entry->linked;
+
+#ifdef WANT_ALL_SCHED_EVENTS
+        TRACE_TASK(prev, "invoked aedzl_schedule.\n");
+#endif
+
+        if (exists)
+                TRACE_TASK(prev,
+                           "blocks:%d out_of_time:%d np:%d sleep:%d preempt:%d "
+                           "state:%d sig:%d\n",
+                           blocks, out_of_time, np, sleep, preempt,
+                           prev->state, signal_pending(prev));
+        if (entry->linked && preempt)
+                TRACE_TASK(prev, "will be preempted by %s/%d\n",
+                           entry->linked->comm, entry->linked->pid);
+
+
+        /* If a task blocks we have no choice but to reschedule.
+         */
+        if (blocks)
+                unlink(entry->scheduled);
+
+        /* Request a sys_exit_np() call if we would like to preempt but cannot.
+         * We need to make sure to update the link structure anyway in case
+         * that we are still linked. Multiple calls to request_exit_np() don't
+         * hurt.
+         */
+        if (np && (out_of_time || preempt || sleep)) {
+                unlink(entry->scheduled);
+                request_exit_np(entry->scheduled);
+        }
+
+        /* Any task that is preemptable and either exhausts its execution
+         * budget or wants to sleep completes. We may have to reschedule after
+         * this. Don't do a job completion if we block (can't have timers running
+         * for blocked jobs). Preemption go first for the same reason.
+         */
+        if (!np && (out_of_time || sleep) && !blocks && !preempt)
+                job_completion(entry->scheduled, !sleep);
+
+        /* Link pending task if we became unlinked.
+         */
+        if (!entry->linked) {
+                link_task_to_cpu(__aedzl_take_ready(&aedzl), entry);
+        }
+
+        /* The final scheduling decision. Do we need to switch for some reason?
+         * If linked is different from scheduled, then select linked as next.
+         */
+        if ((!np || blocks) &&
+            entry->linked != entry->scheduled) {
+                /* Schedule a linked job? */
+                if (entry->linked) {
+                        entry->linked->rt_param.scheduled_on = entry->cpu;
+                        next = entry->linked;
+                }
+                if (entry->scheduled) {
+                        /* not gonna be scheduled soon */
+                        entry->scheduled->rt_param.scheduled_on = NO_CPU;
+                        TRACE_TASK(entry->scheduled, "scheduled_on = NO_CPU\n");
+                }
+        } else
+                /* Only override Linux scheduler if we have a real-time task
+                 * scheduled that needs to continue.
+                 */
+                if (exists)
+                        next = prev;
+
+        raw_spin_unlock(&aedzl_lock);
+
+#ifdef WANT_ALL_SCHED_EVENTS
+        TRACE("aedzl_lock released, next=0x%p\n", next);
+
+        if (next)
+                TRACE_TASK(next, "scheduled at %llu\n", litmus_clock());
+        else if (exists && !next)
+                TRACE("becomes idle at %llu.\n", litmus_clock());
+#endif
+
+
+        return next;
+}
+
+
+/* _finish_switch - we just finished the switch away from prev
+ */
+static void aedzl_finish_switch(struct task_struct *prev)
+{
+        cpu_entry_t*    entry = &__get_cpu_var(aedzl_cpu_entries);
+
+        entry->scheduled = is_realtime(current) ? current : NULL;
+#ifdef WANT_ALL_SCHED_EVENTS
+        TRACE_TASK(prev, "switched away from\n");
+#endif
+}
+
+
+/*      Prepare a task for running in RT mode
+ */
+static void aedzl_task_new(struct task_struct * t, int on_rq, int running)
+{
+        unsigned long           flags;
+        cpu_entry_t*            entry;
+
+        TRACE("edzl: task new %d\n", t->pid);
+
+        raw_spin_lock_irqsave(&aedzl_lock, flags);
+
+    t->rt_param.zl_timer_armed = 0;
+        hrtimer_init(&t->rt_param.zl_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
+        t->rt_param.zl_timer.function = on_zero_laxity;
+
+        /* setup job params */
+        release_at(t, litmus_clock());
+
+        if (running) {
+                entry = &per_cpu(aedzl_cpu_entries, task_cpu(t));
+                BUG_ON(entry->scheduled);
+
+#ifdef CONFIG_RELEASE_MASTER
+                if (entry->cpu != aedzl.release_master) {
+#endif
+                        entry->scheduled = t;
+                        tsk_rt(t)->scheduled_on = task_cpu(t);
+#ifdef CONFIG_RELEASE_MASTER
+                } else {
+                        /* do not schedule on release master */
+                        preempt(entry); /* force resched */
+                        tsk_rt(t)->scheduled_on = NO_CPU;
+                }
+#endif
+        } else {
+                t->rt_param.scheduled_on = NO_CPU;
+        }
+        t->rt_param.linked_on          = NO_CPU;
+
+        aedzl_job_arrival(t);
+        raw_spin_unlock_irqrestore(&aedzl_lock, flags);
+}
+
+static void aedzl_task_wake_up(struct task_struct *task)
+{
+        unsigned long flags;
+        lt_t now;
+
+        TRACE_TASK(task, "wake_up at %llu\n", litmus_clock());
+
+        raw_spin_lock_irqsave(&aedzl_lock, flags);
+        /* We need to take suspensions because of semaphores into
+         * account! If a job resumes after being suspended due to acquiring
+         * a semaphore, it should never be treated as a new job release.
+         */
+        if (get_rt_flags(task) == RT_F_EXIT_SEM) {
+                set_rt_flags(task, RT_F_RUNNING);
+        } else {
+                now = litmus_clock();
+                if (is_tardy(task, now)) {
+                        /* new sporadic release */
+                        release_at(task, now);
+                        sched_trace_task_release(task);
+                }
+                else {
+                        if (task->rt.time_slice) {
+                                /* came back in time before deadline
+                                */
+                                set_rt_flags(task, RT_F_RUNNING);
+                        }
+                }
+        }
+        aedzl_job_arrival(task);
+        raw_spin_unlock_irqrestore(&aedzl_lock, flags);
+}
+
+static void aedzl_task_block(struct task_struct *t)
+{
+        unsigned long flags;
+
+        TRACE_TASK(t, "block at %llu\n", litmus_clock());
+
+        /* unlink if necessary */
+        raw_spin_lock_irqsave(&aedzl_lock, flags);
+        unlink(t);
+        raw_spin_unlock_irqrestore(&aedzl_lock, flags);
+
+        BUG_ON(!is_realtime(t));
+}
+
+
+static void aedzl_task_exit(struct task_struct * t)
+{
+        unsigned long flags;
+
+        /* unlink if necessary */
+        raw_spin_lock_irqsave(&aedzl_lock, flags);
+        unlink(t);
+        if (tsk_rt(t)->scheduled_on != NO_CPU) {
+                aedzl_cpus[tsk_rt(t)->scheduled_on]->scheduled = NULL;
+                tsk_rt(t)->scheduled_on = NO_CPU;
+        }
+
+        if(tsk_rt(t)->zl_timer_armed)
+        {
+                /* BUG if reached? */
+                TRACE_TASK(t, "Canceled armed timer while exiting.\n");
+                hrtimer_cancel(&tsk_rt(t)->zl_timer);
+                tsk_rt(t)->zl_timer_armed = 0;
+        }
+
+        raw_spin_unlock_irqrestore(&aedzl_lock, flags);
+
+        BUG_ON(!is_realtime(t));
+        TRACE_TASK(t, "RIP\n");
+}
+
+static long aedzl_admit_task(struct task_struct* tsk)
+{
+        return 0;
+}
+
+static long aedzl_activate_plugin(void)
+{
+        int cpu;
+        cpu_entry_t *entry;
+
+        bheap_init(&aedzl_cpu_heap);
+#ifdef CONFIG_RELEASE_MASTER
+        aedzl.release_master = atomic_read(&release_master_cpu);
+#endif
+
+        for_each_online_cpu(cpu) {
+                entry = &per_cpu(aedzl_cpu_entries, cpu);
+                bheap_node_init(&entry->hn, entry);
+                atomic_set(&entry->will_schedule, 0);
+                entry->linked    = NULL;
+                entry->scheduled = NULL;
+#ifdef CONFIG_RELEASE_MASTER
+                if (cpu != aedzl.release_master) {
+#endif
+                        TRACE("AEDZL: Initializing CPU #%d.\n", cpu);
+                        update_cpu_position(entry);
+#ifdef CONFIG_RELEASE_MASTER
+                } else {
+                        TRACE("AEDZL: CPU %d is release master.\n", cpu);
+                }
+#endif
+        }
+        return 0;
+}
+
+/*      Plugin object   */
+static struct sched_plugin aedzl_plugin __cacheline_aligned_in_smp = {
+        .plugin_name            = "AEDZL",
+        .finish_switch          = aedzl_finish_switch,
+        .tick                   = aedzl_tick,
+        .task_new               = aedzl_task_new,
+        .complete_job           = complete_job,
+        .task_exit              = aedzl_task_exit,
+        .schedule               = aedzl_schedule,
+        .task_wake_up           = aedzl_task_wake_up,
+        .task_block             = aedzl_task_block,
+        .admit_task             = aedzl_admit_task,
+        .activate_plugin        = aedzl_activate_plugin,
+};
+
+
+static int __init init_aedzl(void)
+{
+        int cpu;
+        cpu_entry_t *entry;
+
+        bheap_init(&aedzl_cpu_heap);
+        /* initialize CPU state */
+        for (cpu = 0; cpu < NR_CPUS; cpu++)  {
+                entry = &per_cpu(aedzl_cpu_entries, cpu);
+                aedzl_cpus[cpu] = entry;
+                atomic_set(&entry->will_schedule, 0);
+                entry->cpu       = cpu;
+                entry->hn        = &aedzl_heap_node[cpu];
+                bheap_node_init(&entry->hn, entry);
+        }
+        edzl_domain_init(&aedzl, NULL, aedzl_release_jobs);
+        return register_sched_plugin(&aedzl_plugin);
+}
+
+
+module_init(init_aedzl);