GPUSync patch for Litmus 2012.1.

1 files changed, 1024 insertions, 38 deletions

diff --git a/litmus/sched_cedf.c b/litmus/sched_cedf.c
index 480c62bc895b..be14dbec6ed2 100644
--- a/litmus/sched_cedf.c
+++ b/litmus/sched_cedf.c
@@ -29,7 +29,7 @@
 #include <linux/percpu.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
-
+#include <linux/uaccess.h>
 #include <linux/module.h>
 
 #include <litmus/litmus.h>
@@ -42,6 +42,16 @@
 #include <litmus/clustered.h>
 
 #include <litmus/bheap.h>
+#include <litmus/binheap.h>
+
+#ifdef CONFIG_LITMUS_LOCKING
+#include <litmus/kfmlp_lock.h>
+#endif
+
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+#include <litmus/rsm_lock.h>
+#include <litmus/ikglp_lock.h>
+#endif
 
 #ifdef CONFIG_SCHED_CPU_AFFINITY
 #include <litmus/affinity.h>
@@ -49,7 +59,27 @@
 
 /* to configure the cluster size */
 #include <litmus/litmus_proc.h>
-#include <linux/uaccess.h>
+
+#ifdef CONFIG_SCHED_CPU_AFFINITY
+#include <litmus/affinity.h>
+#endif
+
+#ifdef CONFIG_LITMUS_SOFTIRQD
+#include <litmus/litmus_softirq.h>
+#endif
+
+#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
+#include <linux/interrupt.h>
+#include <litmus/trace.h>
+#endif
+
+#ifdef CONFIG_LITMUS_NVIDIA
+#include <litmus/nvidia_info.h>
+#endif
+
+#if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA)
+#include <litmus/gpu_affinity.h>
+#endif
 
 /* Reference configuration variable. Determines which cache level is used to
  * group CPUs into clusters.  GLOBAL_CLUSTER, which is the default, means that
@@ -70,7 +100,7 @@ typedef struct  {
         struct task_struct*     linked;         /* only RT tasks */
         struct task_struct*     scheduled;      /* only RT tasks */
         atomic_t                will_schedule;  /* prevent unneeded IPIs */
-        struct bheap_node*      hn;
+        struct binheap_node hn;
 } cpu_entry_t;
 
 /* one cpu_entry_t per CPU */
@@ -83,6 +113,14 @@ DEFINE_PER_CPU(cpu_entry_t, cedf_cpu_entries);
 #define test_will_schedule(cpu) \
         (atomic_read(&per_cpu(cedf_cpu_entries, cpu).will_schedule))
 
+#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
+struct tasklet_head
+{
+        struct tasklet_struct *head;
+        struct tasklet_struct **tail;
+};
+#endif
+
 /*
  * In C-EDF there is a cedf domain _per_ cluster
  * The number of clusters is dynamically determined accordingly to the
@@ -96,10 +134,17 @@ typedef struct clusterdomain {
         /* map of this cluster cpus */
         cpumask_var_t   cpu_map;
         /* the cpus queue themselves according to priority in here */
-        struct bheap_node *heap_node;
-        struct bheap      cpu_heap;
+        struct binheap_handle cpu_heap;
         /* lock for this cluster */
 #define cluster_lock domain.ready_lock
+
+#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
+        struct tasklet_head pending_tasklets;
+#endif
+
+#ifdef CONFIG_LITMUS_DGL_SUPPORT
+        raw_spinlock_t dgl_lock;
+#endif
 } cedf_domain_t;
 
 /* a cedf_domain per cluster; allocation is done at init/activation time */
@@ -108,6 +153,22 @@ cedf_domain_t *cedf;
 #define remote_cluster(cpu)     ((cedf_domain_t *) per_cpu(cedf_cpu_entries, cpu).cluster)
 #define task_cpu_cluster(task)  remote_cluster(get_partition(task))
 
+/* total number of cluster */
+static int num_clusters;
+/* we do not support cluster of different sizes */
+static unsigned int cluster_size;
+
+static int clusters_allocated = 0;
+
+#ifdef CONFIG_LITMUS_DGL_SUPPORT
+static raw_spinlock_t* cedf_get_dgl_spinlock(struct task_struct *t)
+{
+        cedf_domain_t *cluster = task_cpu_cluster(t);
+        return(&cluster->dgl_lock);
+}
+#endif
+
+
 /* Uncomment WANT_ALL_SCHED_EVENTS if you want to see all scheduling
  * decisions in the TRACE() log; uncomment VERBOSE_INIT for verbose
  * information during the initialization of the plugin (e.g., topology)
@@ -115,11 +176,11 @@ cedf_domain_t *cedf;
  */
 #define VERBOSE_INIT
 
-static int cpu_lower_prio(struct bheap_node *_a, struct bheap_node *_b)
+static int cpu_lower_prio(struct binheap_node *_a, struct binheap_node *_b)
 {
-        cpu_entry_t *a, *b;
-        a = _a->value;
-        b = _b->value;
+        cpu_entry_t *a = binheap_entry(_a, cpu_entry_t, hn);
+        cpu_entry_t *b = binheap_entry(_b, cpu_entry_t, hn);
+
         /* Note that a and b are inverted: we want the lowest-priority CPU at
          * the top of the heap.
          */
@@ -133,20 +194,17 @@ static void update_cpu_position(cpu_entry_t *entry)
 {
         cedf_domain_t *cluster = entry->cluster;
 
-        if (likely(bheap_node_in_heap(entry->hn)))
-                bheap_delete(cpu_lower_prio,
-                                &cluster->cpu_heap,
-                                entry->hn);
+        if (likely(binheap_is_in_heap(&entry->hn))) {
+                binheap_delete(&entry->hn, &cluster->cpu_heap);
+        }
 
-        bheap_insert(cpu_lower_prio, &cluster->cpu_heap, entry->hn);
+        binheap_add(&entry->hn, &cluster->cpu_heap, cpu_entry_t, hn);
 }
 
 /* caller must hold cedf lock */
 static cpu_entry_t* lowest_prio_cpu(cedf_domain_t *cluster)
 {
-        struct bheap_node* hn;
-        hn = bheap_peek(cpu_lower_prio, &cluster->cpu_heap);
-        return hn->value;
+        return binheap_top_entry(&cluster->cpu_heap, cpu_entry_t, hn);
 }
 
 
@@ -208,7 +266,7 @@ static noinline void link_task_to_cpu(struct task_struct* linked,
 }
 
 /* unlink - Make sure a task is not linked any longer to an entry
- *          where it was linked before. Must hold cedf_lock.
+ *          where it was linked before. Must hold cluster_lock.
  */
 static noinline void unlink(struct task_struct* t)
 {
@@ -244,7 +302,7 @@ static void preempt(cpu_entry_t *entry)
 }
 
 /* requeue - Put an unlinked task into gsn-edf domain.
- *           Caller must hold cedf_lock.
+ *           Caller must hold cluster_lock.
  */
 static noinline void requeue(struct task_struct* task)
 {
@@ -339,13 +397,17 @@ static void cedf_release_jobs(rt_domain_t* rt, struct bheap* tasks)
         raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags);
 }
 
-/* caller holds cedf_lock */
+/* caller holds cluster_lock */
 static noinline void job_completion(struct task_struct *t, int forced)
 {
         BUG_ON(!t);
 
         sched_trace_task_completion(t, forced);
 
+#ifdef CONFIG_LITMUS_NVIDIA
+        atomic_set(&tsk_rt(t)->nv_int_count, 0);
+#endif
+
         TRACE_TASK(t, "job_completion().\n");
 
         /* set flags */
@@ -389,6 +451,314 @@ static void cedf_tick(struct task_struct* t)
         }
 }
 
+
+
+
+
+
+
+
+
+
+
+
+
+#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
+
+
+static void __do_lit_tasklet(struct tasklet_struct* tasklet, unsigned long flushed)
+{
+        if (!atomic_read(&tasklet->count)) {
+                if(tasklet->owner) {
+                        sched_trace_tasklet_begin(tasklet->owner);
+                }
+
+                if (!test_and_clear_bit(TASKLET_STATE_SCHED, &tasklet->state))
+                {
+                        BUG();
+                }
+                TRACE("%s: Invoking tasklet with owner pid = %d (flushed = %d).\n",
+                          __FUNCTION__,
+                          (tasklet->owner) ? tasklet->owner->pid : -1,
+                          (tasklet->owner) ? 0 : 1);
+                tasklet->func(tasklet->data);
+                tasklet_unlock(tasklet);
+
+                if(tasklet->owner) {
+                        sched_trace_tasklet_end(tasklet->owner, flushed);
+                }
+        }
+        else {
+                BUG();
+        }
+}
+
+
+static void do_lit_tasklets(cedf_domain_t* cluster, struct task_struct* sched_task)
+{
+        int work_to_do = 1;
+        struct tasklet_struct *tasklet = NULL;
+        unsigned long flags;
+
+        while(work_to_do) {
+
+                TS_NV_SCHED_BOTISR_START;
+
+                raw_spin_lock_irqsave(&cluster->cluster_lock, flags);
+
+                if(cluster->pending_tasklets.head != NULL) {
+                        // remove tasklet at head.
+                        struct tasklet_struct *prev = NULL;
+                        tasklet = cluster->pending_tasklets.head;
+
+                        // find a tasklet with prio to execute; skip ones where
+                        // sched_task has a higher priority.
+                        // We use the '!edf' test instead of swaping function arguments since
+                        // both sched_task and owner could be NULL.  In this case, we want to
+                        // still execute the tasklet.
+                        while(tasklet && !edf_higher_prio(tasklet->owner, sched_task)) {
+                                prev = tasklet;
+                                tasklet = tasklet->next;
+                        }
+
+                        if(tasklet) {  // found something to execuite
+                                // remove the tasklet from the queue
+                                if(prev) {
+                                        prev->next = tasklet->next;
+                                        if(prev->next == NULL) {
+                                                TRACE("%s: Tasklet for %d is the last element in tasklet queue.\n", __FUNCTION__, tasklet->owner->pid);
+                                                cluster->pending_tasklets.tail = &(prev);
+                                        }
+                                }
+                                else {
+                                        cluster->pending_tasklets.head = tasklet->next;
+                                        if(tasklet->next == NULL) {
+                                                TRACE("%s: Tasklet for %d is the last element in tasklet queue.\n", __FUNCTION__, tasklet->owner->pid);
+                                                cluster->pending_tasklets.tail = &(cluster->pending_tasklets.head);
+                                        }
+                                }
+                        }
+                        else {
+                                TRACE("%s: No tasklets with eligible priority.\n", __FUNCTION__);
+                        }
+                }
+                else {
+                        TRACE("%s: Tasklet queue is empty.\n", __FUNCTION__);
+                }
+
+                raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags);
+
+                if(tasklet) {
+                        __do_lit_tasklet(tasklet, 0ul);
+                        tasklet = NULL;
+                }
+                else {
+                        work_to_do = 0;
+                }
+
+                TS_NV_SCHED_BOTISR_END;
+        }
+}
+
+static void __add_pai_tasklet(struct tasklet_struct* tasklet, cedf_domain_t* cluster)
+{
+        struct tasklet_struct* step;
+
+        tasklet->next = NULL;  // make sure there are no old values floating around
+
+        step = cluster->pending_tasklets.head;
+        if(step == NULL) {
+                TRACE("%s: tasklet queue empty.  inserting tasklet for %d at head.\n", __FUNCTION__, tasklet->owner->pid);
+                // insert at tail.
+                *(cluster->pending_tasklets.tail) = tasklet;
+                cluster->pending_tasklets.tail = &(tasklet->next);
+        }
+        else if((*(cluster->pending_tasklets.tail) != NULL) &&
+                        edf_higher_prio((*(cluster->pending_tasklets.tail))->owner, tasklet->owner)) {
+                // insert at tail.
+                TRACE("%s: tasklet belongs at end.  inserting tasklet for %d at tail.\n", __FUNCTION__, tasklet->owner->pid);
+
+                *(cluster->pending_tasklets.tail) = tasklet;
+                cluster->pending_tasklets.tail = &(tasklet->next);
+        }
+        else {
+
+                // insert the tasklet somewhere in the middle.
+
+        TRACE("%s: tasklet belongs somewhere in the middle.\n", __FUNCTION__);
+
+                while(step->next && edf_higher_prio(step->next->owner, tasklet->owner)) {
+                        step = step->next;
+                }
+
+                // insert tasklet right before step->next.
+
+                TRACE("%s: inserting tasklet for %d between %d and %d.\n", __FUNCTION__,
+                          tasklet->owner->pid,
+                          (step->owner) ?
+                          step->owner->pid :
+                          -1,
+                          (step->next) ?
+                          ((step->next->owner) ?
+                           step->next->owner->pid :
+                           -1) :
+                          -1);
+
+                tasklet->next = step->next;
+                step->next = tasklet;
+
+                // patch up the head if needed.
+                if(cluster->pending_tasklets.head == step)
+                {
+                        TRACE("%s: %d is the new tasklet queue head.\n", __FUNCTION__, tasklet->owner->pid);
+                        cluster->pending_tasklets.head = tasklet;
+                }
+        }
+}
+
+static void cedf_run_tasklets(struct task_struct* sched_task)
+{
+        cedf_domain_t* cluster;
+
+        preempt_disable();
+
+        cluster = (is_realtime(sched_task)) ?
+                task_cpu_cluster(sched_task) :
+                remote_cluster(smp_processor_id());
+
+        if(cluster && cluster->pending_tasklets.head != NULL) {
+                TRACE("%s: There are tasklets to process.\n", __FUNCTION__);
+                do_lit_tasklets(cluster, sched_task);
+        }
+
+        preempt_enable_no_resched();
+}
+
+
+
+static int cedf_enqueue_pai_tasklet(struct tasklet_struct* tasklet)
+{
+#if 0
+        cedf_domain_t *cluster = NULL;
+        cpu_entry_t *targetCPU = NULL;
+        int thisCPU;
+        int runLocal = 0;
+        int runNow = 0;
+        unsigned long flags;
+
+    if(unlikely((tasklet->owner == NULL) || !is_realtime(tasklet->owner)))
+    {
+        TRACE("%s: No owner associated with this tasklet!\n", __FUNCTION__);
+                return 0;
+    }
+
+        cluster = task_cpu_cluster(tasklet->owner);
+
+        raw_spin_lock_irqsave(&cluster->cluster_lock, flags);
+
+        thisCPU = smp_processor_id();
+
+#ifdef CONFIG_SCHED_CPU_AFFINITY
+        {
+                cpu_entry_t* affinity = NULL;
+
+                // use this CPU if it is in our cluster and isn't running any RT work.
+                if(cpu_isset(thisCPU, *cluster->cpu_map) && (__get_cpu_var(cedf_cpu_entries).linked == NULL)) {
+                        affinity = &(__get_cpu_var(cedf_cpu_entries));
+                }
+                else {
+                        // this CPU is busy or shouldn't run tasklet in this cluster.
+                        // look for available near by CPUs.
+                        // NOTE: Affinity towards owner and not this CPU.  Is this right?
+                        affinity =
+                                cedf_get_nearest_available_cpu(cluster,
+                                                                &per_cpu(cedf_cpu_entries, task_cpu(tasklet->owner)));
+                }
+
+                targetCPU = affinity;
+        }
+#endif
+
+        if (targetCPU == NULL) {
+                targetCPU = lowest_prio_cpu(cluster);
+        }
+
+        if (edf_higher_prio(tasklet->owner, targetCPU->linked)) {
+                if (thisCPU == targetCPU->cpu) {
+                        TRACE("%s: Run tasklet locally (and now).\n", __FUNCTION__);
+                        runLocal = 1;
+                        runNow = 1;
+                }
+                else {
+                        TRACE("%s: Run tasklet remotely (and now).\n", __FUNCTION__);
+                        runLocal = 0;
+                        runNow = 1;
+                }
+        }
+        else {
+                runLocal = 0;
+                runNow = 0;
+        }
+
+        if(!runLocal) {
+                // enqueue the tasklet
+                __add_pai_tasklet(tasklet, cluster);
+        }
+
+        raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags);
+
+
+        if (runLocal /*&& runNow */) {  // runNow == 1 is implied
+                TRACE("%s: Running tasklet on CPU where it was received.\n", __FUNCTION__);
+                __do_lit_tasklet(tasklet, 0ul);
+        }
+        else if (runNow /*&& !runLocal */) {  // runLocal == 0 is implied
+                TRACE("%s: Triggering CPU %d to run tasklet.\n", __FUNCTION__, targetCPU->cpu);
+                preempt(targetCPU);  // need to be protected by cluster_lock?
+        }
+        else {
+                TRACE("%s: Scheduling of tasklet was deferred.\n", __FUNCTION__);
+        }
+#else
+        TRACE("%s: Running tasklet on CPU where it was received.\n", __FUNCTION__);
+        __do_lit_tasklet(tasklet, 0ul);
+#endif
+        return(1); // success
+}
+
+static void cedf_change_prio_pai_tasklet(struct task_struct *old_prio,
+                                                                                 struct task_struct *new_prio)
+{
+        struct tasklet_struct* step;
+        unsigned long flags;
+        cedf_domain_t *cluster;
+        struct task_struct *probe;
+
+        // identify the cluster by the assignment of these tasks.  one should
+        // be non-NULL.
+        probe = (old_prio) ? old_prio : new_prio;
+
+        if(probe) {
+                cluster = task_cpu_cluster(probe);
+
+                if(cluster->pending_tasklets.head != NULL) {
+                        raw_spin_lock_irqsave(&cluster->cluster_lock, flags);
+                        for(step = cluster->pending_tasklets.head; step != NULL; step = step->next) {
+                                if(step->owner == old_prio) {
+                                        TRACE("%s: Found tasklet to change: %d\n", __FUNCTION__, step->owner->pid);
+                                        step->owner = new_prio;
+                                }
+                        }
+                        raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags);
+                }
+        }
+        else {
+                TRACE("%s: Both priorities were NULL\n");
+        }
+}
+
+#endif  // PAI
+
 /* Getting schedule() right is a bit tricky. schedule() may not make any
  * assumptions on the state of the current task since it may be called for a
  * number of reasons. The reasons include a scheduler_tick() determined that it
@@ -465,6 +835,19 @@ static struct task_struct* cedf_schedule(struct task_struct * prev)
         if (blocks)
                 unlink(entry->scheduled);
 
+#if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_AFFINITY_LOCKING)
+        if(exists && is_realtime(entry->scheduled) && tsk_rt(entry->scheduled)->held_gpus) {
+                if(!blocks || tsk_rt(entry->scheduled)->suspend_gpu_tracker_on_block) {
+                        // don't track preemptions or locking protocol suspensions.
+                        TRACE_TASK(entry->scheduled, "stopping GPU tracker.\n");
+                        stop_gpu_tracker(entry->scheduled);
+                }
+                else if(blocks && !tsk_rt(entry->scheduled)->suspend_gpu_tracker_on_block) {
+                        TRACE_TASK(entry->scheduled, "GPU tracker remains on during suspension.\n");
+                }
+        }
+#endif
+
         /* Request a sys_exit_np() call if we would like to preempt but cannot.
          * We need to make sure to update the link structure anyway in case
          * that we are still linked. Multiple calls to request_exit_np() don't
@@ -514,7 +897,7 @@ static struct task_struct* cedf_schedule(struct task_struct * prev)
         raw_spin_unlock(&cluster->cluster_lock);
 
 #ifdef WANT_ALL_SCHED_EVENTS
-        TRACE("cedf_lock released, next=0x%p\n", next);
+        TRACE("cluster_lock released, next=0x%p\n", next);
 
         if (next)
                 TRACE_TASK(next, "scheduled at %llu\n", litmus_clock());
@@ -522,7 +905,6 @@ static struct task_struct* cedf_schedule(struct task_struct * prev)
                 TRACE("becomes idle at %llu.\n", litmus_clock());
 #endif
 
-
         return next;
 }
 
@@ -548,7 +930,7 @@ static void cedf_task_new(struct task_struct * t, int on_rq, int running)
         cpu_entry_t*            entry;
         cedf_domain_t*          cluster;
 
-        TRACE("gsn edf: task new %d\n", t->pid);
+        TRACE("c-edf: task new %d\n", t->pid);
 
         /* the cluster doesn't change even if t is running */
         cluster = task_cpu_cluster(t);
@@ -586,7 +968,7 @@ static void cedf_task_new(struct task_struct * t, int on_rq, int running)
 static void cedf_task_wake_up(struct task_struct *task)
 {
         unsigned long flags;
-        lt_t now;
+        //lt_t now;
         cedf_domain_t *cluster;
 
         TRACE_TASK(task, "wake_up at %llu\n", litmus_clock());
@@ -594,6 +976,8 @@ static void cedf_task_wake_up(struct task_struct *task)
         cluster = task_cpu_cluster(task);
 
         raw_spin_lock_irqsave(&cluster->cluster_lock, flags);
+
+#if 0 // sproadic task model
         /* We need to take suspensions because of semaphores into
          * account! If a job resumes after being suspended due to acquiring
          * a semaphore, it should never be treated as a new job release.
@@ -615,7 +999,13 @@ static void cedf_task_wake_up(struct task_struct *task)
                         }
                 }
         }
-        cedf_job_arrival(task);
+#else
+        set_rt_flags(task, RT_F_RUNNING);  // periodic model
+#endif
+
+        if(tsk_rt(task)->linked_on == NO_CPU)
+                cedf_job_arrival(task);
+
         raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags);
 }
 
@@ -642,6 +1032,10 @@ static void cedf_task_exit(struct task_struct * t)
         unsigned long flags;
         cedf_domain_t *cluster = task_cpu_cluster(t);
 
+#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
+        cedf_change_prio_pai_tasklet(t, NULL);
+#endif
+
         /* unlink if necessary */
         raw_spin_lock_irqsave(&cluster->cluster_lock, flags);
         unlink(t);
@@ -659,13 +1053,536 @@ static void cedf_task_exit(struct task_struct * t)
 
 static long cedf_admit_task(struct task_struct* tsk)
 {
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+        INIT_BINHEAP_HANDLE(&tsk_rt(tsk)->hp_blocked_tasks,
+                                                edf_max_heap_base_priority_order);
+#endif
+
         return task_cpu(tsk) == tsk->rt_param.task_params.cpu ? 0 : -EINVAL;
 }
 
-/* total number of cluster */
-static int num_clusters;
-/* we do not support cluster of different sizes */
-static unsigned int cluster_size;
+
+
+#ifdef CONFIG_LITMUS_LOCKING
+
+#include <litmus/fdso.h>
+
+
+
+/* called with IRQs off */
+static void __increase_priority_inheritance(struct task_struct* t,
+                                                                                    struct task_struct* prio_inh)
+{
+        int linked_on;
+        int check_preempt = 0;
+
+        cedf_domain_t* cluster = task_cpu_cluster(t);
+
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+        /* this sanity check allows for weaker locking in protocols */
+        /* TODO (klitirqd): Skip this check if 't' is a proxy thread (???) */
+        if(__edf_higher_prio(prio_inh, BASE, t, EFFECTIVE)) {
+#endif
+                TRACE_TASK(t, "inherits priority from %s/%d\n",
+                                   prio_inh->comm, prio_inh->pid);
+                tsk_rt(t)->inh_task = prio_inh;
+
+                linked_on  = tsk_rt(t)->linked_on;
+
+                /* If it is scheduled, then we need to reorder the CPU heap. */
+                if (linked_on != NO_CPU) {
+                        TRACE_TASK(t, "%s: linked  on %d\n",
+                                           __FUNCTION__, linked_on);
+                        /* Holder is scheduled; need to re-order CPUs.
+                         * We can't use heap_decrease() here since
+                         * the cpu_heap is ordered in reverse direction, so
+                         * it is actually an increase. */
+                        binheap_delete(&per_cpu(cedf_cpu_entries, linked_on).hn,
+                                                   &cluster->cpu_heap);
+                        binheap_add(&per_cpu(cedf_cpu_entries, linked_on).hn,
+                                                &cluster->cpu_heap, cpu_entry_t, hn);
+
+                } else {
+                        /* holder may be queued: first stop queue changes */
+                        raw_spin_lock(&cluster->domain.release_lock);
+                        if (is_queued(t)) {
+                                TRACE_TASK(t, "%s: is queued\n",
+                                                   __FUNCTION__);
+                                /* We need to update the position of holder in some
+                                 * heap. Note that this could be a release heap if we
+                                 * budget enforcement is used and this job overran. */
+                                check_preempt =
+                                        !bheap_decrease(edf_ready_order, tsk_rt(t)->heap_node);
+                        } else {
+                                /* Nothing to do: if it is not queued and not linked
+                                 * then it is either sleeping or currently being moved
+                                 * by other code (e.g., a timer interrupt handler) that
+                                 * will use the correct priority when enqueuing the
+                                 * task. */
+                                TRACE_TASK(t, "%s: is NOT queued => Done.\n",
+                                                   __FUNCTION__);
+                        }
+                        raw_spin_unlock(&cluster->domain.release_lock);
+
+                        /* If holder was enqueued in a release heap, then the following
+                         * preemption check is pointless, but we can't easily detect
+                         * that case. If you want to fix this, then consider that
+                         * simply adding a state flag requires O(n) time to update when
+                         * releasing n tasks, which conflicts with the goal to have
+                         * O(log n) merges. */
+                        if (check_preempt) {
+                                /* heap_decrease() hit the top level of the heap: make
+                                 * sure preemption checks get the right task, not the
+                                 * potentially stale cache. */
+                                bheap_uncache_min(edf_ready_order,
+                                                                  &cluster->domain.ready_queue);
+                                check_for_preemptions(cluster);
+                        }
+                }
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+        }
+        else {
+                TRACE_TASK(t, "Spurious invalid priority increase. "
+                                   "Inheritance request: %s/%d [eff_prio = %s/%d] to inherit from %s/%d\n"
+                                   "Occurance is likely okay: probably due to (hopefully safe) concurrent priority updates.\n",
+                                   t->comm, t->pid,
+                                   effective_priority(t)->comm, effective_priority(t)->pid,
+                                   (prio_inh) ? prio_inh->comm : "nil",
+                                   (prio_inh) ? prio_inh->pid : -1);
+                WARN_ON(!prio_inh);
+        }
+#endif
+}
+
+/* called with IRQs off */
+static void increase_priority_inheritance(struct task_struct* t, struct task_struct* prio_inh)
+{
+        cedf_domain_t* cluster = task_cpu_cluster(t);
+
+        raw_spin_lock(&cluster->cluster_lock);
+
+        __increase_priority_inheritance(t, prio_inh);
+
+#ifdef CONFIG_LITMUS_SOFTIRQD
+        if(tsk_rt(t)->cur_klitirqd != NULL)
+        {
+                TRACE_TASK(t, "%s/%d inherits a new priority!\n",
+                                   tsk_rt(t)->cur_klitirqd->comm, tsk_rt(t)->cur_klitirqd->pid);
+
+                __increase_priority_inheritance(tsk_rt(t)->cur_klitirqd, prio_inh);
+        }
+#endif
+
+        raw_spin_unlock(&cluster->cluster_lock);
+
+#if defined(CONFIG_LITMUS_PAI_SOFTIRQD) && defined(CONFIG_LITMUS_NVIDIA)
+        if(tsk_rt(t)->held_gpus) {
+                int i;
+                for(i = find_first_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus));
+                        i < NV_DEVICE_NUM;
+                        i = find_next_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus), i+1)) {
+                        pai_check_priority_increase(t, i);
+                }
+        }
+#endif
+}
+
+/* called with IRQs off */
+static void __decrease_priority_inheritance(struct task_struct* t,
+                                                                                        struct task_struct* prio_inh)
+{
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+        if(__edf_higher_prio(t, EFFECTIVE, prio_inh, BASE)) {
+#endif
+                /* A job only stops inheriting a priority when it releases a
+                 * resource. Thus we can make the following assumption.*/
+                if(prio_inh)
+                        TRACE_TASK(t, "EFFECTIVE priority decreased to %s/%d\n",
+                                           prio_inh->comm, prio_inh->pid);
+                else
+                        TRACE_TASK(t, "base priority restored.\n");
+
+                tsk_rt(t)->inh_task = prio_inh;
+
+                if(tsk_rt(t)->scheduled_on != NO_CPU) {
+                        TRACE_TASK(t, "is scheduled.\n");
+
+                        /* Check if rescheduling is necessary. We can't use heap_decrease()
+                         * since the priority was effectively lowered. */
+                        unlink(t);
+                        cedf_job_arrival(t);
+                }
+                else {
+                        cedf_domain_t* cluster = task_cpu_cluster(t);
+                        /* task is queued */
+                        raw_spin_lock(&cluster->domain.release_lock);
+                        if (is_queued(t)) {
+                                TRACE_TASK(t, "is queued.\n");
+
+                                /* decrease in priority, so we have to re-add to binomial heap */
+                                unlink(t);
+                                cedf_job_arrival(t);
+                        }
+                        else {
+                                TRACE_TASK(t, "is not in scheduler. Probably on wait queue somewhere.\n");
+                        }
+                        raw_spin_unlock(&cluster->domain.release_lock);
+                }
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+        }
+        else {
+                TRACE_TASK(t, "Spurious invalid priority decrease. "
+                                   "Inheritance request: %s/%d [eff_prio = %s/%d] to inherit from %s/%d\n"
+                                   "Occurance is likely okay: probably due to (hopefully safe) concurrent priority updates.\n",
+                                   t->comm, t->pid,
+                                   effective_priority(t)->comm, effective_priority(t)->pid,
+                                   (prio_inh) ? prio_inh->comm : "nil",
+                                   (prio_inh) ? prio_inh->pid : -1);
+        }
+#endif
+}
+
+static void decrease_priority_inheritance(struct task_struct* t,
+                                                                                struct task_struct* prio_inh)
+{
+        cedf_domain_t* cluster = task_cpu_cluster(t);
+
+        raw_spin_lock(&cluster->cluster_lock);
+        __decrease_priority_inheritance(t, prio_inh);
+
+#ifdef CONFIG_LITMUS_SOFTIRQD
+        if(tsk_rt(t)->cur_klitirqd != NULL)
+        {
+                TRACE_TASK(t, "%s/%d decreases in priority!\n",
+                                   tsk_rt(t)->cur_klitirqd->comm, tsk_rt(t)->cur_klitirqd->pid);
+
+                __decrease_priority_inheritance(tsk_rt(t)->cur_klitirqd, prio_inh);
+        }
+#endif
+
+        raw_spin_unlock(&cluster->cluster_lock);
+
+#if defined(CONFIG_LITMUS_PAI_SOFTIRQD) && defined(CONFIG_LITMUS_NVIDIA)
+        if(tsk_rt(t)->held_gpus) {
+                int i;
+                for(i = find_first_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus));
+                        i < NV_DEVICE_NUM;
+                        i = find_next_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus), i+1)) {
+                        pai_check_priority_decrease(t, i);
+                }
+        }
+#endif
+}
+
+
+
+
+
+#ifdef CONFIG_LITMUS_SOFTIRQD
+/* called with IRQs off */
+static void increase_priority_inheritance_klitirqd(struct task_struct* klitirqd,
+                                                                                          struct task_struct* old_owner,
+                                                                                          struct task_struct* new_owner)
+{
+        cedf_domain_t* cluster = task_cpu_cluster(klitirqd);
+
+        BUG_ON(!(tsk_rt(klitirqd)->is_proxy_thread));
+
+        raw_spin_lock(&cluster->cluster_lock);
+
+        if(old_owner != new_owner)
+        {
+                if(old_owner)
+                {
+                        // unreachable?
+                        tsk_rt(old_owner)->cur_klitirqd = NULL;
+                }
+
+                TRACE_TASK(klitirqd, "giving ownership to %s/%d.\n",
+                                   new_owner->comm, new_owner->pid);
+
+                tsk_rt(new_owner)->cur_klitirqd = klitirqd;
+        }
+
+        __decrease_priority_inheritance(klitirqd, NULL);  // kludge to clear out cur prio.
+
+        __increase_priority_inheritance(klitirqd,
+                        (tsk_rt(new_owner)->inh_task == NULL) ?
+                                new_owner :
+                                tsk_rt(new_owner)->inh_task);
+
+        raw_spin_unlock(&cluster->cluster_lock);
+}
+
+
+/* called with IRQs off */
+static void decrease_priority_inheritance_klitirqd(struct task_struct* klitirqd,
+                                                                                                   struct task_struct* old_owner,
+                                                                                                   struct task_struct* new_owner)
+{
+        cedf_domain_t* cluster = task_cpu_cluster(klitirqd);
+
+        BUG_ON(!(tsk_rt(klitirqd)->is_proxy_thread));
+
+        raw_spin_lock(&cluster->cluster_lock);
+
+    TRACE_TASK(klitirqd, "priority restored\n");
+
+        __decrease_priority_inheritance(klitirqd, new_owner);
+
+        tsk_rt(old_owner)->cur_klitirqd = NULL;
+
+        raw_spin_unlock(&cluster->cluster_lock);
+}
+#endif // CONFIG_LITMUS_SOFTIRQD
+
+
+
+
+
+
+
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+
+/* called with IRQs off */
+/* preconditions:
+ (1) The 'hp_blocked_tasks_lock' of task 't' is held.
+ (2) The lock 'to_unlock' is held.
+ */
+static void nested_increase_priority_inheritance(struct task_struct* t,
+                                                                                                 struct task_struct* prio_inh,
+                                                                                                 raw_spinlock_t *to_unlock,
+                                                                                                 unsigned long irqflags)
+{
+        struct litmus_lock *blocked_lock = tsk_rt(t)->blocked_lock;
+
+        if(tsk_rt(t)->inh_task != prio_inh) {           // shield redundent calls.
+                increase_priority_inheritance(t, prio_inh);  // increase our prio.
+        }
+
+        raw_spin_unlock(&tsk_rt(t)->hp_blocked_tasks_lock);  // unlock the t's heap.
+
+
+        if(blocked_lock) {
+                if(blocked_lock->ops->propagate_increase_inheritance) {
+                        TRACE_TASK(t, "Inheritor is blocked (...perhaps).  Checking lock %d.\n",
+                                           blocked_lock->ident);
+
+                        // beware: recursion
+                        blocked_lock->ops->propagate_increase_inheritance(blocked_lock,
+                                                                                                                          t, to_unlock,
+                                                                                                                          irqflags);
+                }
+                else {
+                        TRACE_TASK(t, "Inheritor is blocked on lock (%d) that does not support nesting!\n",
+                                           blocked_lock->ident);
+                        unlock_fine_irqrestore(to_unlock, irqflags);
+                }
+        }
+        else {
+                TRACE_TASK(t, "is not blocked.  No propagation.\n");
+                unlock_fine_irqrestore(to_unlock, irqflags);
+        }
+}
+
+/* called with IRQs off */
+/* preconditions:
+ (1) The 'hp_blocked_tasks_lock' of task 't' is held.
+ (2) The lock 'to_unlock' is held.
+ */
+static void nested_decrease_priority_inheritance(struct task_struct* t,
+                                                                                                 struct task_struct* prio_inh,
+                                                                                                 raw_spinlock_t *to_unlock,
+                                                                                                 unsigned long irqflags)
+{
+        struct litmus_lock *blocked_lock = tsk_rt(t)->blocked_lock;
+        decrease_priority_inheritance(t, prio_inh);
+
+        raw_spin_unlock(&tsk_rt(t)->hp_blocked_tasks_lock);  // unlock the t's heap.
+
+        if(blocked_lock) {
+                if(blocked_lock->ops->propagate_decrease_inheritance) {
+                        TRACE_TASK(t, "Inheritor is blocked (...perhaps).  Checking lock %d.\n",
+                                           blocked_lock->ident);
+
+                        // beware: recursion
+                        blocked_lock->ops->propagate_decrease_inheritance(blocked_lock, t,
+                                                                                                                          to_unlock,
+                                                                                                                          irqflags);
+                }
+                else {
+                        TRACE_TASK(t, "Inheritor is blocked on lock (%p) that does not support nesting!\n",
+                                           blocked_lock);
+                        unlock_fine_irqrestore(to_unlock, irqflags);
+                }
+        }
+        else {
+                TRACE_TASK(t, "is not blocked.  No propagation.\n");
+                unlock_fine_irqrestore(to_unlock, irqflags);
+        }
+}
+
+
+/* ******************** RSM MUTEX ********************** */
+
+static struct litmus_lock_ops cedf_rsm_mutex_lock_ops = {
+        .lock   = rsm_mutex_lock,
+        .unlock = rsm_mutex_unlock,
+        .close  = rsm_mutex_close,
+        .deallocate = rsm_mutex_free,
+
+        .propagate_increase_inheritance = rsm_mutex_propagate_increase_inheritance,
+        .propagate_decrease_inheritance = rsm_mutex_propagate_decrease_inheritance,
+
+#ifdef CONFIG_LITMUS_DGL_SUPPORT
+        .dgl_lock = rsm_mutex_dgl_lock,
+        .is_owner = rsm_mutex_is_owner,
+        .enable_priority = rsm_mutex_enable_priority,
+#endif
+};
+
+static struct litmus_lock* cedf_new_rsm_mutex(void)
+{
+        return rsm_mutex_new(&cedf_rsm_mutex_lock_ops);
+}
+
+/* ******************** IKGLP ********************** */
+
+static struct litmus_lock_ops cedf_ikglp_lock_ops = {
+        .lock   = ikglp_lock,
+        .unlock = ikglp_unlock,
+        .close  = ikglp_close,
+        .deallocate = ikglp_free,
+
+        // ikglp can only be an outer-most lock.
+        .propagate_increase_inheritance = NULL,
+        .propagate_decrease_inheritance = NULL,
+};
+
+static struct litmus_lock* cedf_new_ikglp(void* __user arg)
+{
+        // assumes clusters of uniform size.
+        return ikglp_new(cluster_size/num_clusters, &cedf_ikglp_lock_ops, arg);
+}
+
+#endif  /* CONFIG_LITMUS_NESTED_LOCKING */
+
+
+
+
+/* ******************** KFMLP support ********************** */
+
+static struct litmus_lock_ops cedf_kfmlp_lock_ops = {
+        .lock   = kfmlp_lock,
+        .unlock = kfmlp_unlock,
+        .close  = kfmlp_close,
+        .deallocate = kfmlp_free,
+
+        // kfmlp can only be an outer-most lock.
+        .propagate_increase_inheritance = NULL,
+        .propagate_decrease_inheritance = NULL,
+};
+
+
+static struct litmus_lock* cedf_new_kfmlp(void* __user arg)
+{
+        return kfmlp_new(&cedf_kfmlp_lock_ops, arg);
+}
+
+
+/* **** lock constructor **** */
+
+static long cedf_allocate_lock(struct litmus_lock **lock, int type,
+                                                                 void* __user args)
+{
+        int err;
+
+        switch (type) {
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+                case RSM_MUTEX:
+                        *lock = cedf_new_rsm_mutex();
+                        break;
+
+                case IKGLP_SEM:
+                        *lock = cedf_new_ikglp(args);
+                        break;
+#endif
+                case KFMLP_SEM:
+                        *lock = cedf_new_kfmlp(args);
+                        break;
+
+                default:
+                        err = -ENXIO;
+                        goto UNSUPPORTED_LOCK;
+        };
+
+        if (*lock)
+                err = 0;
+        else
+                err = -ENOMEM;
+
+UNSUPPORTED_LOCK:
+        return err;
+}
+
+#endif  // CONFIG_LITMUS_LOCKING
+
+
+#ifdef CONFIG_LITMUS_AFFINITY_LOCKING
+static struct affinity_observer_ops cedf_kfmlp_affinity_ops = {
+        .close = kfmlp_aff_obs_close,
+        .deallocate = kfmlp_aff_obs_free,
+};
+
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+static struct affinity_observer_ops cedf_ikglp_affinity_ops = {
+        .close = ikglp_aff_obs_close,
+        .deallocate = ikglp_aff_obs_free,
+};
+#endif
+
+static long cedf_allocate_affinity_observer(struct affinity_observer **aff_obs,
+                                                                                        int type,
+                                                                                        void* __user args)
+{
+        int err;
+
+        switch (type) {
+
+                case KFMLP_SIMPLE_GPU_AFF_OBS:
+                        *aff_obs = kfmlp_simple_gpu_aff_obs_new(&cedf_kfmlp_affinity_ops, args);
+                        break;
+
+                case KFMLP_GPU_AFF_OBS:
+                        *aff_obs = kfmlp_gpu_aff_obs_new(&cedf_kfmlp_affinity_ops, args);
+                        break;
+
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+                case IKGLP_SIMPLE_GPU_AFF_OBS:
+                        *aff_obs = ikglp_simple_gpu_aff_obs_new(&cedf_ikglp_affinity_ops, args);
+                        break;
+
+                case IKGLP_GPU_AFF_OBS:
+                        *aff_obs = ikglp_gpu_aff_obs_new(&cedf_ikglp_affinity_ops, args);
+                        break;
+#endif
+                default:
+                        err = -ENXIO;
+                        goto UNSUPPORTED_AFF_OBS;
+        };
+
+        if (*aff_obs)
+                err = 0;
+        else
+                err = -ENOMEM;
+
+UNSUPPORTED_AFF_OBS:
+        return err;
+}
+#endif
+
+
+
 
 #ifdef VERBOSE_INIT
 static void print_cluster_topology(cpumask_var_t mask, int cpu)
@@ -680,16 +1597,17 @@ static void print_cluster_topology(cpumask_var_t mask, int cpu)
 }
 #endif
 
-static int clusters_allocated = 0;
-
 static void cleanup_cedf(void)
 {
         int i;
 
+#ifdef CONFIG_LITMUS_NVIDIA
+        shutdown_nvidia_info();
+#endif
+
         if (clusters_allocated) {
                 for (i = 0; i < num_clusters; i++) {
                         kfree(cedf[i].cpus);
-                        kfree(cedf[i].heap_node);
                         free_cpumask_var(cedf[i].cpu_map);
                 }
 
@@ -749,12 +1667,16 @@ static long cedf_activate_plugin(void)
 
                 cedf[i].cpus = kmalloc(cluster_size * sizeof(cpu_entry_t),
                                 GFP_ATOMIC);
-                cedf[i].heap_node = kmalloc(
-                                cluster_size * sizeof(struct bheap_node),
-                                GFP_ATOMIC);
-                bheap_init(&(cedf[i].cpu_heap));
+                INIT_BINHEAP_HANDLE(&(cedf[i].cpu_heap), cpu_lower_prio);
                 edf_domain_init(&(cedf[i].domain), NULL, cedf_release_jobs);
 
+
+#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
+                cedf[i].pending_tasklets.head = NULL;
+                cedf[i].pending_tasklets.tail = &(cedf[i].pending_tasklets.head);
+#endif
+
+
                 if(!zalloc_cpumask_var(&cedf[i].cpu_map, GFP_ATOMIC))
                         return -ENOMEM;
 #ifdef CONFIG_RELEASE_MASTER
@@ -765,6 +1687,10 @@ static long cedf_activate_plugin(void)
         /* cycle through cluster and add cpus to them */
         for (i = 0; i < num_clusters; i++) {
 
+#ifdef CONFIG_LITMUS_DGL_SUPPORT
+                raw_spin_lock_init(&cedf[i].dgl_lock);
+#endif
+
                 for_each_online_cpu(cpu) {
                         /* check if the cpu is already in a cluster */
                         for (j = 0; j < num_clusters; j++)
@@ -795,8 +1721,8 @@ static long cedf_activate_plugin(void)
                                 atomic_set(&entry->will_schedule, 0);
                                 entry->cpu = ccpu;
                                 entry->cluster = &cedf[i];
-                                entry->hn = &(cedf[i].heap_node[cpu_count]);
-                                bheap_node_init(&entry->hn, entry);
+
+                                INIT_BINHEAP_NODE(&entry->hn);
 
                                 cpu_count++;
 
@@ -813,6 +1739,40 @@ static long cedf_activate_plugin(void)
                 }
         }
 
+#ifdef CONFIG_LITMUS_SOFTIRQD
+        {
+                /* distribute the daemons evenly across the clusters. */
+                int* affinity = kmalloc(NR_LITMUS_SOFTIRQD * sizeof(int), GFP_ATOMIC);
+                int num_daemons_per_cluster = NR_LITMUS_SOFTIRQD / num_clusters;
+                int left_over = NR_LITMUS_SOFTIRQD % num_clusters;
+
+                int daemon = 0;
+                for(i = 0; i < num_clusters; ++i)
+                {
+                        int num_on_this_cluster = num_daemons_per_cluster;
+                        if(left_over)
+                        {
+                                ++num_on_this_cluster;
+                                --left_over;
+                        }
+
+                        for(j = 0; j < num_on_this_cluster; ++j)
+                        {
+                                // first CPU of this cluster
+                                affinity[daemon++] = i*cluster_size;
+                        }
+                }
+
+                spawn_klitirqd(affinity);
+
+                kfree(affinity);
+        }
+#endif
+
+#ifdef CONFIG_LITMUS_NVIDIA
+        init_nvidia_info();
+#endif
+
         free_cpumask_var(mask);
         clusters_allocated = 1;
         return 0;
@@ -831,6 +1791,32 @@ static struct sched_plugin cedf_plugin __cacheline_aligned_in_smp = {
         .task_block             = cedf_task_block,
         .admit_task             = cedf_admit_task,
         .activate_plugin        = cedf_activate_plugin,
+        .compare                = edf_higher_prio,
+#ifdef CONFIG_LITMUS_LOCKING
+        .allocate_lock          = cedf_allocate_lock,
+        .increase_prio          = increase_priority_inheritance,
+        .decrease_prio          = decrease_priority_inheritance,
+#endif
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+        .nested_increase_prio           = nested_increase_priority_inheritance,
+        .nested_decrease_prio           = nested_decrease_priority_inheritance,
+        .__compare                                      = __edf_higher_prio,
+#endif
+#ifdef CONFIG_LITMUS_DGL_SUPPORT
+        .get_dgl_spinlock = cedf_get_dgl_spinlock,
+#endif
+#ifdef CONFIG_LITMUS_AFFINITY_LOCKING
+        .allocate_aff_obs = cedf_allocate_affinity_observer,
+#endif
+#ifdef CONFIG_LITMUS_SOFTIRQD
+        .increase_prio_klitirqd = increase_priority_inheritance_klitirqd,
+        .decrease_prio_klitirqd = decrease_priority_inheritance_klitirqd,
+#endif
+#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
+        .enqueue_pai_tasklet = cedf_enqueue_pai_tasklet,
+        .change_prio_pai_tasklet = cedf_change_prio_pai_tasklet,
+        .run_tasklets = cedf_run_tasklets,
+#endif
 };
 
 static struct proc_dir_entry *cluster_file = NULL, *cedf_dir = NULL;