Merge branch 'wip-gpu-cleanup' into wip-2012.3-gpu

Conflicts:
	include/litmus/fpmath.h
	include/litmus/litmus.h
	include/litmus/rt_param.h
	include/litmus/trace.h
	kernel/sched.c
	kernel/softirq.c
	litmus/edf_common.c
	litmus/jobs.c
	litmus/litmus.c
	litmus/locking.c
	litmus/preempt.c
	litmus/sched_cedf.c
	litmus/sched_gsn_edf.c
	litmus/sched_litmus.c
	litmus/sync.c

1 files changed, 1061 insertions, 134 deletions

diff --git a/litmus/sched_gsn_edf.c b/litmus/sched_gsn_edf.c
index b8548b885b35..01791a18e8f3 100644
--- a/litmus/sched_gsn_edf.c
+++ b/litmus/sched_gsn_edf.c
@@ -12,24 +12,54 @@
 #include <linux/percpu.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
+#include <linux/uaccess.h>
+#include <linux/module.h>
 
 #include <litmus/litmus.h>
 #include <litmus/jobs.h>
 #include <litmus/sched_plugin.h>
 #include <litmus/edf_common.h>
 #include <litmus/sched_trace.h>
-#include <litmus/trace.h>
 
 #include <litmus/preempt.h>
 #include <litmus/budget.h>
 
 #include <litmus/bheap.h>
+#include <litmus/binheap.h>
+#include <litmus/trace.h>
+
+#ifdef CONFIG_LITMUS_LOCKING
+#include <litmus/kfmlp_lock.h>
+#endif
+
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+#include <litmus/rsm_lock.h>
+#include <litmus/ikglp_lock.h>
+#endif
 
 #ifdef CONFIG_SCHED_CPU_AFFINITY
 #include <litmus/affinity.h>
 #endif
 
-#include <linux/module.h>
+#ifdef CONFIG_REALTIME_AUX_TASKS
+#include <litmus/aux_tasks.h>
+#endif
+
+#ifdef CONFIG_LITMUS_SOFTIRQD
+#include <litmus/litmus_softirq.h>
+#endif
+
+#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
+#include <linux/interrupt.h>
+#endif
+
+#ifdef CONFIG_LITMUS_NVIDIA
+#include <litmus/nvidia_info.h>
+#endif
+
+#if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA)
+#include <litmus/gpu_affinity.h>
+#endif
 
 /* Overview of GSN-EDF operations.
  *
@@ -104,52 +134,64 @@ typedef struct  {
         int                     cpu;
         struct task_struct*     linked;         /* only RT tasks */
         struct task_struct*     scheduled;      /* only RT tasks */
-        struct bheap_node*      hn;
+        struct binheap_node hn;
 } cpu_entry_t;
 DEFINE_PER_CPU(cpu_entry_t, gsnedf_cpu_entries);
 
 cpu_entry_t* gsnedf_cpus[NR_CPUS];
 
 /* the cpus queue themselves according to priority in here */
-static struct bheap_node gsnedf_heap_node[NR_CPUS];
-static struct bheap      gsnedf_cpu_heap;
+static struct binheap gsnedf_cpu_heap;
 
 static rt_domain_t gsnedf;
 #define gsnedf_lock (gsnedf.ready_lock)
 
+#ifdef CONFIG_LITMUS_DGL_SUPPORT
+static raw_spinlock_t dgl_lock;
+
+static raw_spinlock_t* gsnedf_get_dgl_spinlock(struct task_struct *t)
+{
+        return(&dgl_lock);
+}
+#endif
+
+#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
+struct tasklet_head gsnedf_pending_tasklets;
+#endif
+
 
 /* Uncomment this if you want to see all scheduling decisions in the
  * TRACE() log.
 #define WANT_ALL_SCHED_EVENTS
  */
 
-static int cpu_lower_prio(struct bheap_node *_a, struct bheap_node *_b)
+static int cpu_lower_prio(struct binheap_node *_a, struct binheap_node *_b)
 {
-        cpu_entry_t *a, *b;
-        a = _a->value;
-        b = _b->value;
+        cpu_entry_t *a = binheap_entry(_a, cpu_entry_t, hn);
+        cpu_entry_t *b = binheap_entry(_b, cpu_entry_t, hn);
+
         /* Note that a and b are inverted: we want the lowest-priority CPU at
          * the top of the heap.
          */
         return edf_higher_prio(b->linked, a->linked);
 }
 
+
 /* update_cpu_position - Move the cpu entry to the correct place to maintain
  *                       order in the cpu queue. Caller must hold gsnedf lock.
  */
 static void update_cpu_position(cpu_entry_t *entry)
 {
-        if (likely(bheap_node_in_heap(entry->hn)))
-                bheap_delete(cpu_lower_prio, &gsnedf_cpu_heap, entry->hn);
-        bheap_insert(cpu_lower_prio, &gsnedf_cpu_heap, entry->hn);
+        if (likely(binheap_is_in_heap(&entry->hn))) {
+                binheap_delete(&entry->hn, &gsnedf_cpu_heap);
+        }
+        binheap_add(&entry->hn, &gsnedf_cpu_heap, cpu_entry_t, hn);
 }
 
 /* caller must hold gsnedf lock */
 static cpu_entry_t* lowest_prio_cpu(void)
 {
-        struct bheap_node* hn;
-        hn = bheap_peek(cpu_lower_prio, &gsnedf_cpu_heap);
-        return hn->value;
+        return binheap_top_entry(&gsnedf_cpu_heap, cpu_entry_t, hn);
 }
 
 
@@ -164,8 +206,17 @@ static noinline void link_task_to_cpu(struct task_struct* linked,
         struct task_struct* tmp;
         int on_cpu;
 
+        //int print = (linked != NULL || entry->linked != NULL);
+
         BUG_ON(linked && !is_realtime(linked));
 
+        /*
+        if (print) {
+                TRACE_CUR("linked = %s/%d\n", (linked) ? linked->comm : "(nil)", (linked)? linked->pid : 0);
+                TRACE_CUR("entry->linked = %s/%d\n", (entry->linked) ? entry->linked->comm : "(nil)", (entry->linked)? entry->linked->pid : 0);
+        }
+        */
+
         /* Currently linked task is set to be unlinked. */
         if (entry->linked) {
                 entry->linked->rt_param.linked_on = NO_CPU;
@@ -201,12 +252,18 @@ static noinline void link_task_to_cpu(struct task_struct* linked,
                         linked->rt_param.linked_on = entry->cpu;
         }
         entry->linked = linked;
-#ifdef WANT_ALL_SCHED_EVENTS
-        if (linked)
-                TRACE_TASK(linked, "linked to %d.\n", entry->cpu);
-        else
-                TRACE("NULL linked to %d.\n", entry->cpu);
-#endif
+
+        /*
+        if (print) {
+                //#ifdef WANT_ALL_SCHED_EVENTS
+                if (linked)
+                        TRACE_TASK(linked, "linked to %d.\n", entry->cpu);
+                else
+                        TRACE("NULL linked to %d.\n", entry->cpu);
+                //#endif
+        }
+        */
+
         update_cpu_position(entry);
 }
 
@@ -251,8 +308,17 @@ static noinline void requeue(struct task_struct* task)
         /* sanity check before insertion */
         BUG_ON(is_queued(task));
 
-        if (is_released(task, litmus_clock()))
-                __add_ready(&gsnedf, task);
+        if (is_released(task, litmus_clock())) {
+#ifdef CONFIG_REALTIME_AUX_TASKS
+                if (unlikely(tsk_rt(task)->is_aux_task && !is_running(task))) {
+                        /* aux_task probably transitioned to real-time while it was blocked */
+                        TRACE_CUR("aux task %s/%d is not ready!\n", task->comm, task->pid);
+                        unlink(task); /* really needed? */
+                }
+                else
+#endif
+                        __add_ready(&gsnedf, task);
+        }
         else {
                 /* it has got to wait */
                 add_release(&gsnedf, task);
@@ -326,6 +392,7 @@ static void gsnedf_release_jobs(rt_domain_t* rt, struct bheap* tasks)
         raw_spin_lock_irqsave(&gsnedf_lock, flags);
 
         __merge_ready(rt, tasks);
+
         check_for_preemptions();
 
         raw_spin_unlock_irqrestore(&gsnedf_lock, flags);
@@ -338,12 +405,17 @@ static noinline void job_completion(struct task_struct *t, int forced)
 
         sched_trace_task_completion(t, forced);
 
+#ifdef CONFIG_LITMUS_NVIDIA
+        atomic_set(&tsk_rt(t)->nv_int_count, 0);
+#endif
+
         TRACE_TASK(t, "job_completion().\n");
 
         /* set flags */
         tsk_rt(t)->completed = 1;
         /* prepare for next period */
         prepare_for_next_period(t);
+
         if (is_released(t, litmus_clock()))
                 sched_trace_task_release(t);
         /* unlink */
@@ -362,24 +434,350 @@ static noinline void job_completion(struct task_struct *t, int forced)
  */
 static void gsnedf_tick(struct task_struct* t)
 {
-        if (is_realtime(t) && budget_enforced(t) && budget_exhausted(t)) {
-                if (!is_np(t)) {
-                        /* np tasks will be preempted when they become
-                         * preemptable again
-                         */
-                        litmus_reschedule_local();
-                        TRACE("gsnedf_scheduler_tick: "
-                              "%d is preemptable "
-                              " => FORCE_RESCHED\n", t->pid);
-                } else if (is_user_np(t)) {
-                        TRACE("gsnedf_scheduler_tick: "
-                              "%d is non-preemptable, "
-                              "preemption delayed.\n", t->pid);
-                        request_exit_np(t);
+        if (is_realtime(t) && budget_exhausted(t))
+        {
+                if (budget_signalled(t) && !sigbudget_sent(t)) {
+                        /* signal exhaustion */
+                        send_sigbudget(t);
+                }
+
+                if (budget_enforced(t)) {
+                        if (!is_np(t)) {
+                                /* np tasks will be preempted when they become
+                                 * preemptable again
+                                 */
+                                litmus_reschedule_local();
+                                TRACE("gsnedf_scheduler_tick: "
+                                          "%d is preemptable "
+                                          " => FORCE_RESCHED\n", t->pid);
+                        } else if (is_user_np(t)) {
+                                TRACE("gsnedf_scheduler_tick: "
+                                          "%d is non-preemptable, "
+                                          "preemption delayed.\n", t->pid);
+                                request_exit_np(t);
+                        }
+                }
+        }
+
+        /*
+        if(is_realtime(t)) {
+                TRACE_TASK(t, "tick %llu\n", litmus_clock());
+        }
+         */
+}
+
+
+
+
+#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
+
+
+static void __do_lit_tasklet(struct tasklet_struct* tasklet, unsigned long flushed)
+{
+        if (!atomic_read(&tasklet->count)) {
+                if(tasklet->owner) {
+                        sched_trace_tasklet_begin(tasklet->owner);
+                }
+
+                if (!test_and_clear_bit(TASKLET_STATE_SCHED, &tasklet->state))
+                {
+                        BUG();
+                }
+                TRACE("%s: Invoking tasklet with owner pid = %d (flushed = %d).\n",
+                          __FUNCTION__,
+                          (tasklet->owner) ? tasklet->owner->pid : -1,
+                          (tasklet->owner) ? 0 : 1);
+                tasklet->func(tasklet->data);
+                tasklet_unlock(tasklet);
+
+                if(tasklet->owner) {
+                        sched_trace_tasklet_end(tasklet->owner, flushed);
+                }
+        }
+        else {
+                BUG();
+        }
+}
+
+static void do_lit_tasklets(struct task_struct* sched_task)
+{
+        int work_to_do = 1;
+        struct tasklet_struct *tasklet = NULL;
+        unsigned long flags;
+
+        while(work_to_do) {
+
+                TS_NV_SCHED_BOTISR_START;
+
+                // execute one tasklet that has higher priority
+                raw_spin_lock_irqsave(&gsnedf_lock, flags);
+
+                if(gsnedf_pending_tasklets.head != NULL) {
+                        struct tasklet_struct *prev = NULL;
+                        tasklet = gsnedf_pending_tasklets.head;
+
+                        while(tasklet && edf_higher_prio(sched_task, tasklet->owner)) {
+                                prev = tasklet;
+                                tasklet = tasklet->next;
+                        }
+
+                        // remove the tasklet from the queue
+                        if(prev) {
+                                prev->next = tasklet->next;
+                                if(prev->next == NULL) {
+                                        TRACE("%s: Tasklet for %d is the last element in tasklet queue.\n", __FUNCTION__, tasklet->owner->pid);
+                                        gsnedf_pending_tasklets.tail = &(prev);
+                                }
+                        }
+                        else {
+                                gsnedf_pending_tasklets.head = tasklet->next;
+                                if(tasklet->next == NULL) {
+                                        TRACE("%s: Tasklet for %d is the last element in tasklet queue.\n", __FUNCTION__, tasklet->owner->pid);
+                                        gsnedf_pending_tasklets.tail = &(gsnedf_pending_tasklets.head);
+                                }
+                        }
+                }
+                else {
+                        TRACE("%s: Tasklet queue is empty.\n", __FUNCTION__);
+                }
+
+                raw_spin_unlock_irqrestore(&gsnedf_lock, flags);
+
+                if(tasklet) {
+                        __do_lit_tasklet(tasklet, 0ul);
+                        tasklet = NULL;
+                }
+                else {
+                        work_to_do = 0;
+                }
+
+                TS_NV_SCHED_BOTISR_END;
+        }
+}
+
+//static void do_lit_tasklets(struct task_struct* sched_task)
+//{
+//      int work_to_do = 1;
+//      struct tasklet_struct *tasklet = NULL;
+//      //struct tasklet_struct *step;
+//      unsigned long flags;
+//
+//      while(work_to_do) {
+//
+//              TS_NV_SCHED_BOTISR_START;
+//
+//              // remove tasklet at head of list if it has higher priority.
+//              raw_spin_lock_irqsave(&gsnedf_lock, flags);
+//
+//              if(gsnedf_pending_tasklets.head != NULL) {
+//                      // remove tasklet at head.
+//                      tasklet = gsnedf_pending_tasklets.head;
+//
+//                      if(edf_higher_prio(tasklet->owner, sched_task)) {
+//
+//                              if(NULL == tasklet->next) {
+//                                      // tasklet is at the head, list only has one element
+//                                      TRACE("%s: Tasklet for %d is the last element in tasklet queue.\n", __FUNCTION__, tasklet->owner->pid);
+//                                      gsnedf_pending_tasklets.tail = &(gsnedf_pending_tasklets.head);
+//                              }
+//
+//                              // remove the tasklet from the queue
+//                              gsnedf_pending_tasklets.head = tasklet->next;
+//
+//                              TRACE("%s: Removed tasklet for %d from tasklet queue.\n", __FUNCTION__, tasklet->owner->pid);
+//                      }
+//                      else {
+//                              TRACE("%s: Pending tasklet (%d) does not have priority to run on this CPU (%d).\n", __FUNCTION__, tasklet->owner->pid, smp_processor_id());
+//                              tasklet = NULL;
+//                      }
+//              }
+//              else {
+//                      TRACE("%s: Tasklet queue is empty.\n", __FUNCTION__);
+//              }
+//
+//              raw_spin_unlock_irqrestore(&gsnedf_lock, flags);
+//
+//              TS_NV_SCHED_BOTISR_END;
+//
+//              if(tasklet) {
+//                      __do_lit_tasklet(tasklet, 0ul);
+//                      tasklet = NULL;
+//              }
+//              else {
+//                      work_to_do = 0;
+//              }
+//      }
+//
+//      //TRACE("%s: exited.\n", __FUNCTION__);
+//}
+
+static void __add_pai_tasklet(struct tasklet_struct* tasklet)
+{
+        struct tasklet_struct* step;
+
+        tasklet->next = NULL;  // make sure there are no old values floating around
+
+        step = gsnedf_pending_tasklets.head;
+        if(step == NULL) {
+                TRACE("%s: tasklet queue empty.  inserting tasklet for %d at head.\n", __FUNCTION__, tasklet->owner->pid);
+                // insert at tail.
+                *(gsnedf_pending_tasklets.tail) = tasklet;
+                gsnedf_pending_tasklets.tail = &(tasklet->next);
+        }
+        else if((*(gsnedf_pending_tasklets.tail) != NULL) &&
+                        edf_higher_prio((*(gsnedf_pending_tasklets.tail))->owner, tasklet->owner)) {
+                // insert at tail.
+                TRACE("%s: tasklet belongs at end.  inserting tasklet for %d at tail.\n", __FUNCTION__, tasklet->owner->pid);
+
+                *(gsnedf_pending_tasklets.tail) = tasklet;
+                gsnedf_pending_tasklets.tail = &(tasklet->next);
+        }
+        else {
+                // insert the tasklet somewhere in the middle.
+
+        TRACE("%s: tasklet belongs somewhere in the middle.\n", __FUNCTION__);
+
+                while(step->next && edf_higher_prio(step->next->owner, tasklet->owner)) {
+                        step = step->next;
+                }
+
+                // insert tasklet right before step->next.
+
+                TRACE("%s: inserting tasklet for %d between %d and %d.\n", __FUNCTION__, tasklet->owner->pid, step->owner->pid, (step->next) ? step->next->owner->pid : -1);
+
+                tasklet->next = step->next;
+                step->next = tasklet;
+
+                // patch up the head if needed.
+                if(gsnedf_pending_tasklets.head == step)
+                {
+                        TRACE("%s: %d is the new tasklet queue head.\n", __FUNCTION__, tasklet->owner->pid);
+                        gsnedf_pending_tasklets.head = tasklet;
+                }
+        }
+}
+
+static void gsnedf_run_tasklets(struct task_struct* sched_task)
+{
+        preempt_disable();
+
+        if(gsnedf_pending_tasklets.head != NULL) {
+                TRACE("%s: There are tasklets to process.\n", __FUNCTION__);
+                do_lit_tasklets(sched_task);
+        }
+
+        preempt_enable_no_resched();
+}
+
+static int gsnedf_enqueue_pai_tasklet(struct tasklet_struct* tasklet)
+{
+        cpu_entry_t *targetCPU = NULL;
+        int thisCPU;
+        int runLocal = 0;
+        int runNow = 0;
+        unsigned long flags;
+
+    if(unlikely((tasklet->owner == NULL) || !is_realtime(tasklet->owner)))
+    {
+        TRACE("%s: No owner associated with this tasklet!\n", __FUNCTION__);
+                return 0;
+    }
+
+
+        raw_spin_lock_irqsave(&gsnedf_lock, flags);
+
+        thisCPU = smp_processor_id();
+
+#ifdef CONFIG_SCHED_CPU_AFFINITY
+        {
+                cpu_entry_t* affinity = NULL;
+
+                // use this CPU if it is in our cluster and isn't running any RT work.
+                if(
+#ifdef CONFIG_RELEASE_MASTER
+                   (thisCPU != gsnedf.release_master) &&
+#endif
+                   (__get_cpu_var(gsnedf_cpu_entries).linked == NULL)) {
+                        affinity = &(__get_cpu_var(gsnedf_cpu_entries));
+                }
+                else {
+                        // this CPU is busy or shouldn't run tasklet in this cluster.
+                        // look for available near by CPUs.
+                        // NOTE: Affinity towards owner and not this CPU.  Is this right?
+                        affinity =
+                                gsnedf_get_nearest_available_cpu(
+                                        &per_cpu(gsnedf_cpu_entries, task_cpu(tasklet->owner)));
+                }
+
+                targetCPU = affinity;
+        }
+#endif
+
+        if (targetCPU == NULL) {
+                targetCPU = lowest_prio_cpu();
+        }
+
+        if (edf_higher_prio(tasklet->owner, targetCPU->linked)) {
+                if (thisCPU == targetCPU->cpu) {
+                        TRACE("%s: Run tasklet locally (and now).\n", __FUNCTION__);
+                        runLocal = 1;
+                        runNow = 1;
+                }
+                else {
+                        TRACE("%s: Run tasklet remotely (and now).\n", __FUNCTION__);
+                        runLocal = 0;
+                        runNow = 1;
+                }
+        }
+        else {
+                runLocal = 0;
+                runNow = 0;
+        }
+
+        if(!runLocal) {
+                // enqueue the tasklet
+                __add_pai_tasklet(tasklet);
+        }
+
+        raw_spin_unlock_irqrestore(&gsnedf_lock, flags);
+
+
+        if (runLocal /*&& runNow */) {  // runNow == 1 is implied
+                TRACE("%s: Running tasklet on CPU where it was received.\n", __FUNCTION__);
+                __do_lit_tasklet(tasklet, 0ul);
+        }
+        else if (runNow /*&& !runLocal */) {  // runLocal == 0 is implied
+                TRACE("%s: Triggering CPU %d to run tasklet.\n", __FUNCTION__, targetCPU->cpu);
+                preempt(targetCPU);  // need to be protected by cedf_lock?
+        }
+        else {
+                TRACE("%s: Scheduling of tasklet was deferred.\n", __FUNCTION__);
+        }
+
+        return(1); // success
+}
+
+static void gsnedf_change_prio_pai_tasklet(struct task_struct *old_prio,
+                                                                                   struct task_struct *new_prio)
+{
+        struct tasklet_struct* step;
+        unsigned long flags;
+
+        if(gsnedf_pending_tasklets.head != NULL) {
+                raw_spin_lock_irqsave(&gsnedf_lock, flags);
+                for(step = gsnedf_pending_tasklets.head; step != NULL; step = step->next) {
+                        if(step->owner == old_prio) {
+                                TRACE("%s: Found tasklet to change: %d\n", __FUNCTION__, step->owner->pid);
+                                step->owner = new_prio;
+                        }
                 }
+                raw_spin_unlock_irqrestore(&gsnedf_lock, flags);
         }
 }
 
+#endif  // end PAI
+
+
 /* Getting schedule() right is a bit tricky. schedule() may not make any
  * assumptions on the state of the current task since it may be called for a
  * number of reasons. The reasons include a scheduler_tick() determined that it
@@ -404,9 +802,11 @@ static void gsnedf_tick(struct task_struct* t)
 static struct task_struct* gsnedf_schedule(struct task_struct * prev)
 {
         cpu_entry_t* entry = &__get_cpu_var(gsnedf_cpu_entries);
-        int out_of_time, sleep, preempt, np, exists, blocks;
+        int out_of_time, signal_budget, sleep, preempt, np, exists, blocks;
         struct task_struct* next = NULL;
 
+        //int completion = 0;
+
 #ifdef CONFIG_RELEASE_MASTER
         /* Bail out early if we are the release master.
          * The release master never schedules any real-time tasks.
@@ -427,8 +827,13 @@ static struct task_struct* gsnedf_schedule(struct task_struct * prev)
         /* (0) Determine state */
         exists      = entry->scheduled != NULL;
         blocks      = exists && !is_running(entry->scheduled);
-        out_of_time = exists && budget_enforced(entry->scheduled)
-                && budget_exhausted(entry->scheduled);
+        out_of_time = exists &&
+                budget_enforced(entry->scheduled) &&
+                budget_exhausted(entry->scheduled);
+        signal_budget = exists &&
+                budget_signalled(entry->scheduled) &&
+                budget_exhausted(entry->scheduled) &&
+                !sigbudget_sent(entry->scheduled);
         np          = exists && is_np(entry->scheduled);
         sleep       = exists && is_completed(entry->scheduled);
         preempt     = entry->scheduled != entry->linked;
@@ -437,21 +842,36 @@ static struct task_struct* gsnedf_schedule(struct task_struct * prev)
         TRACE_TASK(prev, "invoked gsnedf_schedule.\n");
 #endif
 
-        if (exists)
+        if (exists) {
                 TRACE_TASK(prev,
-                           "blocks:%d out_of_time:%d np:%d sleep:%d preempt:%d "
+                           "blocks:%d out_of_time:%d signal_budget: %d np:%d sleep:%d preempt:%d "
                            "state:%d sig:%d\n",
-                           blocks, out_of_time, np, sleep, preempt,
+                           blocks, out_of_time, signal_budget, np, sleep, preempt,
                            prev->state, signal_pending(prev));
+        }
+
         if (entry->linked && preempt)
                 TRACE_TASK(prev, "will be preempted by %s/%d\n",
                            entry->linked->comm, entry->linked->pid);
 
+        /* Send the signal that the budget has been exhausted */
+        if (signal_budget) {
+                send_sigbudget(entry->scheduled);
+        }
 
         /* If a task blocks we have no choice but to reschedule.
          */
-        if (blocks)
+        if (blocks) {
                 unlink(entry->scheduled);
+        }
+
+#if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_AFFINITY_LOCKING)
+        if(exists && is_realtime(entry->scheduled) && tsk_rt(entry->scheduled)->held_gpus) {
+                if(!blocks || tsk_rt(entry->scheduled)->suspend_gpu_tracker_on_block) {
+                        stop_gpu_tracker(entry->scheduled);
+                }
+        }
+#endif
 
         /* Request a sys_exit_np() call if we would like to preempt but cannot.
          * We need to make sure to update the link structure anyway in case
@@ -468,8 +888,10 @@ static struct task_struct* gsnedf_schedule(struct task_struct * prev)
          * this. Don't do a job completion if we block (can't have timers running
          * for blocked jobs).
          */
-        if (!np && (out_of_time || sleep) && !blocks)
+        if (!np && (out_of_time || sleep) && !blocks) {
                 job_completion(entry->scheduled, !sleep);
+                //completion = 1;
+        }
 
         /* Link pending task if we became unlinked.
          */
@@ -492,12 +914,21 @@ static struct task_struct* gsnedf_schedule(struct task_struct * prev)
                         entry->scheduled->rt_param.scheduled_on = NO_CPU;
                         TRACE_TASK(entry->scheduled, "scheduled_on = NO_CPU\n");
                 }
-        } else
+        }
+        else
+        {
                 /* Only override Linux scheduler if we have a real-time task
                  * scheduled that needs to continue.
                  */
                 if (exists)
                         next = prev;
+        }
+
+#if 0
+        if (completion) {
+                TRACE_CUR("switching away from a completion\n");
+        }
+#endif
 
         sched_state_task_picked();
 
@@ -512,7 +943,6 @@ static struct task_struct* gsnedf_schedule(struct task_struct * prev)
                 TRACE("becomes idle at %llu.\n", litmus_clock());
 #endif
 
-
         return next;
 }
 
@@ -524,6 +954,7 @@ static void gsnedf_finish_switch(struct task_struct *prev)
         cpu_entry_t*    entry = &__get_cpu_var(gsnedf_cpu_entries);
 
         entry->scheduled = is_realtime(current) ? current : NULL;
+
 #ifdef WANT_ALL_SCHED_EVENTS
         TRACE_TASK(prev, "switched away from\n");
 #endif
@@ -537,7 +968,7 @@ static void gsnedf_task_new(struct task_struct * t, int on_rq, int running)
         unsigned long           flags;
         cpu_entry_t*            entry;
 
-        TRACE("gsn edf: task new %d\n", t->pid);
+        TRACE("gsn edf: task new = %d on_rq = %d running = %d\n", t->pid, on_rq, running);
 
         raw_spin_lock_irqsave(&gsnedf_lock, flags);
 
@@ -572,11 +1003,14 @@ static void gsnedf_task_new(struct task_struct * t, int on_rq, int running)
 static void gsnedf_task_wake_up(struct task_struct *task)
 {
         unsigned long flags;
-        lt_t now;
+        //lt_t now;
 
         TRACE_TASK(task, "wake_up at %llu\n", litmus_clock());
 
         raw_spin_lock_irqsave(&gsnedf_lock, flags);
+
+#if 0
+        /* sporadic task model. will increment job numbers automatically */
         now = litmus_clock();
         if (is_tardy(task, now)) {
                 /* new sporadic release */
@@ -590,6 +1024,25 @@ static void gsnedf_task_wake_up(struct task_struct *task)
                         tsk_rt(task)->completed = 0;
                 }
         }
+#else
+        /* don't force job to end.  rely on user to say when jobs complete */
+        tsk_rt(task)->completed = 0;
+#endif
+
+#ifdef CONFIG_REALTIME_AUX_TASKS
+        if (tsk_rt(task)->has_aux_tasks && !tsk_rt(task)->hide_from_aux_tasks) {
+                TRACE_CUR("%s/%d is ready so aux tasks may not inherit.\n", task->comm, task->pid);
+                disable_aux_task_owner(task);
+        }
+#endif
+
+#ifdef CONFIG_LITMUS_NVIDIA
+        if (tsk_rt(task)->held_gpus && !tsk_rt(task)->hide_from_gpu) {
+                TRACE_CUR("%s/%d is ready so gpu klmirqd tasks may not inherit.\n", task->comm, task->pid);
+                disable_gpu_owner(task);
+        }
+#endif
+
         gsnedf_job_arrival(task);
         raw_spin_unlock_irqrestore(&gsnedf_lock, flags);
 }
@@ -602,7 +1055,25 @@ static void gsnedf_task_block(struct task_struct *t)
 
         /* unlink if necessary */
         raw_spin_lock_irqsave(&gsnedf_lock, flags);
+
         unlink(t);
+
+#ifdef CONFIG_REALTIME_AUX_TASKS
+        if (tsk_rt(t)->has_aux_tasks && !tsk_rt(t)->hide_from_aux_tasks) {
+
+                TRACE_CUR("%s/%d is blocked so aux tasks may inherit.\n", t->comm, t->pid);
+                enable_aux_task_owner(t);
+        }
+#endif
+
+#ifdef CONFIG_LITMUS_NVIDIA
+        if (tsk_rt(t)->held_gpus && !tsk_rt(t)->hide_from_gpu) {
+
+                TRACE_CUR("%s/%d is blocked so aux tasks may inherit.\n", t->comm, t->pid);
+                enable_gpu_owner(t);
+        }
+#endif
+
         raw_spin_unlock_irqrestore(&gsnedf_lock, flags);
 
         BUG_ON(!is_realtime(t));
@@ -613,8 +1084,30 @@ static void gsnedf_task_exit(struct task_struct * t)
 {
         unsigned long flags;
 
+#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
+        gsnedf_change_prio_pai_tasklet(t, NULL);
+#endif
+
         /* unlink if necessary */
         raw_spin_lock_irqsave(&gsnedf_lock, flags);
+
+#ifdef CONFIG_REALTIME_AUX_TASKS
+        /* make sure we clean up on our way out */
+        if (unlikely(tsk_rt(t)->is_aux_task)) {
+                exit_aux_task(t);
+        }
+        else if(tsk_rt(t)->has_aux_tasks) {
+                disable_aux_task_owner(t);
+        }
+#endif
+
+#ifdef CONFIG_LITMUS_NVIDIA
+        /* make sure we clean up on our way out */
+        if(tsk_rt(t)->held_gpus) {
+                disable_gpu_owner(t);
+        }
+#endif
+
         unlink(t);
         if (tsk_rt(t)->scheduled_on != NO_CPU) {
                 gsnedf_cpus[tsk_rt(t)->scheduled_on]->scheduled = NULL;
@@ -623,106 +1116,413 @@ static void gsnedf_task_exit(struct task_struct * t)
         raw_spin_unlock_irqrestore(&gsnedf_lock, flags);
 
         BUG_ON(!is_realtime(t));
-        TRACE_TASK(t, "RIP\n");
+        TRACE_TASK(t, "RIP\n");
 }
 
 
 static long gsnedf_admit_task(struct task_struct* tsk)
 {
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+        INIT_BINHEAP_HANDLE(&tsk_rt(tsk)->hp_blocked_tasks,
+                                                edf_max_heap_base_priority_order);
+#endif
+
         return 0;
 }
 
+
+
+
+
+
 #ifdef CONFIG_LITMUS_LOCKING
 
 #include <litmus/fdso.h>
 
 /* called with IRQs off */
-static void set_priority_inheritance(struct task_struct* t, struct task_struct* prio_inh)
+static int __increase_priority_inheritance(struct task_struct* t,
+                                                                                    struct task_struct* prio_inh)
 {
+        int success = 1;
         int linked_on;
         int check_preempt = 0;
 
-        raw_spin_lock(&gsnedf_lock);
+        if (prio_inh && prio_inh == effective_priority(t)) {
+                /* relationship already established. */
+                TRACE_TASK(t, "already has effective priority of %s/%d\n",
+                                        prio_inh->comm, prio_inh->pid);
+                goto out;
+        }
 
-        TRACE_TASK(t, "inherits priority from %s/%d\n", prio_inh->comm, prio_inh->pid);
-        tsk_rt(t)->inh_task = prio_inh;
-
-        linked_on  = tsk_rt(t)->linked_on;
-
-        /* If it is scheduled, then we need to reorder the CPU heap. */
-        if (linked_on != NO_CPU) {
-                TRACE_TASK(t, "%s: linked  on %d\n",
-                           __FUNCTION__, linked_on);
-                /* Holder is scheduled; need to re-order CPUs.
-                 * We can't use heap_decrease() here since
-                 * the cpu_heap is ordered in reverse direction, so
-                 * it is actually an increase. */
-                bheap_delete(cpu_lower_prio, &gsnedf_cpu_heap,
-                            gsnedf_cpus[linked_on]->hn);
-                bheap_insert(cpu_lower_prio, &gsnedf_cpu_heap,
-                            gsnedf_cpus[linked_on]->hn);
-        } else {
-                /* holder may be queued: first stop queue changes */
-                raw_spin_lock(&gsnedf.release_lock);
-                if (is_queued(t)) {
-                        TRACE_TASK(t, "%s: is queued\n",
-                                   __FUNCTION__);
-                        /* We need to update the position of holder in some
-                         * heap. Note that this could be a release heap if we
-                         * budget enforcement is used and this job overran. */
-                        check_preempt =
-                                !bheap_decrease(edf_ready_order,
-                                               tsk_rt(t)->heap_node);
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+        /* this sanity check allows for weaker locking in protocols */
+        if(__edf_higher_prio(prio_inh, BASE, t, EFFECTIVE)) {
+#endif
+                TRACE_TASK(t, "inherits priority from %s/%d\n",
+                                   prio_inh->comm, prio_inh->pid);
+                tsk_rt(t)->inh_task = prio_inh;
+
+                linked_on  = tsk_rt(t)->linked_on;
+
+                /* If it is scheduled, then we need to reorder the CPU heap. */
+                if (linked_on != NO_CPU) {
+                        TRACE_TASK(t, "%s: linked  on %d\n",
+                                   __FUNCTION__, linked_on);
+                        /* Holder is scheduled; need to re-order CPUs.
+                         * We can't use heap_decrease() here since
+                         * the cpu_heap is ordered in reverse direction, so
+                         * it is actually an increase. */
+                        binheap_delete(&gsnedf_cpus[linked_on]->hn, &gsnedf_cpu_heap);
+                        binheap_add(&gsnedf_cpus[linked_on]->hn,
+                                        &gsnedf_cpu_heap, cpu_entry_t, hn);
                 } else {
-                        /* Nothing to do: if it is not queued and not linked
-                         * then it is either sleeping or currently being moved
-                         * by other code (e.g., a timer interrupt handler) that
-                         * will use the correct priority when enqueuing the
-                         * task. */
-                        TRACE_TASK(t, "%s: is NOT queued => Done.\n",
-                                   __FUNCTION__);
-                }
-                raw_spin_unlock(&gsnedf.release_lock);
-
-                /* If holder was enqueued in a release heap, then the following
-                 * preemption check is pointless, but we can't easily detect
-                 * that case. If you want to fix this, then consider that
-                 * simply adding a state flag requires O(n) time to update when
-                 * releasing n tasks, which conflicts with the goal to have
-                 * O(log n) merges. */
-                if (check_preempt) {
-                        /* heap_decrease() hit the top level of the heap: make
-                         * sure preemption checks get the right task, not the
-                         * potentially stale cache. */
-                        bheap_uncache_min(edf_ready_order,
-                                         &gsnedf.ready_queue);
-                        check_for_preemptions();
+                        /* holder may be queued: first stop queue changes */
+                        raw_spin_lock(&gsnedf.release_lock);
+                        if (is_queued(t)) {
+                                TRACE_TASK(t, "%s: is queued\n",
+                                           __FUNCTION__);
+                                /* We need to update the position of holder in some
+                                 * heap. Note that this could be a release heap if we
+                                 * budget enforcement is used and this job overran. */
+                                check_preempt =
+                                        !bheap_decrease(edf_ready_order,
+                                                           tsk_rt(t)->heap_node);
+                        } else {
+                                /* Nothing to do: if it is not queued and not linked
+                                 * then it is either sleeping or currently being moved
+                                 * by other code (e.g., a timer interrupt handler) that
+                                 * will use the correct priority when enqueuing the
+                                 * task. */
+                                TRACE_TASK(t, "%s: is NOT queued => Done.\n",
+                                           __FUNCTION__);
+                        }
+                        raw_spin_unlock(&gsnedf.release_lock);
+
+                        /* If holder was enqueued in a release heap, then the following
+                         * preemption check is pointless, but we can't easily detect
+                         * that case. If you want to fix this, then consider that
+                         * simply adding a state flag requires O(n) time to update when
+                         * releasing n tasks, which conflicts with the goal to have
+                         * O(log n) merges. */
+                        if (check_preempt) {
+                                /* heap_decrease() hit the top level of the heap: make
+                                 * sure preemption checks get the right task, not the
+                                 * potentially stale cache. */
+                                bheap_uncache_min(edf_ready_order,
+                                                 &gsnedf.ready_queue);
+                                check_for_preemptions();
+                        }
+
+#ifdef CONFIG_REALTIME_AUX_TASKS
+                        /* propagate to aux tasks */
+                        if (tsk_rt(t)->has_aux_tasks) {
+                                aux_task_owner_increase_priority(t);
+                        }
+#endif
+
+#ifdef CONFIG_LITMUS_NVIDIA
+                        /* propagate to gpu klmirqd */
+                        if (tsk_rt(t)->held_gpus) {
+                                gpu_owner_increase_priority(t);
+                        }
+#endif
+
                 }
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+        }
+        else {
+                TRACE_TASK(t, "Spurious invalid priority increase. "
+                                          "Inheritance request: %s/%d [eff_prio = %s/%d] to inherit from %s/%d\n"
+                                          "Occurance is likely okay: probably due to (hopefully safe) concurrent priority updates.\n",
+                                   t->comm, t->pid,
+                                   effective_priority(t)->comm, effective_priority(t)->pid,
+                                   (prio_inh) ? prio_inh->comm : "nil",
+                                   (prio_inh) ? prio_inh->pid : -1);
+                WARN_ON(!prio_inh);
+                success = 0;
         }
+#endif
 
-        raw_spin_unlock(&gsnedf_lock);
+out:
+        return success;
 }
 
 /* called with IRQs off */
-static void clear_priority_inheritance(struct task_struct* t)
+static void increase_priority_inheritance(struct task_struct* t, struct task_struct* prio_inh)
 {
+        int success;
+
         raw_spin_lock(&gsnedf_lock);
 
-        /* A job only stops inheriting a priority when it releases a
-         * resource. Thus we can make the following assumption.*/
-        BUG_ON(tsk_rt(t)->scheduled_on == NO_CPU);
+        success = __increase_priority_inheritance(t, prio_inh);
 
-        TRACE_TASK(t, "priority restored\n");
-        tsk_rt(t)->inh_task = NULL;
+        raw_spin_unlock(&gsnedf_lock);
 
-        /* Check if rescheduling is necessary. We can't use heap_decrease()
-         * since the priority was effectively lowered. */
-        unlink(t);
-        gsnedf_job_arrival(t);
+#if defined(CONFIG_LITMUS_PAI_SOFTIRQD) && defined(CONFIG_LITMUS_NVIDIA)
+        if(tsk_rt(t)->held_gpus) {
+                int i;
+                for(i = find_first_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus));
+                        i < NV_DEVICE_NUM;
+                        i = find_next_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus), i+1)) {
+                        pai_check_priority_increase(t, i);
+                }
+        }
+#endif
+}
+
+
+/* called with IRQs off */
+static int __decrease_priority_inheritance(struct task_struct* t,
+                                                                                        struct task_struct* prio_inh)
+{
+        int success = 1;
+
+        if (prio_inh == tsk_rt(t)->inh_task) {
+                /* relationship already established. */
+                TRACE_TASK(t, "already inherits priority from %s/%d\n",
+                                   (prio_inh) ? prio_inh->comm : "(nil)",
+                                   (prio_inh) ? prio_inh->pid : 0);
+                goto out;
+        }
+
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+        if(__edf_higher_prio(t, EFFECTIVE, prio_inh, BASE)) {
+#endif
+                /* A job only stops inheriting a priority when it releases a
+                 * resource. Thus we can make the following assumption.*/
+                if(prio_inh)
+                        TRACE_TASK(t, "EFFECTIVE priority decreased to %s/%d\n",
+                                           prio_inh->comm, prio_inh->pid);
+                else
+                        TRACE_TASK(t, "base priority restored.\n");
+
+                tsk_rt(t)->inh_task = prio_inh;
+
+                if(tsk_rt(t)->scheduled_on != NO_CPU) {
+                        TRACE_TASK(t, "is scheduled.\n");
+
+                        /* Check if rescheduling is necessary. We can't use heap_decrease()
+                         * since the priority was effectively lowered. */
+                        unlink(t);
+                        gsnedf_job_arrival(t);
+                }
+                else {
+                        /* task is queued */
+                        raw_spin_lock(&gsnedf.release_lock);
+                        if (is_queued(t)) {
+                                TRACE_TASK(t, "is queued.\n");
+
+                                /* decrease in priority, so we have to re-add to binomial heap */
+                                unlink(t);
+                                gsnedf_job_arrival(t);
+                        }
+                        else {
+                                TRACE_TASK(t, "is not in scheduler. Probably on wait queue somewhere.\n");
+                        }
+                        raw_spin_unlock(&gsnedf.release_lock);
+                }
+
+#ifdef CONFIG_REALTIME_AUX_TASKS
+                /* propagate to aux tasks */
+                if (tsk_rt(t)->has_aux_tasks) {
+                        aux_task_owner_decrease_priority(t);
+                }
+#endif
+
+#ifdef CONFIG_LITMUS_NVIDIA
+                /* propagate to gpu */
+                if (tsk_rt(t)->held_gpus) {
+                        gpu_owner_decrease_priority(t);
+                }
+#endif
+
+
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+        }
+        else {
+                TRACE_TASK(t, "Spurious invalid priority decrease. "
+                                   "Inheritance request: %s/%d [eff_prio = %s/%d] to inherit from %s/%d\n"
+                                   "Occurance is likely okay: probably due to (hopefully safe) concurrent priority updates.\n",
+                                   t->comm, t->pid,
+                                   effective_priority(t)->comm, effective_priority(t)->pid,
+                                   (prio_inh) ? prio_inh->comm : "nil",
+                                   (prio_inh) ? prio_inh->pid : -1);
+                success = 0;
+        }
+#endif
+
+out:
+        return success;
+}
+
+static void decrease_priority_inheritance(struct task_struct* t,
+                                                                                  struct task_struct* prio_inh)
+{
+        int success;
+
+        raw_spin_lock(&gsnedf_lock);
+
+        success = __decrease_priority_inheritance(t, prio_inh);
 
         raw_spin_unlock(&gsnedf_lock);
+
+#if defined(CONFIG_LITMUS_PAI_SOFTIRQD) && defined(CONFIG_LITMUS_NVIDIA)
+        if(tsk_rt(t)->held_gpus) {
+                int i;
+                for(i = find_first_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus));
+                        i < NV_DEVICE_NUM;
+                        i = find_next_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus), i+1)) {
+                        pai_check_priority_decrease(t, i);
+                }
+        }
+#endif
+}
+
+
+
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+
+/* called with IRQs off */
+/* preconditions:
+ (1) The 'hp_blocked_tasks_lock' of task 't' is held.
+ (2) The lock 'to_unlock' is held.
+ */
+static void nested_increase_priority_inheritance(struct task_struct* t,
+                                                                                                 struct task_struct* prio_inh,
+                                                                                                 raw_spinlock_t *to_unlock,
+                                                                                                 unsigned long irqflags)
+{
+        struct litmus_lock *blocked_lock = tsk_rt(t)->blocked_lock;
+
+        if(tsk_rt(t)->inh_task != prio_inh) {           // shield redundent calls.
+                increase_priority_inheritance(t, prio_inh);  // increase our prio.
+        }
+
+        raw_spin_unlock(&tsk_rt(t)->hp_blocked_tasks_lock);  // unlock the t's heap.
+
+
+        if(blocked_lock) {
+                if(blocked_lock->ops->propagate_increase_inheritance) {
+                        TRACE_TASK(t, "Inheritor is blocked (...perhaps).  Checking lock %d.\n",
+                                           blocked_lock->ident);
+
+                        // beware: recursion
+                        blocked_lock->ops->propagate_increase_inheritance(blocked_lock,
+                                                                                                                          t, to_unlock,
+                                                                                                                          irqflags);
+                }
+                else {
+                        TRACE_TASK(t, "Inheritor is blocked on lock (%d) that does not support nesting!\n",
+                                           blocked_lock->ident);
+                        unlock_fine_irqrestore(to_unlock, irqflags);
+                }
+        }
+        else {
+                TRACE_TASK(t, "is not blocked.  No propagation.\n");
+                unlock_fine_irqrestore(to_unlock, irqflags);
+        }
+}
+
+/* called with IRQs off */
+/* preconditions:
+ (1) The 'hp_blocked_tasks_lock' of task 't' is held.
+ (2) The lock 'to_unlock' is held.
+ */
+static void nested_decrease_priority_inheritance(struct task_struct* t,
+                                                                                                 struct task_struct* prio_inh,
+                                                                                                 raw_spinlock_t *to_unlock,
+                                                                                                 unsigned long irqflags)
+{
+        struct litmus_lock *blocked_lock = tsk_rt(t)->blocked_lock;
+        decrease_priority_inheritance(t, prio_inh);
+
+        raw_spin_unlock(&tsk_rt(t)->hp_blocked_tasks_lock);  // unlock the t's heap.
+
+        if(blocked_lock) {
+                if(blocked_lock->ops->propagate_decrease_inheritance) {
+                        TRACE_TASK(t, "Inheritor is blocked (...perhaps).  Checking lock %d.\n",
+                                           blocked_lock->ident);
+
+                        // beware: recursion
+                        blocked_lock->ops->propagate_decrease_inheritance(blocked_lock, t,
+                                                                                                                          to_unlock,
+                                                                                                                          irqflags);
+                }
+                else {
+                        TRACE_TASK(t, "Inheritor is blocked on lock (%p) that does not support nesting!\n",
+                                           blocked_lock);
+                        unlock_fine_irqrestore(to_unlock, irqflags);
+                }
+        }
+        else {
+                TRACE_TASK(t, "is not blocked.  No propagation.\n");
+                unlock_fine_irqrestore(to_unlock, irqflags);
+        }
+}
+
+
+/* ******************** RSM MUTEX ********************** */
+
+static struct litmus_lock_ops gsnedf_rsm_mutex_lock_ops = {
+        .lock   = rsm_mutex_lock,
+        .unlock = rsm_mutex_unlock,
+        .close  = rsm_mutex_close,
+        .deallocate = rsm_mutex_free,
+
+        .propagate_increase_inheritance = rsm_mutex_propagate_increase_inheritance,
+        .propagate_decrease_inheritance = rsm_mutex_propagate_decrease_inheritance,
+
+#ifdef CONFIG_LITMUS_DGL_SUPPORT
+        .dgl_lock = rsm_mutex_dgl_lock,
+        .is_owner = rsm_mutex_is_owner,
+        .enable_priority = rsm_mutex_enable_priority,
+#endif
+};
+
+static struct litmus_lock* gsnedf_new_rsm_mutex(void)
+{
+        return rsm_mutex_new(&gsnedf_rsm_mutex_lock_ops);
 }
 
+/* ******************** IKGLP ********************** */
+
+static struct litmus_lock_ops gsnedf_ikglp_lock_ops = {
+        .lock   = ikglp_lock,
+        .unlock = ikglp_unlock,
+        .close  = ikglp_close,
+        .deallocate = ikglp_free,
+
+        // ikglp can only be an outer-most lock.
+        .propagate_increase_inheritance = NULL,
+        .propagate_decrease_inheritance = NULL,
+};
+
+static struct litmus_lock* gsnedf_new_ikglp(void* __user arg)
+{
+        return ikglp_new(num_online_cpus(), &gsnedf_ikglp_lock_ops, arg);
+}
+
+#endif  /* CONFIG_LITMUS_NESTED_LOCKING */
+
+
+/* ******************** KFMLP support ********************** */
+
+static struct litmus_lock_ops gsnedf_kfmlp_lock_ops = {
+        .lock   = kfmlp_lock,
+        .unlock = kfmlp_unlock,
+        .close  = kfmlp_close,
+        .deallocate = kfmlp_free,
+
+        // kfmlp can only be an outer-most lock.
+        .propagate_increase_inheritance = NULL,
+        .propagate_decrease_inheritance = NULL,
+};
+
+
+static struct litmus_lock* gsnedf_new_kfmlp(void* __user arg)
+{
+        return kfmlp_new(&gsnedf_kfmlp_lock_ops, arg);
+}
 
 /* ******************** FMLP support ********************** */
 
@@ -789,7 +1589,7 @@ int gsnedf_fmlp_lock(struct litmus_lock* l)
                 if (edf_higher_prio(t, sem->hp_waiter)) {
                         sem->hp_waiter = t;
                         if (edf_higher_prio(t, sem->owner))
-                                set_priority_inheritance(sem->owner, sem->hp_waiter);
+                                increase_priority_inheritance(sem->owner, sem->hp_waiter);
                 }
 
                 TS_LOCK_SUSPEND;
@@ -802,7 +1602,7 @@ int gsnedf_fmlp_lock(struct litmus_lock* l)
                  * there is only one wake up per release.
                  */
 
-                schedule();
+                suspend_for_lock();
 
                 TS_LOCK_RESUME;
 
@@ -857,7 +1657,7 @@ int gsnedf_fmlp_unlock(struct litmus_lock* l)
                         /* Well, if next is not the highest-priority waiter,
                          * then it ought to inherit the highest-priority
                          * waiter's priority. */
-                        set_priority_inheritance(next, sem->hp_waiter);
+                        increase_priority_inheritance(next, sem->hp_waiter);
                 }
 
                 /* wake up next */
@@ -868,7 +1668,7 @@ int gsnedf_fmlp_unlock(struct litmus_lock* l)
 
         /* we lose the benefit of priority inheritance (if any) */
         if (tsk_rt(t)->inh_task)
-                clear_priority_inheritance(t);
+                decrease_priority_inheritance(t, NULL);
 
 out:
         spin_unlock_irqrestore(&sem->wait.lock, flags);
@@ -906,6 +1706,11 @@ static struct litmus_lock_ops gsnedf_fmlp_lock_ops = {
         .lock   = gsnedf_fmlp_lock,
         .unlock = gsnedf_fmlp_unlock,
         .deallocate = gsnedf_fmlp_free,
+
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+        .propagate_increase_inheritance = NULL,
+        .propagate_decrease_inheritance = NULL
+#endif
 };
 
 static struct litmus_lock* gsnedf_new_fmlp(void)
@@ -924,31 +1729,110 @@ static struct litmus_lock* gsnedf_new_fmlp(void)
         return &sem->litmus_lock;
 }
 
-/* **** lock constructor **** */
-
 
 static long gsnedf_allocate_lock(struct litmus_lock **lock, int type,
-                                 void* __user unused)
+                                 void* __user args)
 {
-        int err = -ENXIO;
+        int err;
 
-        /* GSN-EDF currently only supports the FMLP for global resources. */
         switch (type) {
 
         case FMLP_SEM:
                 /* Flexible Multiprocessor Locking Protocol */
                 *lock = gsnedf_new_fmlp();
-                if (*lock)
-                        err = 0;
-                else
-                        err = -ENOMEM;
+                break;
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+    case RSM_MUTEX:
+                *lock = gsnedf_new_rsm_mutex();
                 break;
 
+        case IKGLP_SEM:
+                *lock = gsnedf_new_ikglp(args);
+                break;
+#endif
+        case KFMLP_SEM:
+                *lock = gsnedf_new_kfmlp(args);
+                break;
+        default:
+                err = -ENXIO;
+                goto UNSUPPORTED_LOCK;
+        };
+
+        if (*lock)
+                err = 0;
+        else
+                err = -ENOMEM;
+
+UNSUPPORTED_LOCK:
+        return err;
+}
+
+#endif  // CONFIG_LITMUS_LOCKING
+
+
+
+
+
+#ifdef CONFIG_LITMUS_AFFINITY_LOCKING
+static struct affinity_observer_ops gsnedf_kfmlp_affinity_ops = {
+        .close = kfmlp_aff_obs_close,
+        .deallocate = kfmlp_aff_obs_free,
+};
+
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+static struct affinity_observer_ops gsnedf_ikglp_affinity_ops = {
+        .close = ikglp_aff_obs_close,
+        .deallocate = ikglp_aff_obs_free,
+};
+#endif
+
+static long gsnedf_allocate_affinity_observer(
+                                                                struct affinity_observer **aff_obs,
+                                                                int type,
+                                                                void* __user args)
+{
+        int err;
+
+        switch (type) {
+
+                case KFMLP_SIMPLE_GPU_AFF_OBS:
+                        *aff_obs = kfmlp_simple_gpu_aff_obs_new(&gsnedf_kfmlp_affinity_ops, args);
+                        break;
+
+                case KFMLP_GPU_AFF_OBS:
+                        *aff_obs = kfmlp_gpu_aff_obs_new(&gsnedf_kfmlp_affinity_ops, args);
+                        break;
+
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+                case IKGLP_SIMPLE_GPU_AFF_OBS:
+                        *aff_obs = ikglp_simple_gpu_aff_obs_new(&gsnedf_ikglp_affinity_ops, args);
+                        break;
+
+                case IKGLP_GPU_AFF_OBS:
+                        *aff_obs = ikglp_gpu_aff_obs_new(&gsnedf_ikglp_affinity_ops, args);
+                        break;
+#endif
+                default:
+                        err = -ENXIO;
+                        goto UNSUPPORTED_AFF_OBS;
         };
 
+        if (*aff_obs)
+                err = 0;
+        else
+                err = -ENOMEM;
+
+UNSUPPORTED_AFF_OBS:
         return err;
 }
+#endif
+
 
+#if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_SOFTIRQD)
+static int gsnedf_map_gpu_to_cpu(int gpu)
+{
+        return -1;  // No CPU affinity needed.
+}
 #endif
 
 
@@ -957,14 +1841,14 @@ static long gsnedf_activate_plugin(void)
         int cpu;
         cpu_entry_t *entry;
 
-        bheap_init(&gsnedf_cpu_heap);
+        INIT_BINHEAP_HANDLE(&gsnedf_cpu_heap, cpu_lower_prio);
 #ifdef CONFIG_RELEASE_MASTER
         gsnedf.release_master = atomic_read(&release_master_cpu);
 #endif
 
         for_each_online_cpu(cpu) {
                 entry = &per_cpu(gsnedf_cpu_entries, cpu);
-                bheap_node_init(&entry->hn, entry);
+                INIT_BINHEAP_NODE(&entry->hn);
                 entry->linked    = NULL;
                 entry->scheduled = NULL;
 #ifdef CONFIG_RELEASE_MASTER
@@ -978,6 +1862,20 @@ static long gsnedf_activate_plugin(void)
                 }
 #endif
         }
+
+#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
+        gsnedf_pending_tasklets.head = NULL;
+        gsnedf_pending_tasklets.tail = &(gsnedf_pending_tasklets.head);
+#endif
+
+#ifdef CONFIG_LITMUS_SOFTIRQD
+    init_klmirqd();
+#endif
+
+#ifdef CONFIG_LITMUS_NVIDIA
+        init_nvidia_info();
+#endif
+
         return 0;
 }
 
@@ -994,8 +1892,32 @@ static struct sched_plugin gsn_edf_plugin __cacheline_aligned_in_smp = {
         .task_block             = gsnedf_task_block,
         .admit_task             = gsnedf_admit_task,
         .activate_plugin        = gsnedf_activate_plugin,
+        .compare                = edf_higher_prio,
 #ifdef CONFIG_LITMUS_LOCKING
         .allocate_lock          = gsnedf_allocate_lock,
+        .increase_prio          = increase_priority_inheritance,
+        .decrease_prio          = decrease_priority_inheritance,
+        .__increase_prio        = __increase_priority_inheritance,
+        .__decrease_prio        = __decrease_priority_inheritance,
+#endif
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+        .nested_increase_prio           = nested_increase_priority_inheritance,
+        .nested_decrease_prio           = nested_decrease_priority_inheritance,
+        .__compare                                      = __edf_higher_prio,
+#endif
+#ifdef CONFIG_LITMUS_DGL_SUPPORT
+        .get_dgl_spinlock = gsnedf_get_dgl_spinlock,
+#endif
+#ifdef CONFIG_LITMUS_AFFINITY_LOCKING
+        .allocate_aff_obs = gsnedf_allocate_affinity_observer,
+#endif
+#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
+        .enqueue_pai_tasklet = gsnedf_enqueue_pai_tasklet,
+        .change_prio_pai_tasklet = gsnedf_change_prio_pai_tasklet,
+        .run_tasklets = gsnedf_run_tasklets,
+#endif
+#if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_SOFTIRQD)
+        .map_gpu_to_cpu = gsnedf_map_gpu_to_cpu,
 #endif
 };
 
@@ -1005,15 +1927,20 @@ static int __init init_gsn_edf(void)
         int cpu;
         cpu_entry_t *entry;
 
-        bheap_init(&gsnedf_cpu_heap);
+        INIT_BINHEAP_HANDLE(&gsnedf_cpu_heap, cpu_lower_prio);
         /* initialize CPU state */
-        for (cpu = 0; cpu < NR_CPUS; cpu++)  {
+        for (cpu = 0; cpu < NR_CPUS; ++cpu)  {
                 entry = &per_cpu(gsnedf_cpu_entries, cpu);
                 gsnedf_cpus[cpu] = entry;
                 entry->cpu       = cpu;
-                entry->hn        = &gsnedf_heap_node[cpu];
-                bheap_node_init(&entry->hn, entry);
+
+                INIT_BINHEAP_NODE(&entry->hn);
         }
+
+#ifdef CONFIG_LITMUS_DGL_SUPPORT
+        raw_spin_lock_init(&dgl_lock);
+#endif
+
         edf_domain_init(&gsnedf, NULL, gsnedf_release_jobs);
         return register_sched_plugin(&gsn_edf_plugin);
 }