Fix partitioned C-EDF

4 files changed, 66 insertions, 21 deletions

diff --git a/kernel/softirq.c b/kernel/softirq.c
index ef70dbe92a97..9c20a9ec06b2 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -485,7 +485,10 @@ static int __klmirqd_nv_tasklet_schedule(struct tasklet_struct *t,
 
 void __tasklet_schedule(struct tasklet_struct *t)
 {
-#ifdef CONFIG_LITMUS_NVIDIA
+#if defined(CONFIG_LITMUS_NVIDIA) && ( \
+                defined(CONFIG_LITMUS_NVIDIA_NONSPLIT_INTERRUPTS) || \
+                defined(CONFIG_LITMUS_SOFTIRQD) || \
+                defined(CONFIG_LITMUS_PAI_SOFTIRQD))
         if(is_nvidia_func(t->func))
         {
 #if defined(CONFIG_LITMUS_NVIDIA_NONSPLIT_INTERRUPTS)
@@ -527,7 +530,10 @@ EXPORT_SYMBOL(___tasklet_schedule);
 
 void __tasklet_hi_schedule(struct tasklet_struct *t)
 {
-#ifdef CONFIG_LITMUS_NVIDIA
+#if defined(CONFIG_LITMUS_NVIDIA) && ( \
+                defined(CONFIG_LITMUS_NVIDIA_NONSPLIT_INTERRUPTS) || \
+                defined(CONFIG_LITMUS_SOFTIRQD) || \
+                defined(CONFIG_LITMUS_PAI_SOFTIRQD))
         if(is_nvidia_func(t->func))
         {
 #if defined(CONFIG_LITMUS_NVIDIA_NONSPLIT_INTERRUPTS)
@@ -568,7 +574,10 @@ EXPORT_SYMBOL(___tasklet_hi_schedule);
 void __tasklet_hi_schedule_first(struct tasklet_struct *t)
 {
         BUG_ON(!irqs_disabled());
-#ifdef CONFIG_LITMUS_NVIDIA
+#if defined(CONFIG_LITMUS_NVIDIA) && ( \
+                defined(CONFIG_LITMUS_NVIDIA_NONSPLIT_INTERRUPTS) || \
+                defined(CONFIG_LITMUS_SOFTIRQD) || \
+                defined(CONFIG_LITMUS_PAI_SOFTIRQD))
         if(is_nvidia_func(t->func))
         {
 #if defined(CONFIG_LITMUS_NVIDIA_NONSPLIT_INTERRUPTS)
diff --git a/litmus/Kconfig b/litmus/Kconfig
index 4bd371d20a1a..644d2d55bfb6 100644
--- a/litmus/Kconfig
+++ b/litmus/Kconfig
@@ -489,7 +489,7 @@ config LITMUS_AFFINITY_AWARE_GPU_ASSINGMENT
 
 config NV_DEVICE_NUM
            int "Number of NVIDIA GPUs."
-           depends on LITMUS_SOFTIRQD || LITMUS_PAI_SOFTIRQD
+           depends on LITMUS_NVIDIA
            range 1 16
            default "1"
            help
diff --git a/litmus/nvidia_info.c b/litmus/nvidia_info.c
index fc4d5297fe6f..e87e56542a23 100644
--- a/litmus/nvidia_info.c
+++ b/litmus/nvidia_info.c
@@ -474,7 +474,6 @@ static int gpu_owner_max_priority_order(struct binheap_node *a,
 static int init_nv_device_reg(void)
 {
         int i;
-        char name[MAX_KLMIRQD_NAME_LEN+1];
 
 #ifdef CONFIG_LITMUS_SOFTIRQD
         if (!klmirqd_is_ready()) {
@@ -498,6 +497,7 @@ static int init_nv_device_reg(void)
 
 #ifdef CONFIG_LITMUS_SOFTIRQD
                 {
+                        char name[MAX_KLMIRQD_NAME_LEN+1];
                         int default_cpu = litmus->map_gpu_to_cpu(i);
 
                         /* spawn the interrupt thread */
diff --git a/litmus/sched_cedf.c b/litmus/sched_cedf.c
index fc8f277a1958..efb11bb8c656 100644
--- a/litmus/sched_cedf.c
+++ b/litmus/sched_cedf.c
@@ -326,8 +326,7 @@ static noinline void requeue(struct task_struct* task)
 #endif
                         __add_ready(&cluster->domain, task);
         else {
-                /* it has got to wait */
-                add_release(&cluster->domain, task);
+                TRACE_TASK(task, "not requeueing non-yet-released job\n");
         }
 }
 
@@ -412,6 +411,7 @@ static void cedf_release_jobs(rt_domain_t* rt, struct bheap* tasks)
 /* caller holds cluster_lock */
 static noinline void job_completion(struct task_struct *t, int forced)
 {
+        lt_t now;
         BUG_ON(!t);
 
         sched_trace_task_completion(t, forced);
@@ -426,14 +426,23 @@ static noinline void job_completion(struct task_struct *t, int forced)
         tsk_rt(t)->completed = 1;
         /* prepare for next period */
         prepare_for_next_period(t);
-        if (is_released(t, litmus_clock()))
+
+        now = litmus_clock();
+        if (is_released(t, now)) {
+                /* log here to capture overheads */
                 sched_trace_task_release(t);
-        /* unlink */
+        }
+
         unlink(t);
-        /* requeue
-         * But don't requeue a blocking task. */
-        if (is_running(t))
-                cedf_job_arrival(t);
+
+        /* release or arm next job */
+        tsk_rt(t)->completed = 0;
+        if (is_running(t)) {
+                if (!is_released(t, now))
+                        add_release(&task_cpu_cluster(t)->domain, t);
+                else
+                        cedf_job_arrival(t);
+        }
 }
 
 /* cedf_tick - this function is called for every local timer
@@ -1191,6 +1200,18 @@ static int __increase_priority_inheritance(struct task_struct* t,
 
         cluster = task_cpu_cluster(t);
 
+#if 0
+        if (prio_inh && task_cpu_cluster(prio_inh) != cluster) {
+                WARN_ONCE(1, "Illegal to inherit between clusters. " \
+                                        "target (%s/%d) on cluster w/ CPU %d and " \
+                                        "inh_prio (%s/%d) on w/ CPU %d\n", \
+                                        t->comm, t->pid, cluster->cpus[0]->cpu, \
+                                        prio_inh->comm, prio_inh->pid, \
+                                        task_cpu_cluster(prio_inh)->cpus[0]->cpu);
+                return 1;
+        }
+#endif
+
 #ifdef CONFIG_LITMUS_NESTED_LOCKING
         /* this sanity check allows for weaker locking in protocols */
         /* TODO (klmirqd): Skip this check if 't' is a proxy thread (???) */
@@ -1316,6 +1337,7 @@ static void increase_priority_inheritance(struct task_struct* t, struct task_str
 static int __decrease_priority_inheritance(struct task_struct* t,
                                                                                         struct task_struct* prio_inh)
 {
+        cedf_domain_t* cluster;
         int success = 1;
 
         if (prio_inh == tsk_rt(t)->inh_task) {
@@ -1347,6 +1369,20 @@ static int __decrease_priority_inheritance(struct task_struct* t,
 #endif
         }
 
+        cluster = task_cpu_cluster(t);
+
+#if 0
+        if (prio_inh && task_cpu_cluster(prio_inh) != cluster) {
+                WARN_ONCE(1, "Illegal to inherit between clusters. " \
+                                        "target (%s/%d) on cluster w/ CPU %d and " \
+                                        "inh_prio (%s/%d) on w/ CPU %d\n", \
+                                        t->comm, t->pid, cluster->cpus[0]->cpu, \
+                                        prio_inh->comm, prio_inh->pid, \
+                                        task_cpu_cluster(prio_inh)->cpus[0]->cpu);
+                return 1;
+        }
+#endif
+
 #ifdef CONFIG_LITMUS_NESTED_LOCKING
         if(__edf_higher_prio(t, EFFECTIVE, prio_inh, BASE)) {
 #endif
@@ -1371,15 +1407,16 @@ static int __decrease_priority_inheritance(struct task_struct* t,
                         cedf_job_arrival(t);
                 }
                 else {
-                        cedf_domain_t* cluster = task_cpu_cluster(t);
                         /* task is queued */
                         raw_spin_lock(&cluster->domain.release_lock);
                         if (is_queued(t)) {
                                 TRACE_TASK(t, "is queued.\n");
 
-                                /* decrease in priority, so we have to re-add to binomial heap */
                                 unlink(t);
-                                cedf_job_arrival(t);
+                                if (is_released(t, litmus_clock()))
+                                        cedf_job_arrival(t);
+                                else
+                                        add_release(&cluster->domain, t);
                         }
                         else {
                                 TRACE_TASK(t, "is not in scheduler. Probably on wait queue somewhere.\n");
@@ -1912,16 +1949,15 @@ static long cedf_activate_plugin(void)
 
                                 entry = &per_cpu(cedf_cpu_entries, ccpu);
                                 cedf[i].cpus[cpu_count] = entry;
-                                atomic_set(&entry->will_schedule, 0);
+
+                                memset(entry, 0, sizeof(*entry));
                                 entry->cpu = ccpu;
                                 entry->cluster = &cedf[i];
-
                                 INIT_BINHEAP_NODE(&entry->hn);
+                                mb();
 
-                                cpu_count++;
+                                ++cpu_count;
 
-                                entry->linked = NULL;
-                                entry->scheduled = NULL;
 #ifdef CONFIG_RELEASE_MASTER
                                 /* only add CPUs that should schedule jobs */
                                 if (entry->cpu != entry->cluster->domain.release_master)