1 files changed, 247 insertions, 33 deletions

diff --git a/litmus/litmus.c b/litmus/litmus.c
index dc94be71bfb6..2911e7ec7029 100644
--- a/litmus/litmus.c
+++ b/litmus/litmus.c
@@ -23,6 +23,14 @@
 #include <litmus/affinity.h>
 #endif
 
+#ifdef CONFIG_LITMUS_NVIDIA
+#include <litmus/nvidia_info.h>
+#endif
+
+#ifdef CONFIG_REALTIME_AUX_TASKS
+#include <litmus/aux_tasks.h>
+#endif
+
 /* Number of RT tasks that exist in the system */
 atomic_t rt_task_count          = ATOMIC_INIT(0);
 
@@ -135,6 +143,16 @@ asmlinkage long sys_set_rt_task_param(pid_t pid, struct rt_task __user * param)
                        pid, tp.budget_policy);
                 goto out_unlock;
         }
+        if (tp.budget_signal_policy != NO_SIGNALS &&
+            tp.budget_signal_policy != QUANTUM_SIGNALS &&
+            tp.budget_signal_policy != PRECISE_SIGNALS)
+        {
+                printk(KERN_INFO "litmus: real-time task %d rejected "
+                       "because unsupported budget signalling policy "
+                       "specified (%d)\n",
+                       pid, tp.budget_signal_policy);
+                goto out_unlock;
+        }
 
         target->rt_param.task_params = tp;
 
@@ -272,6 +290,7 @@ asmlinkage long sys_query_job_no(unsigned int __user *job)
         return retval;
 }
 
+
 /* sys_null_call() is only used for determining raw system call
  * overheads (kernel entry, kernel exit). It has no useful side effects.
  * If ts is non-NULL, then the current Feather-Trace time is recorded.
@@ -289,12 +308,117 @@ asmlinkage long sys_null_call(cycles_t __user *ts)
         return ret;
 }
 
+
+asmlinkage long sys_sched_trace_event(int event, struct st_inject_args __user *__args)
+{
+        long retval = 0;
+        struct task_struct* t = current;
+
+        struct st_inject_args args;
+
+        if (is_realtime(t)) {
+                printk(KERN_WARNING "Only non-real-time tasks may inject sched_trace events.\n");
+                retval = -EINVAL;
+                goto out;
+        }
+
+        if (__args && copy_from_user(&args, __args, sizeof(args))) {
+                retval = -EFAULT;
+                goto out;
+        }
+
+        switch(event) {
+                /*************************************/
+                /* events that don't need parameters */
+                /*************************************/
+                case ST_INJECT_NAME:
+                        sched_trace_task_name(t);
+                        break;
+                case ST_INJECT_PARAM:
+                        /* presumes sporadic_task_ns() has already been called
+                         * and valid data has been initialized even if the calling
+                         * task is SCHED_NORMAL. */
+                        sched_trace_task_param(t);
+                        break;
+
+                /*******************************/
+                /* events that need parameters */
+                /*******************************/
+                case ST_INJECT_COMPLETION:
+                        if (!__args) {
+                                retval = -EINVAL;
+                                goto out;
+                        }
+
+                        /* slam in the data */
+                        t->rt_param.job_params.job_no = args.job_no;
+
+                        sched_trace_task_completion(t, 0);
+                        break;
+                case ST_INJECT_RELEASE:
+                        if (!__args) {
+                                retval = -EINVAL;
+                                goto out;
+                        }
+
+                        /* slam in the data */
+                        tsk_rt(t)->job_params.release = args.release;
+                        tsk_rt(t)->job_params.deadline = args.deadline;
+
+                        sched_trace_task_release(t);
+                        break;
+
+                /**********************/
+                /* unsupported events */
+                /**********************/
+                default:
+                        retval = -EINVAL;
+                        break;
+        }
+
+out:
+        return retval;
+}
+
+
+#if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_AFFINITY_LOCKING)
+void init_gpu_affinity_state(struct task_struct* p)
+{
+        // under-damped
+        //p->rt_param.gpu_fb_param_a = _frac(14008, 10000);
+        //p->rt_param.gpu_fb_param_b = _frac(16024, 10000);
+
+#if 0
+        // emperical;
+        p->rt_param.gpu_fb_param_a[0] = _frac(7550, 10000);
+        p->rt_param.gpu_fb_param_b[0] = _frac(45800, 10000);
+
+        p->rt_param.gpu_fb_param_a[1] = _frac(8600, 10000);
+        p->rt_param.gpu_fb_param_b[1] = _frac(40000, 10000);
+
+        p->rt_param.gpu_fb_param_a[2] = _frac(6890, 10000);
+        p->rt_param.gpu_fb_param_b[2] = _frac(40000, 10000);
+
+        p->rt_param.gpu_fb_param_a[3] = _frac(7580, 10000);
+        p->rt_param.gpu_fb_param_b[3] = _frac(34590, 10000);
+#endif
+        p->rt_param.gpu_migration = MIG_NONE;
+        p->rt_param.last_gpu = -1;
+}
+#endif
+
 /* p is a real-time task. Re-init its state as a best-effort task. */
 static void reinit_litmus_state(struct task_struct* p, int restore)
 {
         struct rt_task  user_config = {};
         void*  ctrl_page     = NULL;
 
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+        binheap_order_t prio_order = NULL;
+#endif
+
+        TRACE_TASK(p, "reinit_litmus_state: restore = %d\n", restore);
+
         if (restore) {
                 /* Safe user-space provided configuration data.
                  * and allocated page. */
@@ -302,48 +426,57 @@ static void reinit_litmus_state(struct task_struct* p, int restore)
                 ctrl_page   = p->rt_param.ctrl_page;
         }
 
+#ifdef CONFIG_LITMUS_NVIDIA
+        WARN_ON(p->rt_param.held_gpus != 0);
+#endif
+
+#ifdef CONFIG_LITMUS_LOCKING
         /* We probably should not be inheriting any task's priority
          * at this point in time.
          */
         WARN_ON(p->rt_param.inh_task);
+#endif
+
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+        prio_order = p->rt_param.hp_blocked_tasks.compare;
+#endif
 
         /* Cleanup everything else. */
         memset(&p->rt_param, 0, sizeof(p->rt_param));
 
+#ifdef CONFIG_REALTIME_AUX_TASKS
+        /* also clear out the aux_data. the !restore case is only called on
+         * fork (initial thread creation). */
+        if (!restore) {
+                memset(&p->aux_data, 0, sizeof(p->aux_data));
+        }
+#endif
+
         /* Restore preserved fields. */
         if (restore) {
                 p->rt_param.task_params = user_config;
                 p->rt_param.ctrl_page   = ctrl_page;
         }
-}
 
-long litmus_admit_task(struct task_struct* tsk)
-{
-        long retval = 0;
+#ifdef CONFIG_LITMUS_NVIDIA
+        INIT_BINHEAP_NODE(&p->rt_param.gpu_owner_node);
+#endif
 
-        BUG_ON(is_realtime(tsk));
+#if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_AFFINITY_LOCKING)
+        init_gpu_affinity_state(p);
+#endif
 
-        tsk_rt(tsk)->heap_node = NULL;
-        tsk_rt(tsk)->rel_heap = NULL;
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+        INIT_BINHEAP_HANDLE(&p->rt_param.hp_blocked_tasks, prio_order);
+        raw_spin_lock_init(&p->rt_param.hp_blocked_tasks_lock);
+#endif
+}
 
-        if (get_rt_relative_deadline(tsk) == 0 ||
-            get_exec_cost(tsk) >
-                        min(get_rt_relative_deadline(tsk), get_rt_period(tsk)) ) {
-                TRACE_TASK(tsk,
-                        "litmus admit: invalid task parameters "
-                        "(e = %lu, p = %lu, d = %lu)\n",
-                        get_exec_cost(tsk), get_rt_period(tsk),
-                        get_rt_relative_deadline(tsk));
-                retval = -EINVAL;
-                goto out;
-        }
 
-        if (!cpu_online(get_partition(tsk))) {
-                TRACE_TASK(tsk, "litmus admit: cpu %d is not online\n",
-                           get_partition(tsk));
-                retval = -EINVAL;
-                goto out;
-        }
+
+long __litmus_admit_task(struct task_struct* tsk)
+{
+        long retval = 0;
 
         INIT_LIST_HEAD(&tsk_rt(tsk)->list);
 
@@ -360,6 +493,17 @@ long litmus_admit_task(struct task_struct* tsk)
                 bheap_node_init(&tsk_rt(tsk)->heap_node, tsk);
         }
 
+#ifdef CONFIG_LITMUS_NVIDIA
+        atomic_set(&tsk_rt(tsk)->nv_int_count, 0);
+#endif
+#if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_AFFINITY_LOCKING)
+        init_gpu_affinity_state(tsk);
+#endif
+#ifdef CONFIG_LITMUS_NESTED_LOCKING
+        tsk_rt(tsk)->blocked_lock = NULL;
+        raw_spin_lock_init(&tsk_rt(tsk)->hp_blocked_tasks_lock);
+#endif
+
         preempt_disable();
 
         retval = litmus->admit_task(tsk);
@@ -372,14 +516,56 @@ long litmus_admit_task(struct task_struct* tsk)
 
         preempt_enable();
 
-out:
         if (retval) {
                 bheap_node_free(tsk_rt(tsk)->heap_node);
                 release_heap_free(tsk_rt(tsk)->rel_heap);
         }
+
+out:
+        return retval;
+}
+
+long litmus_admit_task(struct task_struct* tsk)
+{
+        long retval = 0;
+
+        BUG_ON(is_realtime(tsk));
+
+        if (get_rt_relative_deadline(tsk) == 0 ||
+            get_exec_cost(tsk) >
+                        min(get_rt_relative_deadline(tsk), get_rt_period(tsk)) ) {
+                TRACE_TASK(tsk,
+                        "litmus admit: invalid task parameters "
+                        "(e = %lu, p = %lu, d = %lu)\n",
+                        get_exec_cost(tsk), get_rt_period(tsk),
+                        get_rt_relative_deadline(tsk));
+                retval = -EINVAL;
+                goto out;
+        }
+
+        if (!cpu_online(get_partition(tsk))) {
+                TRACE_TASK(tsk, "litmus admit: cpu %d is not online\n",
+                           get_partition(tsk));
+                retval = -EINVAL;
+                goto out;
+        }
+
+        retval = __litmus_admit_task(tsk);
+
+out:
         return retval;
 }
 
+void litmus_pre_exit_task(struct task_struct* tsk)
+{
+        if (is_realtime(tsk)) {
+                if (tsk_rt(tsk)->rsrc_exit_cb) {
+                        int ret = tsk_rt(tsk)->rsrc_exit_cb(tsk);
+                        WARN_ON(ret != 0);
+                }
+        }
+}
+
 void litmus_exit_task(struct task_struct* tsk)
 {
         if (is_realtime(tsk)) {
@@ -388,7 +574,7 @@ void litmus_exit_task(struct task_struct* tsk)
                 litmus->task_exit(tsk);
 
                 BUG_ON(bheap_node_in_heap(tsk_rt(tsk)->heap_node));
-                bheap_node_free(tsk_rt(tsk)->heap_node);
+                bheap_node_free(tsk_rt(tsk)->heap_node);
                 release_heap_free(tsk_rt(tsk)->rel_heap);
 
                 atomic_dec(&rt_task_count);
@@ -406,14 +592,19 @@ static int do_plugin_switch(void *_plugin)
                 ret = litmus->deactivate_plugin();
                 if (0 != ret)
                         goto out;
-                ret = plugin->activate_plugin();
+
+                litmus = plugin;  /* optimistic switch */
+                mb();
+
+                ret = litmus->activate_plugin();
                 if (0 != ret) {
                         printk(KERN_INFO "Can't activate %s (%d).\n",
-                               plugin->plugin_name, ret);
-                        plugin = &linux_sched_plugin;
+                               litmus->plugin_name, ret);
+                        litmus = &linux_sched_plugin; /* fail to Linux */
+                        ret = litmus->activate_plugin();
+                        BUG_ON(ret);
                 }
-                printk(KERN_INFO "Switching to LITMUS^RT plugin %s.\n", plugin->plugin_name);
-                litmus = plugin;
+                printk(KERN_INFO "Switched to LITMUS^RT plugin %s.\n", litmus->plugin_name);
         } else
                 ret = -EBUSY;
 out:
@@ -429,6 +620,12 @@ int switch_sched_plugin(struct sched_plugin* plugin)
 {
         BUG_ON(!plugin);
 
+#ifdef CONFIG_LITMUS_SOFTIRQD
+        if (!klmirqd_is_dead()) {
+                kill_klmirqd();
+        }
+#endif
+
         if (atomic_read(&rt_task_count) == 0)
                 return stop_machine(do_plugin_switch, plugin, NULL);
         else
@@ -441,18 +638,33 @@ int switch_sched_plugin(struct sched_plugin* plugin)
 void litmus_fork(struct task_struct* p)
 {
         if (is_realtime(p)) {
+                TRACE_TASK(p, "fork, is real-time\n");
+
                 /* clean out any litmus related state, don't preserve anything */
                 reinit_litmus_state(p, 0);
+
                 /* Don't let the child be a real-time task.  */
                 p->sched_reset_on_fork = 1;
-        } else
+
+        } else {
                 /* non-rt tasks might have ctrl_page set */
                 tsk_rt(p)->ctrl_page = NULL;
 
+                reinit_litmus_state(p, 0);
+        }
+
         /* od tables are never inherited across a fork */
         p->od_table = NULL;
 }
 
+/* Called right before copy_process() returns a forked thread. */
+void litmus_post_fork_thread(struct task_struct* p)
+{
+#ifdef CONFIG_REALTIME_AUX_TASKS
+        make_aux_task_if_required(p);
+#endif
+}
+
 /* Called upon execve().
  * current is doing the exec.
  * Don't let address space specific stuff leak.
@@ -486,8 +698,10 @@ void exit_litmus(struct task_struct *dead_tsk)
         }
 
         /* main cleanup only for RT tasks */
-        if (is_realtime(dead_tsk))
+        if (is_realtime(dead_tsk)) {
+                litmus_pre_exit_task(dead_tsk); /* todo: double check that no Linux rq lock is held */
                 litmus_exit_task(dead_tsk);
+        }
 }