Merge branch 'wip-gpu-cleanup' into wip-2012.3-gpu

Conflicts:
	include/litmus/fpmath.h
	include/litmus/litmus.h
	include/litmus/rt_param.h
	include/litmus/trace.h
	kernel/sched.c
	kernel/softirq.c
	litmus/edf_common.c
	litmus/jobs.c
	litmus/litmus.c
	litmus/locking.c
	litmus/preempt.c
	litmus/sched_cedf.c
	litmus/sched_gsn_edf.c
	litmus/sched_litmus.c
	litmus/sync.c

1 files changed, 275 insertions, 7 deletions

diff --git a/litmus/sched_task_trace.c b/litmus/sched_task_trace.c
index 5ef8d09ab41f..f7f575346b54 100644
--- a/litmus/sched_task_trace.c
+++ b/litmus/sched_task_trace.c
@@ -7,6 +7,7 @@
 #include <linux/module.h>
 #include <linux/sched.h>
 #include <linux/percpu.h>
+#include <linux/hardirq.h>
 
 #include <litmus/ftdev.h>
 #include <litmus/litmus.h>
@@ -16,13 +17,13 @@
 #include <litmus/ftdev.h>
 
 
-#define NO_EVENTS               (1 << CONFIG_SCHED_TASK_TRACE_SHIFT)
+#define NUM_EVENTS              (1 << (CONFIG_SCHED_TASK_TRACE_SHIFT+11))
 
 #define now() litmus_clock()
 
 struct local_buffer {
-        struct st_event_record record[NO_EVENTS];
-        char   flag[NO_EVENTS];
+        struct st_event_record record[NUM_EVENTS];
+        char   flag[NUM_EVENTS];
         struct ft_buffer ftbuf;
 };
 
@@ -41,7 +42,7 @@ static int __init init_sched_task_trace(void)
         int i, ok = 0, err;
         printk("Allocated %u sched_trace_xxx() events per CPU "
                "(buffer size: %d bytes)\n",
-               NO_EVENTS, (int) sizeof(struct local_buffer));
+               NUM_EVENTS, (int) sizeof(struct local_buffer));
 
         err = ftdev_init(&st_dev, THIS_MODULE,
                         num_online_cpus(), "sched_trace");
@@ -50,7 +51,7 @@ static int __init init_sched_task_trace(void)
 
         for (i = 0; i < st_dev.minor_cnt; i++) {
                 buf = &per_cpu(st_event_buffer, i);
-                ok += init_ft_buffer(&buf->ftbuf, NO_EVENTS,
+                ok += init_ft_buffer(&buf->ftbuf, NUM_EVENTS,
                                      sizeof(struct st_event_record),
                                      buf->flag,
                                      buf->record);
@@ -154,7 +155,8 @@ feather_callback void do_sched_trace_task_switch_to(unsigned long id,
 {
         struct task_struct *t = (struct task_struct*) _task;
         struct st_event_record* rec;
-        if (is_realtime(t)) {
+        //if (is_realtime(t))  /* comment out to trace EVERYTHING */
+        {
                 rec = get_record(ST_SWITCH_TO, t);
                 if (rec) {
                         rec->data.switch_to.when      = now();
@@ -169,7 +171,8 @@ feather_callback void do_sched_trace_task_switch_away(unsigned long id,
 {
         struct task_struct *t = (struct task_struct*) _task;
         struct st_event_record* rec;
-        if (is_realtime(t)) {
+        //if (is_realtime(t))  /* comment out to trace EVERYTHING */
+        {
                 rec = get_record(ST_SWITCH_AWAY, t);
                 if (rec) {
                         rec->data.switch_away.when      = now();
@@ -188,6 +191,9 @@ feather_callback void do_sched_trace_task_completion(unsigned long id,
         if (rec) {
                 rec->data.completion.when   = now();
                 rec->data.completion.forced = forced;
+#ifdef LITMUS_NVIDIA
+                rec->data.completion.nv_int_count = (u16)atomic_read(&tsk_rt(t)->nv_int_count);
+#endif
                 put_record(rec);
         }
 }
@@ -239,3 +245,265 @@ feather_callback void do_sched_trace_action(unsigned long id,
                 put_record(rec);
         }
 }
+
+
+
+
+feather_callback void do_sched_trace_prediction_err(unsigned long id,
+                                                                                                        unsigned long _task,
+                                                                                                        unsigned long _distance,
+                                                                                                        unsigned long _rel_err)
+{
+        struct task_struct *t = (struct task_struct*) _task;
+        struct st_event_record *rec = get_record(ST_PREDICTION_ERR, t);
+
+        if (rec) {
+                gpu_migration_dist_t* distance = (gpu_migration_dist_t*) _distance;
+                fp_t* rel_err = (fp_t*) _rel_err;
+
+                rec->data.prediction_err.distance = *distance;
+                rec->data.prediction_err.rel_err = rel_err->val;
+                put_record(rec);
+        }
+}
+
+
+feather_callback void do_sched_trace_migration(unsigned long id,
+                                                                                                        unsigned long _task,
+                                                                                                        unsigned long _mig_info)
+{
+        struct task_struct *t = (struct task_struct*) _task;
+        struct st_event_record *rec = get_record(ST_MIGRATION, t);
+
+        if (rec) {
+                struct migration_info* mig_info = (struct migration_info*) _mig_info;
+
+                rec->hdr.extra = mig_info->distance;
+                rec->data.migration.observed = mig_info->observed;
+                rec->data.migration.estimated = mig_info->estimated;
+
+                put_record(rec);
+        }
+}
+
+
+
+
+
+
+
+
+
+feather_callback void do_sched_trace_tasklet_release(unsigned long id,
+                                                                                                   unsigned long _owner)
+{
+        struct task_struct *t = (struct task_struct*) _owner;
+        struct st_event_record *rec = get_record(ST_TASKLET_RELEASE, t);
+
+        if (rec) {
+                rec->data.tasklet_release.when = now();
+                put_record(rec);
+        }
+}
+
+
+feather_callback void do_sched_trace_tasklet_begin(unsigned long id,
+                                                                                                   unsigned long _owner)
+{
+        struct task_struct *t = (struct task_struct*) _owner;
+        struct st_event_record *rec = get_record(ST_TASKLET_BEGIN, t);
+
+        if (rec) {
+                rec->data.tasklet_begin.when = now();
+
+                if(!in_interrupt())
+                        rec->data.tasklet_begin.exe_pid = current->pid;
+                else
+                        rec->data.tasklet_begin.exe_pid = 0;
+
+                put_record(rec);
+        }
+}
+EXPORT_SYMBOL(do_sched_trace_tasklet_begin);
+
+
+feather_callback void do_sched_trace_tasklet_end(unsigned long id,
+                                                                                                 unsigned long _owner,
+                                                                                                 unsigned long _flushed)
+{
+        struct task_struct *t = (struct task_struct*) _owner;
+        struct st_event_record *rec = get_record(ST_TASKLET_END, t);
+
+        if (rec) {
+                rec->data.tasklet_end.when = now();
+                rec->data.tasklet_end.flushed = _flushed;
+
+                if(!in_interrupt())
+                        rec->data.tasklet_end.exe_pid = current->pid;
+                else
+                        rec->data.tasklet_end.exe_pid = 0;
+
+                put_record(rec);
+        }
+}
+EXPORT_SYMBOL(do_sched_trace_tasklet_end);
+
+
+feather_callback void do_sched_trace_work_release(unsigned long id,
+                                                                                                         unsigned long _owner)
+{
+        struct task_struct *t = (struct task_struct*) _owner;
+        struct st_event_record *rec = get_record(ST_WORK_RELEASE, t);
+
+        if (rec) {
+                rec->data.work_release.when = now();
+                put_record(rec);
+        }
+}
+
+
+feather_callback void do_sched_trace_work_begin(unsigned long id,
+                                                                                                unsigned long _owner,
+                                                                                                unsigned long _exe)
+{
+        struct task_struct *t = (struct task_struct*) _owner;
+        struct st_event_record *rec = get_record(ST_WORK_BEGIN, t);
+
+        if (rec) {
+                struct task_struct *exe = (struct task_struct*) _exe;
+                rec->data.work_begin.exe_pid = exe->pid;
+                rec->data.work_begin.when = now();
+                put_record(rec);
+        }
+}
+EXPORT_SYMBOL(do_sched_trace_work_begin);
+
+
+feather_callback void do_sched_trace_work_end(unsigned long id,
+                                                                                          unsigned long _owner,
+                                                                                          unsigned long _exe,
+                                                                                          unsigned long _flushed)
+{
+        struct task_struct *t = (struct task_struct*) _owner;
+        struct st_event_record *rec = get_record(ST_WORK_END, t);
+
+        if (rec) {
+                struct task_struct *exe = (struct task_struct*) _exe;
+                rec->data.work_end.exe_pid = exe->pid;
+                rec->data.work_end.flushed = _flushed;
+                rec->data.work_end.when = now();
+                put_record(rec);
+        }
+}
+EXPORT_SYMBOL(do_sched_trace_work_end);
+
+
+feather_callback void do_sched_trace_eff_prio_change(unsigned long id,
+                                                                                          unsigned long _task,
+                                                                                          unsigned long _inh)
+{
+        struct task_struct *t = (struct task_struct*) _task;
+        struct st_event_record *rec = get_record(ST_EFF_PRIO_CHANGE, t);
+
+        if (rec) {
+                struct task_struct *inh = (struct task_struct*) _inh;
+                rec->data.effective_priority_change.when = now();
+                rec->data.effective_priority_change.inh_pid = (inh != NULL) ?
+                        inh->pid :
+                        0xffff;
+
+                put_record(rec);
+        }
+}
+
+/* pray for no nesting of nv interrupts on same CPU... */
+struct tracing_interrupt_map
+{
+        int active;
+        int count;
+        unsigned long data[128]; // assume nesting less than 128...
+        unsigned long serial[128];
+};
+DEFINE_PER_CPU(struct tracing_interrupt_map, active_interrupt_tracing);
+
+
+DEFINE_PER_CPU(u32, intCounter);
+
+feather_callback void do_sched_trace_nv_interrupt_begin(unsigned long id,
+                                                                                                unsigned long _device)
+{
+        struct st_event_record *rec;
+        u32 serialNum;
+
+        {
+                u32* serial;
+                struct tracing_interrupt_map* int_map = &per_cpu(active_interrupt_tracing, smp_processor_id());
+                if(!int_map->active == 0xcafebabe)
+                {
+                        int_map->count++;
+                }
+                else
+                {
+                        int_map->active = 0xcafebabe;
+                        int_map->count = 1;
+                }
+                //int_map->data[int_map->count-1] = _device;
+
+                serial = &per_cpu(intCounter, smp_processor_id());
+                *serial += num_online_cpus();
+                serialNum = *serial;
+                int_map->serial[int_map->count-1] = serialNum;
+        }
+
+        rec = get_record(ST_NV_INTERRUPT_BEGIN, NULL);
+        if(rec) {
+                u32 device = _device;
+                rec->data.nv_interrupt_begin.when = now();
+                rec->data.nv_interrupt_begin.device = device;
+                rec->data.nv_interrupt_begin.serialNumber = serialNum;
+                put_record(rec);
+        }
+}
+EXPORT_SYMBOL(do_sched_trace_nv_interrupt_begin);
+
+/*
+int is_interrupt_tracing_active(void)
+{
+        struct tracing_interrupt_map* int_map = &per_cpu(active_interrupt_tracing, smp_processor_id());
+        if(int_map->active == 0xcafebabe)
+                return 1;
+        return 0;
+}
+*/
+
+feather_callback void do_sched_trace_nv_interrupt_end(unsigned long id, unsigned long _device)
+{
+        struct tracing_interrupt_map* int_map = &per_cpu(active_interrupt_tracing, smp_processor_id());
+        if(int_map->active == 0xcafebabe)
+        {
+                struct st_event_record *rec = get_record(ST_NV_INTERRUPT_END, NULL);
+
+                int_map->count--;
+                if(int_map->count == 0)
+                        int_map->active = 0;
+
+                if(rec) {
+                        u32 device = _device;
+                        rec->data.nv_interrupt_end.when = now();
+                        //rec->data.nv_interrupt_end.device = int_map->data[int_map->count];
+                        rec->data.nv_interrupt_end.device = device;
+                        rec->data.nv_interrupt_end.serialNumber = int_map->serial[int_map->count];
+                        put_record(rec);
+                }
+        }
+}
+EXPORT_SYMBOL(do_sched_trace_nv_interrupt_end);
+
+
+
+
+
+
+
+
+