Merge branch 'wip-merge-2.6.36' of ssh://cvs.cs.unc.edu/cvs/proj/litmus/repo/litmus2010 into wip-merge-2.6.36

16 files changed, 263 insertions, 230 deletions

diff --git a/litmus/Kconfig b/litmus/Kconfig
index 9888589ef126..d62c417f261e 100644
--- a/litmus/Kconfig
+++ b/litmus/Kconfig
@@ -108,6 +108,26 @@ config SCHED_TASK_TRACE
           Say Yes for debugging.
           Say No for overhead tracing.
 
+config SCHED_TASK_TRACE_SHIFT
+       int "Buffer size for sched_trace_xxx() events"
+       depends on SCHED_TASK_TRACE
+       range 8 13
+       default 9
+       help
+
+         Select the buffer size of sched_trace_xxx() events as a power of two.
+         These buffers are statically allocated as per-CPU data. Each event
+         requires 24 bytes storage plus one additional flag byte. Too large
+         buffers can cause issues with the per-cpu allocator (and waste
+         memory). Too small buffers can cause scheduling events to be lost. The
+         "right" size is workload dependent and depends on the number of tasks,
+         each task's period, each task's number of suspensions, and how often
+         the buffer is flushed.
+
+         Examples: 12 =>   4k events
+                   10 =>   1k events
+                    8 =>  512 events
+
 config SCHED_OVERHEAD_TRACE
         bool "Record timestamps for overhead measurements"
         depends on FEATHER_TRACE
@@ -129,6 +149,31 @@ config SCHED_DEBUG_TRACE
           Say Yes for debugging.
           Say No for overhead tracing.
 
+config SCHED_DEBUG_TRACE_SHIFT
+       int "Buffer size for TRACE() buffer"
+       depends on SCHED_DEBUG_TRACE
+       range 14 22
+       default 18
+       help
+
+        Select the amount of memory needed per for the TRACE() buffer, as a
+        power of two. The TRACE() buffer is global and statically allocated. If
+        the buffer is too small, there will be holes in the TRACE() log if the
+        buffer-flushing task is starved.
+
+        The default should be sufficient for most systems. Increase the buffer
+        size if the log contains holes. Reduce the buffer size when running on
+        a memory-constrained system.
+
+        Examples: 14 =>  16KB
+                  18 => 256KB
+                  20 =>   1MB
+
+        This buffer is exported to usespace using a misc device as
+        'litmus/log'. On a system with default udev rules, a corresponding
+        character device node should be created at /dev/litmus/log. The buffer
+        can be flushed using cat, e.g., 'cat /dev/litmus/log > my_log_file.txt'.
+
 endmenu
 
 endmenu
diff --git a/litmus/Makefile b/litmus/Makefile
index 7bd1abdcb84a..b7366b530749 100644
--- a/litmus/Makefile
+++ b/litmus/Makefile
@@ -3,6 +3,7 @@
 #
 
 obj-y     = sched_plugin.o litmus.o \
+            preempt.o \
             litmus_proc.o \
             budget.o \
             jobs.o \
diff --git a/litmus/budget.c b/litmus/budget.c
index b99177a26313..310e9a3d4172 100644
--- a/litmus/budget.c
+++ b/litmus/budget.c
@@ -1,7 +1,9 @@
+#include <linux/sched.h>
 #include <linux/percpu.h>
 #include <linux/hrtimer.h>
 
 #include <litmus/litmus.h>
+#include <litmus/preempt.h>
 
 struct enforcement_timer {
         /* The enforcement timer is used to accurately police
@@ -23,7 +25,7 @@ static enum hrtimer_restart on_enforcement_timeout(struct hrtimer *timer)
         TRACE("enforcement timer fired.\n");
         et->armed = 0;
         /* activate scheduler */
-        set_tsk_need_resched(current);
+        litmus_reschedule_local();
         local_irq_restore(flags);
 
         return  HRTIMER_NORESTART;
diff --git a/litmus/litmus.c b/litmus/litmus.c
index 99c35ac99870..0756d0156f8f 100644
--- a/litmus/litmus.c
+++ b/litmus/litmus.c
@@ -6,20 +6,16 @@
 #include <asm/uaccess.h>
 #include <linux/uaccess.h>
 #include <linux/sysrq.h>
-
+#include <linux/sched.h>
 #include <linux/module.h>
 #include <linux/slab.h>
 
 #include <litmus/litmus.h>
-#include <linux/sched.h>
-
 #include <litmus/bheap.h>
-
 #include <litmus/trace.h>
-
 #include <litmus/rt_domain.h>
-
 #include <litmus/litmus_proc.h>
+#include <litmus/sched_trace.h>
 
 /* Number of RT tasks that exist in the system */
 atomic_t rt_task_count          = ATOMIC_INIT(0);
diff --git a/litmus/litmus_proc.c b/litmus/litmus_proc.c
index c10a6a6d3975..81ea5c35d291 100644
--- a/litmus/litmus_proc.c
+++ b/litmus/litmus_proc.c
@@ -2,6 +2,7 @@
  * litmus_proc.c -- Implementation of the /proc/litmus directory tree.
  */
 
+#include <linux/sched.h>
 #include <linux/uaccess.h>
 
 #include <litmus/litmus.h>
diff --git a/litmus/preempt.c b/litmus/preempt.c
new file mode 100644
index 000000000000..ebe2e3461895
--- /dev/null
+++ b/litmus/preempt.c
@@ -0,0 +1,131 @@
+#include <linux/sched.h>
+
+#include <litmus/litmus.h>
+#include <litmus/preempt.h>
+
+/* The rescheduling state of each processor.
+ */
+DEFINE_PER_CPU_SHARED_ALIGNED(atomic_t, resched_state);
+
+void sched_state_will_schedule(struct task_struct* tsk)
+{
+        /* Litmus hack: we only care about processor-local invocations of
+         * set_tsk_need_resched(). We can't reliably set the flag remotely
+         * since it might race with other updates to the scheduling state.  We
+         * can't rely on the runqueue lock protecting updates to the sched
+         * state since processors do not acquire the runqueue locks for all
+         * updates to the sched state (to avoid acquiring two runqueue locks at
+         * the same time). Further, if tsk is residing on a remote processor,
+         * then that processor doesn't actually know yet that it is going to
+         * reschedule; it still must receive an IPI (unless a local invocation
+         * races).
+         */
+        if (likely(task_cpu(tsk) == smp_processor_id())) {
+                VERIFY_SCHED_STATE(TASK_SCHEDULED | SHOULD_SCHEDULE | TASK_PICKED | WILL_SCHEDULE);
+                if (is_in_sched_state(TASK_PICKED | PICKED_WRONG_TASK))
+                        set_sched_state(PICKED_WRONG_TASK);
+                else
+                        set_sched_state(WILL_SCHEDULE);
+        } else
+                /* Litmus tasks should never be subject to a remote
+                 * set_tsk_need_resched(). */
+                BUG_ON(is_realtime(tsk));
+        TRACE_TASK(tsk, "set_tsk_need_resched() ret:%p\n",
+                   __builtin_return_address(0));
+}
+
+/* Called by the IPI handler after another CPU called smp_send_resched(). */
+void sched_state_ipi(void)
+{
+        /* If the IPI was slow, we might be in any state right now. The IPI is
+         * only meaningful if we are in SHOULD_SCHEDULE. */
+        if (is_in_sched_state(SHOULD_SCHEDULE)) {
+                /* Cause scheduler to be invoked.
+                 * This will cause a transition to WILL_SCHEDULE. */
+                set_tsk_need_resched(current);
+                TRACE_STATE("IPI -> set_tsk_need_resched(%s/%d)\n",
+                            current->comm, current->pid);
+        } else {
+                /* ignore */
+                TRACE_STATE("ignoring IPI in state %x (%s)\n",
+                            get_sched_state(),
+                            sched_state_name(get_sched_state()));
+        }
+}
+
+/* Called by plugins to cause a CPU to reschedule. IMPORTANT: the caller must
+ * hold the lock that is used to serialize scheduling decisions. */
+void litmus_reschedule(int cpu)
+{
+        int picked_transition_ok = 0;
+        int scheduled_transition_ok = 0;
+
+        /* The (remote) CPU could be in any state. */
+
+        /* The critical states are TASK_PICKED and TASK_SCHEDULED, as the CPU
+         * is not aware of the need to reschedule at this point. */
+
+        /* is a context switch in progress? */
+        if (cpu_is_in_sched_state(cpu, TASK_PICKED))
+                picked_transition_ok = sched_state_transition_on(
+                        cpu, TASK_PICKED, PICKED_WRONG_TASK);
+
+        if (!picked_transition_ok &&
+            cpu_is_in_sched_state(cpu, TASK_SCHEDULED)) {
+                /* We either raced with the end of the context switch, or the
+                 * CPU was in TASK_SCHEDULED anyway. */
+                scheduled_transition_ok = sched_state_transition_on(
+                        cpu, TASK_SCHEDULED, SHOULD_SCHEDULE);
+        }
+
+        /* If the CPU was in state TASK_SCHEDULED, then we need to cause the
+         * scheduler to be invoked. */
+        if (scheduled_transition_ok) {
+                if (smp_processor_id() == cpu)
+                        set_tsk_need_resched(current);
+                else
+                        smp_send_reschedule(cpu);
+        }
+
+        TRACE_STATE("%s picked-ok:%d sched-ok:%d\n",
+                    __FUNCTION__,
+                    picked_transition_ok,
+                    scheduled_transition_ok);
+}
+
+void litmus_reschedule_local(void)
+{
+        if (is_in_sched_state(TASK_PICKED))
+                set_sched_state(PICKED_WRONG_TASK);
+        else if (is_in_sched_state(TASK_SCHEDULED | SHOULD_SCHEDULE)) {
+                set_sched_state(WILL_SCHEDULE);
+                set_tsk_need_resched(current);
+        }
+}
+
+#ifdef CONFIG_DEBUG_KERNEL
+
+void sched_state_plugin_check(void)
+{
+        if (!is_in_sched_state(TASK_PICKED | PICKED_WRONG_TASK)) {
+                TRACE("!!!! plugin did not call sched_state_task_picked()!"
+                      "Calling sched_state_task_picked() is mandatory---fix this.\n");
+                set_sched_state(TASK_PICKED);
+        }
+}
+
+#define NAME_CHECK(x) case x:  return #x
+const char* sched_state_name(int s)
+{
+        switch (s) {
+                NAME_CHECK(TASK_SCHEDULED);
+                NAME_CHECK(SHOULD_SCHEDULE);
+                NAME_CHECK(WILL_SCHEDULE);
+                NAME_CHECK(TASK_PICKED);
+                NAME_CHECK(PICKED_WRONG_TASK);
+        default:
+                return "UNKNOWN";
+        };
+}
+
+#endif
diff --git a/litmus/sched_cedf.c b/litmus/sched_cedf.c
index a729d97535e9..111e4fb1c62b 100644
--- a/litmus/sched_cedf.c
+++ b/litmus/sched_cedf.c
@@ -34,6 +34,7 @@
 
 #include <litmus/litmus.h>
 #include <litmus/jobs.h>
+#include <litmus/preempt.h>
 #include <litmus/sched_plugin.h>
 #include <litmus/edf_common.h>
 #include <litmus/sched_trace.h>
@@ -209,12 +210,6 @@ static noinline void unlink(struct task_struct* t)
 {
         cpu_entry_t *entry;
 
-        if (unlikely(!t)) {
-                TRACE_BUG_ON(!t);
-                return;
-        }
-
-
         if (t->rt_param.linked_on != NO_CPU) {
                 /* unlink */
                 entry = &per_cpu(cedf_cpu_entries, t->rt_param.linked_on);
@@ -341,7 +336,7 @@ static void cedf_tick(struct task_struct* t)
                         /* np tasks will be preempted when they become
                          * preemptable again
                          */
-                        set_tsk_need_resched(t);
+                        litmus_reschedule_local();
                         set_will_schedule();
                         TRACE("cedf_scheduler_tick: "
                               "%d is preemptable "
@@ -466,6 +461,7 @@ static struct task_struct* cedf_schedule(struct task_struct * prev)
                 if (exists)
                         next = prev;
 
+        sched_state_task_picked();
         raw_spin_unlock(&cluster->lock);
 
 #ifdef WANT_ALL_SCHED_EVENTS
diff --git a/litmus/sched_gsn_edf.c b/litmus/sched_gsn_edf.c
index e101768740ad..e9c5e531b1ae 100644
--- a/litmus/sched_gsn_edf.c
+++ b/litmus/sched_gsn_edf.c
@@ -18,6 +18,8 @@
 #include <litmus/edf_common.h>
 #include <litmus/sched_trace.h>
 
+#include <litmus/preempt.h>
+
 #include <litmus/bheap.h>
 
 #include <linux/module.h>
@@ -95,21 +97,12 @@ typedef struct  {
         int                     cpu;
         struct task_struct*     linked;         /* only RT tasks */
         struct task_struct*     scheduled;      /* only RT tasks */
-        atomic_t                will_schedule;  /* prevent unneeded IPIs */
         struct bheap_node*      hn;
 } cpu_entry_t;
 DEFINE_PER_CPU(cpu_entry_t, gsnedf_cpu_entries);
 
 cpu_entry_t* gsnedf_cpus[NR_CPUS];
 
-#define set_will_schedule() \
-        (atomic_set(&__get_cpu_var(gsnedf_cpu_entries).will_schedule, 1))
-#define clear_will_schedule() \
-        (atomic_set(&__get_cpu_var(gsnedf_cpu_entries).will_schedule, 0))
-#define test_will_schedule(cpu) \
-        (atomic_read(&per_cpu(gsnedf_cpu_entries, cpu).will_schedule))
-
-
 /* the cpus queue themselves according to priority in here */
 static struct bheap_node gsnedf_heap_node[NR_CPUS];
 static struct bheap      gsnedf_cpu_heap;
@@ -217,11 +210,6 @@ static noinline void unlink(struct task_struct* t)
 {
         cpu_entry_t *entry;
 
-        if (unlikely(!t)) {
-                TRACE_BUG_ON(!t);
-                return;
-        }
-
         if (t->rt_param.linked_on != NO_CPU) {
                 /* unlink */
                 entry = &per_cpu(gsnedf_cpu_entries, t->rt_param.linked_on);
@@ -341,8 +329,7 @@ static void gsnedf_tick(struct task_struct* t)
                         /* np tasks will be preempted when they become
                          * preemptable again
                          */
-                        set_tsk_need_resched(t);
-                        set_will_schedule();
+                        litmus_reschedule_local();
                         TRACE("gsnedf_scheduler_tick: "
                               "%d is preemptable "
                               " => FORCE_RESCHED\n", t->pid);
@@ -391,7 +378,6 @@ static struct task_struct* gsnedf_schedule(struct task_struct * prev)
 #endif
 
         raw_spin_lock(&gsnedf_lock);
-        clear_will_schedule();
 
         /* sanity checking */
         BUG_ON(entry->scheduled && entry->scheduled != prev);
@@ -473,6 +459,8 @@ static struct task_struct* gsnedf_schedule(struct task_struct * prev)
                 if (exists)
                         next = prev;
 
+        sched_state_task_picked();
+
         raw_spin_unlock(&gsnedf_lock);
 
 #ifdef WANT_ALL_SCHED_EVENTS
@@ -780,7 +768,6 @@ static long gsnedf_activate_plugin(void)
         for_each_online_cpu(cpu) {
                 entry = &per_cpu(gsnedf_cpu_entries, cpu);
                 bheap_node_init(&entry->hn, entry);
-                atomic_set(&entry->will_schedule, 0);
                 entry->linked    = NULL;
                 entry->scheduled = NULL;
 #ifdef CONFIG_RELEASE_MASTER
@@ -829,7 +816,6 @@ static int __init init_gsn_edf(void)
         for (cpu = 0; cpu < NR_CPUS; cpu++)  {
                 entry = &per_cpu(gsnedf_cpu_entries, cpu);
                 gsnedf_cpus[cpu] = entry;
-                atomic_set(&entry->will_schedule, 0);
                 entry->cpu       = cpu;
                 entry->hn        = &gsnedf_heap_node[cpu];
                 bheap_node_init(&entry->hn, entry);
diff --git a/litmus/sched_litmus.c b/litmus/sched_litmus.c
index 65873152e68f..e6952896dc4b 100644
--- a/litmus/sched_litmus.c
+++ b/litmus/sched_litmus.c
@@ -3,6 +3,7 @@
 #include <litmus/litmus.h>
 #include <litmus/budget.h>
 #include <litmus/sched_plugin.h>
+#include <litmus/preempt.h>
 
 static void update_time_litmus(struct rq *rq, struct task_struct *p)
 {
@@ -52,6 +53,8 @@ litmus_schedule(struct rq *rq, struct task_struct *prev)
         /* let the plugin schedule */
         next = litmus->schedule(prev);
 
+        sched_state_plugin_check();
+
         /* check if a global plugin pulled a task from a different RQ */
         if (next && task_rq(next) != rq) {
                 /* we need to migrate the task */
@@ -198,7 +201,7 @@ static void yield_task_litmus(struct rq *rq)
          * then determine if a preemption is still required.
          */
         clear_exit_np(current);
-        set_tsk_need_resched(current);
+        litmus_reschedule_local();
 }
 
 /* Plugins are responsible for this.
diff --git a/litmus/sched_pfair.c b/litmus/sched_pfair.c
index ea77d3295290..c7d5cf7aa2b3 100644
--- a/litmus/sched_pfair.c
+++ b/litmus/sched_pfair.c
@@ -16,6 +16,7 @@
 
 #include <litmus/litmus.h>
 #include <litmus/jobs.h>
+#include <litmus/preempt.h>
 #include <litmus/rt_domain.h>
 #include <litmus/sched_plugin.h>
 #include <litmus/sched_trace.h>
@@ -241,11 +242,7 @@ static void check_preempt(struct task_struct* t)
                 PTRACE_TASK(t, "linked_on:%d, scheduled_on:%d\n",
                            tsk_rt(t)->linked_on, tsk_rt(t)->scheduled_on);
                 /* preempt */
-                if (cpu == smp_processor_id())
-                        set_tsk_need_resched(current);
-                else {
-                        smp_send_reschedule(cpu);
-                }
+                litmus_reschedule(cpu);
         }
 }
 
@@ -545,7 +542,7 @@ static void pfair_tick(struct task_struct* t)
 
         if (state->local != current
             && (is_realtime(current) || is_present(state->local)))
-                set_tsk_need_resched(current);
+                litmus_reschedule_local();
 }
 
 static int safe_to_schedule(struct task_struct* t, int cpu)
@@ -577,7 +574,7 @@ static struct task_struct* pfair_schedule(struct task_struct * prev)
                 if (next)
                         tsk_rt(next)->scheduled_on = state->cpu;
         }
-
+        sched_state_task_picked();
         raw_spin_unlock(&pfair_lock);
 
         if (next)
diff --git a/litmus/sched_plugin.c b/litmus/sched_plugin.c
index d706a08fb313..d912a6494d20 100644
--- a/litmus/sched_plugin.c
+++ b/litmus/sched_plugin.c
@@ -6,10 +6,11 @@
 
 #include <linux/list.h>
 #include <linux/spinlock.h>
+#include <linux/sched.h>
 
 #include <litmus/litmus.h>
 #include <litmus/sched_plugin.h>
-
+#include <litmus/preempt.h>
 #include <litmus/jobs.h>
 
 /*
@@ -18,36 +19,30 @@
  * non-preemptive section aware and does not invoke the scheduler / send
  * IPIs if the to-be-preempted task is actually non-preemptive.
  */
-void preempt_if_preemptable(struct task_struct* t, int on_cpu)
+void preempt_if_preemptable(struct task_struct* t, int cpu)
 {
         /* t is the real-time task executing on CPU on_cpu If t is NULL, then
          * on_cpu is currently scheduling background work.
          */
 
-        int send_ipi;
+        int reschedule = 0;
 
-        if (smp_processor_id() == on_cpu) {
-                /* local CPU case */
-                if (t) {
+        if (!t)
+                /* move non-real-time task out of the way */
+                reschedule = 1;
+        else {
+                if (smp_processor_id() == cpu) {
+                        /* local CPU case */
                         /* check if we need to poke userspace */
                         if (is_user_np(t))
                                 /* yes, poke it */
                                 request_exit_np(t);
-                        else
-                                /* no, see if we are allowed to preempt the
+                        else if (!is_kernel_np(t))
+                                /* only if we are allowed to preempt the
                                  * currently-executing task */
-                                if (!is_kernel_np(t))
-                                        set_tsk_need_resched(t);
-                } else
-                        /* move non-real-time task out of the way */
-                        set_tsk_need_resched(current);
-        } else {
-                /* remote CPU case */
-                if (!t)
-                        /* currently schedules non-real-time work */
-                        send_ipi = 1;
-                else {
-                        /* currently schedules real-time work */
+                                reschedule = 1;
+                } else {
+                        /* remote CPU case */
                         if (is_user_np(t)) {
                                 /* need to notify user space of delayed
                                  * preemption */
@@ -59,14 +54,14 @@ void preempt_if_preemptable(struct task_struct* t, int on_cpu)
                                 mb();
                         }
                         /* Only send an ipi if remote task might have raced our
-                         * request, i.e., send an IPI to make sure if it exited
-                         * its critical section.
+                         * request, i.e., send an IPI to make sure in case it
+                         * exited its critical section.
                          */
-                        send_ipi = !is_np(t) && !is_kernel_np(t);
+                        reschedule = !is_np(t) && !is_kernel_np(t);
                 }
-                if (likely(send_ipi))
-                        smp_send_reschedule(on_cpu);
         }
+        if (likely(reschedule))
+                litmus_reschedule(cpu);
 }
 
 
@@ -80,6 +75,7 @@ static void litmus_dummy_finish_switch(struct task_struct * prev)
 
 static struct task_struct* litmus_dummy_schedule(struct task_struct * prev)
 {
+        sched_state_task_picked();
         return NULL;
 }
 
diff --git a/litmus/sched_psn_edf.c b/litmus/sched_psn_edf.c
index 73f64730bd5e..b89823d5c026 100644
--- a/litmus/sched_psn_edf.c
+++ b/litmus/sched_psn_edf.c
@@ -12,14 +12,14 @@
 #include <linux/sched.h>
 #include <linux/list.h>
 #include <linux/spinlock.h>
-
 #include <linux/module.h>
 
 #include <litmus/litmus.h>
 #include <litmus/jobs.h>
+#include <litmus/preempt.h>
 #include <litmus/sched_plugin.h>
 #include <litmus/edf_common.h>
-
+#include <litmus/sched_trace.h>
 
 typedef struct {
         rt_domain_t             domain;
@@ -109,7 +109,7 @@ static void psnedf_tick(struct task_struct *t)
 
         if (is_realtime(t) && budget_enforced(t) && budget_exhausted(t)) {
                 if (!is_np(t)) {
-                        set_tsk_need_resched(t);
+                        litmus_reschedule_local();
                         TRACE("psnedf_scheduler_tick: "
                               "%d is preemptable "
                               " => FORCE_RESCHED\n", t->pid);
@@ -205,6 +205,7 @@ static struct task_struct* psnedf_schedule(struct task_struct * prev)
         }
 
         pedf->scheduled = next;
+        sched_state_task_picked();
         raw_spin_unlock(&pedf->slock);
 
         return next;
diff --git a/litmus/sched_task_trace.c b/litmus/sched_task_trace.c
index 39a543e22d41..e5098ddb1ec9 100644
--- a/litmus/sched_task_trace.c
+++ b/litmus/sched_task_trace.c
@@ -18,7 +18,7 @@
 
 /* set MAJOR to 0 to have it dynamically assigned */
 #define FT_TASK_TRACE_MAJOR     253
-#define NO_EVENTS               4096 /* this is a buffer of 12 4k pages per CPU */
+#define NO_EVENTS               (1 << CONFIG_SCHED_TASK_TRACE_SHIFT)
 
 #define now() litmus_clock()
 
@@ -41,6 +41,9 @@ static int __init init_sched_task_trace(void)
 {
         struct local_buffer* buf;
         int i, ok = 0;
+        printk("Allocated %u sched_trace_xxx() events per CPU "
+               "(buffer size: %d bytes)\n",
+               NO_EVENTS, (int) sizeof(struct local_buffer));
         ftdev_init(&st_dev, THIS_MODULE);
         for (i = 0; i < NR_CPUS; i++) {
                 buf = &per_cpu(st_event_buffer, i);
diff --git a/litmus/sched_trace.c b/litmus/sched_trace.c
index 8051b51a2022..f4171fddbbb1 100644
--- a/litmus/sched_trace.c
+++ b/litmus/sched_trace.c
@@ -2,7 +2,7 @@
  * sched_trace.c -- record scheduling events to a byte stream.
  */
 #include <linux/spinlock.h>
-#include <linux/semaphore.h>
+#include <linux/mutex.h>
 
 #include <linux/fs.h>
 #include <linux/slab.h>
@@ -18,133 +18,23 @@
 
 #define SCHED_TRACE_NAME "litmus/log"
 
-/* Allocate a buffer of about 32k per CPU */
-#define LITMUS_TRACE_BUF_PAGES 8
-#define LITMUS_TRACE_BUF_SIZE (PAGE_SIZE * LITMUS_TRACE_BUF_PAGES * NR_CPUS)
+/* Compute size of TRACE() buffer */
+#define LITMUS_TRACE_BUF_SIZE (1 << CONFIG_SCHED_DEBUG_TRACE_SHIFT)
 
 /* Max length of one read from the buffer */
 #define MAX_READ_LEN (64 * 1024)
 
-/* Max length for one write --- from kernel --- to the buffer */
+/* Max length for one write --- by TRACE() --- to the buffer. This is used to
+ * allocate a per-cpu buffer for printf() formatting. */
 #define MSG_SIZE 255
 
-/* Inner ring buffer structure */
-typedef struct {
-        rwlock_t        del_lock;
 
-        /* the buffer */
-        struct kfifo    kfifo;
-} ring_buffer_t;
+static DEFINE_MUTEX(reader_mutex);
+static atomic_t reader_cnt = ATOMIC_INIT(0);
+static DEFINE_KFIFO(debug_buffer, char, LITMUS_TRACE_BUF_SIZE);
 
-/* Main buffer structure */
-typedef struct {
-        ring_buffer_t           buf;
-        atomic_t                reader_cnt;
-        struct semaphore        reader_mutex;
-} trace_buffer_t;
 
-
-/*
- * Inner buffer management functions
- */
-void rb_init(ring_buffer_t* buf)
-{
-        rwlock_init(&buf->del_lock);
-}
-
-int rb_alloc_buf(ring_buffer_t* buf, unsigned int size)
-{
-        unsigned long flags;
-        int ret = 0;
-
-        write_lock_irqsave(&buf->del_lock, flags);
-
-        /* kfifo size must be a power of 2
-         * atm kfifo alloc is automatically rounding the size
-         */
-        ret = kfifo_alloc(&buf->kfifo, size, GFP_ATOMIC);
-
-        write_unlock_irqrestore(&buf->del_lock, flags);
-
-        if(ret < 0)
-                printk(KERN_ERR "kfifo_alloc failed\n");
-
-        return ret;
-}
-
-int rb_free_buf(ring_buffer_t* buf)
-{
-        unsigned long flags;
-
-        write_lock_irqsave(&buf->del_lock, flags);
-
-        BUG_ON(!kfifo_initialized(&buf->kfifo));
-        kfifo_free(&buf->kfifo);
-
-        write_unlock_irqrestore(&buf->del_lock, flags);
-
-        return 0;
-}
-
-/*
- * Assumption: concurrent writes are serialized externally
- *
- * Will only succeed if there is enough space for all len bytes.
- */
-int rb_put(ring_buffer_t* buf, char* mem, size_t len)
-{
-        unsigned long flags;
-        int error = 0;
-
-        read_lock_irqsave(&buf->del_lock, flags);
-
-        if (!kfifo_initialized(&buf->kfifo)) {
-                error = -ENODEV;
-                goto out;
-        }
-
-        if((kfifo_in(&buf->kfifo, mem, len)) < len) {
-                error = -ENOMEM;
-                goto out;
-        }
-
- out:
-        read_unlock_irqrestore(&buf->del_lock, flags);
-        return error;
-}
-
-/* Assumption: concurrent reads are serialized externally */
-int rb_get(ring_buffer_t* buf, char* mem, size_t len)
-{
-        unsigned long flags;
-        int error = 0;
-
-        read_lock_irqsave(&buf->del_lock, flags);
-        if (!kfifo_initialized(&buf->kfifo)) {
-                error = -ENODEV;
-                goto out;
-        }
-
-        error = kfifo_out(&buf->kfifo, (unsigned char*)mem, len);
-
- out:
-        read_unlock_irqrestore(&buf->del_lock, flags);
-        return error;
-}
-
-/*
- * Device Driver management
- */
 static DEFINE_RAW_SPINLOCK(log_buffer_lock);
-static trace_buffer_t log_buffer;
-
-static void init_log_buffer(void)
-{
-        rb_init(&log_buffer.buf);
-        atomic_set(&log_buffer.reader_cnt,0);
-        init_MUTEX(&log_buffer.reader_mutex);
-}
-
 static DEFINE_PER_CPU(char[MSG_SIZE], fmt_buffer);
 
 /*
@@ -163,6 +53,10 @@ void sched_trace_log_message(const char* fmt, ...)
         size_t          len;
         char*           buf;
 
+        if (!atomic_read(&reader_cnt))
+                /* early exit if nobody is listening */
+                return;
+
         va_start(args, fmt);
         local_irq_save(flags);
 
@@ -171,32 +65,33 @@ void sched_trace_log_message(const char* fmt, ...)
         len = vscnprintf(buf, MSG_SIZE, fmt, args);
 
         raw_spin_lock(&log_buffer_lock);
-        /* Don't copy the trailing null byte, we don't want null bytes
-         * in a text file.
+        /* Don't copy the trailing null byte, we don't want null bytes in a
+         * text file.
          */
-        rb_put(&log_buffer.buf, buf, len);
+        kfifo_in(&debug_buffer, buf, len);
         raw_spin_unlock(&log_buffer_lock);
 
         local_irq_restore(flags);
         va_end(args);
 }
 
+
 /*
  * log_read - Read the trace buffer
  *
  * This function is called as a file operation from userspace.
  * Readers can sleep. Access is serialized through reader_mutex
  */
-static ssize_t log_read(struct file *filp, char __user *to, size_t len,
-                      loff_t *f_pos)
+static ssize_t log_read(struct file *filp,
+                        char __user *to, size_t len,
+                        loff_t *f_pos)
 {
         /* we ignore f_pos, this is strictly sequential */
 
         ssize_t error = -EINVAL;
-        char*   mem;
-        trace_buffer_t *tbuf = filp->private_data;
+        char* mem;
 
-        if (down_interruptible(&tbuf->reader_mutex)) {
+        if (mutex_lock_interruptible(&reader_mutex)) {
                 error = -ERESTARTSYS;
                 goto out;
         }
@@ -210,14 +105,14 @@ static ssize_t log_read(struct file *filp, char __user *to, size_t len,
                 goto out_unlock;
         }
 
-        error = rb_get(&tbuf->buf, mem, len);
+        error = kfifo_out(&debug_buffer, mem, len);
         while (!error) {
                 set_current_state(TASK_INTERRUPTIBLE);
                 schedule_timeout(110);
                 if (signal_pending(current))
                         error = -ERESTARTSYS;
                 else
-                        error = rb_get(&tbuf->buf, mem, len);
+                        error = kfifo_out(&debug_buffer, mem, len);
         }
 
         if (error > 0 && copy_to_user(to, mem, error))
@@ -225,7 +120,7 @@ static ssize_t log_read(struct file *filp, char __user *to, size_t len,
 
         kfree(mem);
  out_unlock:
-        up(&tbuf->reader_mutex);
+        mutex_unlock(&reader_mutex);
  out:
         return error;
 }
@@ -243,36 +138,23 @@ extern int trace_recurse;
 static int log_open(struct inode *in, struct file *filp)
 {
         int error = -EINVAL;
-        trace_buffer_t* tbuf;
-
-        tbuf = &log_buffer;
 
-        if (down_interruptible(&tbuf->reader_mutex)) {
+        if (mutex_lock_interruptible(&reader_mutex)) {
                 error = -ERESTARTSYS;
                 goto out;
         }
 
-        /* first open must allocate buffers */
-        if (atomic_inc_return(&tbuf->reader_cnt) == 1) {
-                if ((error = rb_alloc_buf(&tbuf->buf, LITMUS_TRACE_BUF_SIZE)))
-                {
-                        atomic_dec(&tbuf->reader_cnt);
-                        goto out_unlock;
-                }
-        }
-
+        atomic_inc(&reader_cnt);
         error = 0;
-        filp->private_data = tbuf;
 
         printk(KERN_DEBUG
                "sched_trace kfifo with buffer starting at: 0x%p\n",
-               (tbuf->buf.kfifo).buf);
+               debug_buffer.buf);
 
         /* override printk() */
         trace_override++;
 
- out_unlock:
-        up(&tbuf->reader_mutex);
+        mutex_unlock(&reader_mutex);
  out:
         return error;
 }
@@ -280,26 +162,20 @@ static int log_open(struct inode *in, struct file *filp)
 static int log_release(struct inode *in, struct file *filp)
 {
         int error = -EINVAL;
-        trace_buffer_t* tbuf = filp->private_data;
-
-        BUG_ON(!filp->private_data);
 
-        if (down_interruptible(&tbuf->reader_mutex)) {
+        if (mutex_lock_interruptible(&reader_mutex)) {
                 error = -ERESTARTSYS;
                 goto out;
         }
 
-        /* last release must deallocate buffers */
-        if (atomic_dec_return(&tbuf->reader_cnt) == 0) {
-                error = rb_free_buf(&tbuf->buf);
-        }
+        atomic_dec(&reader_cnt);
 
         /* release printk() overriding */
         trace_override--;
 
         printk(KERN_DEBUG "sched_trace kfifo released\n");
 
-        up(&tbuf->reader_mutex);
+        mutex_unlock(&reader_mutex);
  out:
         return error;
 }
@@ -333,7 +209,7 @@ void dump_trace_buffer(int max)
         /* potential, but very unlikely, race... */
         trace_recurse = 1;
         while ((max == 0 || count++ < max) &&
-               (len = rb_get(&log_buffer.buf, line, sizeof(line) - 1)) > 0) {
+               (len = kfifo_out(&debug_buffer, line, sizeof(line - 1))) > 0) {
                 line[len] = '\0';
                 printk("%s", line);
         }
@@ -355,7 +231,6 @@ static struct sysrq_key_op sysrq_dump_trace_buffer_op = {
 static int __init init_sched_trace(void)
 {
         printk("Initializing TRACE() device\n");
-        init_log_buffer();
 
 #ifdef CONFIG_MAGIC_SYSRQ
         /* offer some debugging help */
@@ -365,7 +240,6 @@ static int __init init_sched_trace(void)
                 printk("Could not register dump-trace-buffer(Y) magic sysrq.\n");
 #endif
 
-
         return misc_register(&litmus_log_dev);
 }
 
diff --git a/litmus/srp.c b/litmus/srp.c
index 71639b991630..cb577598ce3e 100644
--- a/litmus/srp.c
+++ b/litmus/srp.c
@@ -3,12 +3,12 @@
 /* ************************************************************************** */
 
 #include <asm/atomic.h>
+#include <linux/sched.h>
 #include <linux/wait.h>
+
 #include <litmus/litmus.h>
 #include <litmus/sched_plugin.h>
-
 #include <litmus/fdso.h>
-
 #include <litmus/trace.h>
 
 
diff --git a/litmus/trace.c b/litmus/trace.c
index 440376998dc9..b3a6b47aad6e 100644
--- a/litmus/trace.c
+++ b/litmus/trace.c
@@ -1,3 +1,4 @@
+#include <linux/sched.h>
 #include <linux/module.h>
 
 #include <litmus/ftdev.h>