[ported from 2008.3] Add tracing support and hook up Litmus KConfig for x86

- fix requesting more than 2^11 pages (MAX_ORDER)
  to system allocator

Still to be merged:
- feather-trace generic implementation

1 files changed, 510 insertions, 0 deletions

diff --git a/litmus/sched_trace.c b/litmus/sched_trace.c
new file mode 100644
index 000000000000..5befc1e21b1f
--- /dev/null
+++ b/litmus/sched_trace.c
@@ -0,0 +1,510 @@
+/*
+ * sched_trace.c -- record scheduling events to a byte stream.
+ *
+ * TODO: Move ring buffer to a lockfree implementation.
+ */
+
+#include <linux/spinlock.h>
+#include <linux/fs.h>
+#include <linux/cdev.h>
+#include <linux/semaphore.h>
+#include <asm/uaccess.h>
+#include <linux/module.h>
+#include <linux/sysrq.h>
+
+#include <litmus/sched_trace.h>
+#include <litmus/litmus.h>
+
+typedef struct {
+        /*      guard read and write pointers                   */
+        spinlock_t      lock;
+        /*      guard against concurrent freeing of buffer      */
+        rwlock_t        del_lock;
+
+        /*      memory allocated for ring buffer                */
+        unsigned long   order;
+        char*           buf;
+        char*           end;
+
+        /*      Read/write pointer. May not cross.
+         *      They point to the position of next write and
+         *      last read.
+         */
+        char*           writep;
+        char*           readp;
+
+} ring_buffer_t;
+
+#define EMPTY_RING_BUFFER {     \
+        .lock     = SPIN_LOCK_UNLOCKED,         \
+        .del_lock = RW_LOCK_UNLOCKED,           \
+        .buf      = NULL,                       \
+        .end      = NULL,                       \
+        .writep   = NULL,                       \
+        .readp    = NULL                        \
+}
+
+void rb_init(ring_buffer_t* buf)
+{
+        *buf = (ring_buffer_t) EMPTY_RING_BUFFER;
+}
+
+int rb_alloc_buf(ring_buffer_t* buf, unsigned long order)
+{
+        unsigned long flags;
+        int error = 0;
+        char *mem;
+
+        /* do memory allocation while not atomic */
+        mem = (char *) __get_free_pages(GFP_KERNEL, order);
+        if (!mem)
+                return -ENOMEM;
+        write_lock_irqsave(&buf->del_lock, flags);
+        BUG_ON(buf->buf);
+        buf->buf = mem;
+        buf->end = buf->buf + PAGE_SIZE * (1 << order) - 1;
+        memset(buf->buf, 0xff, buf->end - buf->buf);
+        buf->order = order;
+        buf->writep = buf->buf + 1;
+        buf->readp  = buf->buf;
+        write_unlock_irqrestore(&buf->del_lock, flags);
+        return error;
+}
+
+int rb_free_buf(ring_buffer_t* buf)
+{
+        unsigned long flags;
+        int error = 0;
+        write_lock_irqsave(&buf->del_lock, flags);
+        BUG_ON(!buf->buf);
+        free_pages((unsigned long) buf->buf, buf->order);
+        buf->buf    = NULL;
+        buf->end    = NULL;
+        buf->writep = NULL;
+        buf->readp  = NULL;
+        write_unlock_irqrestore(&buf->del_lock, flags);
+        return error;
+}
+
+/* Assumption: concurrent writes are serialized externally
+ *
+ * Will only succeed if there is enough space for all len bytes.
+ */
+int rb_put(ring_buffer_t* buf, char* mem, size_t len)
+{
+        unsigned long flags;
+        char* r , *w;
+        int error = 0;
+        read_lock_irqsave(&buf->del_lock, flags);
+        if (!buf->buf) {
+                error = -ENODEV;
+                goto out;
+        }
+        spin_lock(&buf->lock);
+        r = buf->readp;
+        w = buf->writep;
+        spin_unlock(&buf->lock);
+        if (r < w && buf->end - w >= len - 1) {
+                /* easy case: there is enough space in the buffer
+                 * to write it in one continous chunk*/
+                memcpy(w, mem, len);
+                w += len;
+                if (w > buf->end)
+                        /* special case: fit exactly into buffer
+                         * w is now buf->end + 1
+                         */
+                        w = buf->buf;
+        } else if (w < r && r - w >= len) { /* >= len because  may not cross */
+                /* we are constrained by the read pointer but we there
+                 * is enough space
+                 */
+                memcpy(w, mem, len);
+                w += len;
+        } else if (r <= w && buf->end - w < len - 1) {
+                /* the wrap around case: there may or may not be space */
+                if ((buf->end - w) + (r - buf->buf) >= len - 1) {
+                        /* copy chunk that fits at the end */
+                        memcpy(w, mem, buf->end - w + 1);
+                        mem += buf->end - w + 1;
+                        len -= (buf->end - w + 1);
+                        w = buf->buf;
+                        /* copy the rest */
+                        memcpy(w, mem, len);
+                        w += len;
+                }
+                else
+                        error = -ENOMEM;
+        } else {
+                error = -ENOMEM;
+        }
+        if (!error) {
+                spin_lock(&buf->lock);
+                buf->writep = w;
+                spin_unlock(&buf->lock);
+        }
+ out:
+        read_unlock_irqrestore(&buf->del_lock, flags);
+        return error;
+}
+
+/* Assumption: concurrent reads are serialized externally */
+int rb_get(ring_buffer_t* buf, char* mem, size_t len)
+{
+        unsigned long flags;
+        char* r , *w;
+        int error = 0;
+        read_lock_irqsave(&buf->del_lock, flags);
+        if (!buf->buf) {
+                error = -ENODEV;
+                goto out;
+        }
+        spin_lock(&buf->lock);
+        r = buf->readp;
+        w = buf->writep;
+        spin_unlock(&buf->lock);
+
+        if (w <= r && buf->end - r >= len) {
+                /* easy case: there is enough data in the buffer
+                 * to get it in one  chunk*/
+                memcpy(mem, r + 1, len);
+                r += len;
+                error = len;
+
+        } else if (r + 1 < w && w - r - 1 >= len) {
+                /* we are constrained by the write pointer but
+                 * there is enough data
+                 */
+                memcpy(mem, r + 1, len);
+                r += len;
+                error = len;
+
+        } else if (r + 1 < w && w - r - 1 < len) {
+                /* we are constrained by the write pointer and there
+                 * there is not enough data
+                 */
+                memcpy(mem, r + 1, w - r - 1);
+                error = w - r - 1;
+                r    += w - r - 1;
+
+        } else if (w <= r && buf->end - r < len) {
+                /* the wrap around case: there may or may not be enough data
+                 * first let's get what is available
+                 */
+                memcpy(mem, r + 1, buf->end - r);
+                error += (buf->end - r);
+                mem   += (buf->end - r);
+                len   -= (buf->end - r);
+                r     += (buf->end - r);
+
+                if (w > buf->buf) {
+                        /* there is more to get */
+                        r = buf->buf - 1;
+                        if (w - r >= len) {
+                                /* plenty */
+                                memcpy(mem, r + 1, len);
+                                error += len;
+                                r     += len;
+                        } else {
+                                memcpy(mem, r + 1, w - r - 1);
+                                error += w - r - 1;
+                                r     += w - r - 1;
+                        }
+                }
+        } /* nothing available */
+
+        if (error > 0) {
+                spin_lock(&buf->lock);
+                buf->readp = r;
+                spin_unlock(&buf->lock);
+        }
+ out:
+        read_unlock_irqrestore(&buf->del_lock, flags);
+        return error;
+}
+
+
+
+/******************************************************************************/
+/*                        DEVICE FILE DRIVER                                  */
+/******************************************************************************/
+
+
+
+/* Allocate a buffer of about 1 MB per CPU.
+ *
+ */
+#define BUFFER_ORDER 8
+
+typedef struct {
+        ring_buffer_t           buf;
+        atomic_t                reader_cnt;
+        struct semaphore        reader_mutex;
+} trace_buffer_t;
+
+
+/* This does not initialize the semaphore!! */
+
+#define EMPTY_TRACE_BUFFER \
+        { .buf = EMPTY_RING_BUFFER, .reader_cnt = ATOMIC_INIT(0)}
+
+static spinlock_t               log_buffer_lock = SPIN_LOCK_UNLOCKED;
+static trace_buffer_t           log_buffer = EMPTY_TRACE_BUFFER;
+
+static void init_log_buffer(void)
+{
+        /* only initialize the mutex, the rest was initialized as part
+         * of the static initialization macro
+         */
+        init_MUTEX(&log_buffer.reader_mutex);
+}
+
+static ssize_t log_read(struct file *filp, char __user *to, size_t len,
+                      loff_t *f_pos)
+{
+        /*      we ignore f_pos, this is strictly sequential */
+
+        ssize_t error = -EINVAL;
+        char*   mem;
+        trace_buffer_t *buf = filp->private_data;
+
+        if (down_interruptible(&buf->reader_mutex)) {
+                error = -ERESTARTSYS;
+                goto out;
+        }
+
+        if (len > 64 * 1024)
+                len = 64 * 1024;
+        mem = kmalloc(len, GFP_KERNEL);
+        if (!mem) {
+                error = -ENOMEM;
+                goto out_unlock;
+        }
+
+        error = rb_get(&buf->buf, mem, len);
+        while (!error) {
+                set_current_state(TASK_INTERRUPTIBLE);
+                schedule_timeout(110);
+                if (signal_pending(current))
+                        error = -ERESTARTSYS;
+                else
+                        error = rb_get(&buf->buf, mem, len);
+        }
+
+        if (error > 0 && copy_to_user(to, mem, error))
+                error = -EFAULT;
+
+        kfree(mem);
+ out_unlock:
+        up(&buf->reader_mutex);
+ out:
+        return error;
+}
+
+/* defined in kernel/printk.c */
+extern int trace_override;
+extern int trace_recurse;
+
+/* log_open - open the global log message ring buffer.
+ */
+static int log_open(struct inode *in, struct file *filp)
+{
+        int error               = -EINVAL;
+        trace_buffer_t* buf;
+
+        buf = &log_buffer;
+
+        if (down_interruptible(&buf->reader_mutex)) {
+                error = -ERESTARTSYS;
+                goto out;
+        }
+
+        /*      first open must allocate buffers        */
+        if (atomic_inc_return(&buf->reader_cnt) == 1) {
+                if ((error = rb_alloc_buf(&buf->buf, BUFFER_ORDER)))
+                {
+                        atomic_dec(&buf->reader_cnt);
+                        goto out_unlock;
+                }
+        }
+
+        error = 0;
+        filp->private_data = buf;
+        printk(KERN_DEBUG "sched_trace buf: from 0x%p to 0x%p  length: %x\n",
+               buf->buf.buf, buf->buf.end,
+               (unsigned int) (buf->buf.end - buf->buf.buf));
+
+        /* override printk() */
+        trace_override++;
+
+ out_unlock:
+        up(&buf->reader_mutex);
+ out:
+        return error;
+}
+
+static int log_release(struct inode *in, struct file *filp)
+{
+        int error               = -EINVAL;
+        trace_buffer_t* buf     = filp->private_data;
+
+        BUG_ON(!filp->private_data);
+
+        if (down_interruptible(&buf->reader_mutex)) {
+                error = -ERESTARTSYS;
+                goto out;
+        }
+
+        /*      last release must deallocate buffers    */
+        if (atomic_dec_return(&buf->reader_cnt) == 0) {
+                error = rb_free_buf(&buf->buf);
+        }
+
+        /* release printk() overriding */
+        trace_override--;
+
+        up(&buf->reader_mutex);
+ out:
+        return error;
+}
+
+/******************************************************************************/
+/*                          Device Registration                               */
+/******************************************************************************/
+
+/* the major numbes are from the unassigned/local use block
+ *
+ * This should be converted to dynamic allocation at some point...
+ */
+#define LOG_MAJOR       251
+
+/* log_fops  - The file operations for accessing the global LITMUS log message
+ *             buffer.
+ *
+ * Except for opening the device file it uses the same operations as trace_fops.
+ */
+struct file_operations log_fops = {
+        .owner   = THIS_MODULE,
+        .open    = log_open,
+        .release = log_release,
+        .read    = log_read,
+};
+
+static int __init register_buffer_dev(const char* name,
+                                      struct file_operations* fops,
+                                      int major, int count)
+{
+        dev_t  trace_dev;
+        struct cdev *cdev;
+        int error = 0;
+
+        trace_dev = MKDEV(major, 0);
+        error     = register_chrdev_region(trace_dev, count, name);
+        if (error)
+        {
+                printk(KERN_WARNING "sched trace: "
+                       "Could not register major/minor number %d\n", major);
+                return error;
+        }
+        cdev = cdev_alloc();
+        if (!cdev) {
+                printk(KERN_WARNING "sched trace: "
+                        "Could not get a cdev for %s.\n", name);
+                return -ENOMEM;
+        }
+        cdev->owner = THIS_MODULE;
+        cdev->ops   = fops;
+        error = cdev_add(cdev, trace_dev, count);
+        if (error) {
+                printk(KERN_WARNING "sched trace: "
+                        "add_cdev failed for %s.\n", name);
+                return -ENOMEM;
+        }
+        return error;
+
+}
+
+#ifdef CONFIG_MAGIC_SYSRQ
+
+static void sysrq_dump_trace_buffer(int key, struct tty_struct *tty)
+{
+        dump_trace_buffer(100);
+}
+
+static struct sysrq_key_op sysrq_dump_trace_buffer_op = {
+        .handler        = sysrq_dump_trace_buffer,
+        .help_msg       = "dump-trace-buffer(Y)",
+        .action_msg     = "writing content of TRACE() buffer",
+};
+
+#endif
+
+static int __init init_sched_trace(void)
+{
+        printk("Initializing TRACE() device\n");
+        init_log_buffer();
+
+#ifdef CONFIG_MAGIC_SYSRQ
+        /* offer some debugging help */
+        if (!register_sysrq_key('y', &sysrq_dump_trace_buffer_op))
+                printk("Registered dump-trace-buffer(Y) magic sysrq.\n");
+        else
+                printk("Could not register dump-trace-buffer(Y) magic sysrq.\n");
+#endif
+
+
+        return register_buffer_dev("litmus_log", &log_fops,
+                                   LOG_MAJOR, 1);
+}
+
+module_init(init_sched_trace);
+
+#define MSG_SIZE 255
+static DEFINE_PER_CPU(char[MSG_SIZE], fmt_buffer);
+
+/* sched_trace_log_message - This is the only function that accesses the the
+ *                           log buffer inside the kernel for writing.
+ *                           Concurrent access to it is serialized via the
+ *                           log_buffer_lock.
+ *
+ *                           The maximum length of a formatted message is 255.
+ */
+void sched_trace_log_message(const char* fmt, ...)
+{
+        unsigned long   flags;
+        va_list         args;
+        size_t          len;
+        char*           buf;
+
+        va_start(args, fmt);
+        local_irq_save(flags);
+
+        /* format message */
+        buf = __get_cpu_var(fmt_buffer);
+        len = vscnprintf(buf, MSG_SIZE, fmt, args);
+
+        spin_lock(&log_buffer_lock);
+        /* Don't copy the trailing null byte, we don't want null bytes
+         * in a text file.
+         */
+        rb_put(&log_buffer.buf, buf, len);
+        spin_unlock(&log_buffer_lock);
+
+        local_irq_restore(flags);
+        va_end(args);
+}
+
+void dump_trace_buffer(int max)
+{
+        char line[80];
+        int len;
+        int count = 0;
+
+        /* potentially, but very unlikely race... */
+        trace_recurse = 1;
+        while ((max == 0 || count++ < max) &&
+               (len = rb_get(&log_buffer.buf, line, sizeof(line) - 1)) > 0) {
+                line[len] = '\0';
+                printk("%s", line);
+        }
+        trace_recurse = 0;
+}