2 files changed, 195 insertions, 58 deletions

diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 1b8eca7650d4..d7ad030a4c49 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -15,6 +15,7 @@
 #include <linux/kallsyms.h>
 #include <linux/seq_file.h>
 #include <linux/debugfs.h>
+#include <linux/pagemap.h>
 #include <linux/hardirq.h>
 #include <linux/linkage.h>
 #include <linux/uaccess.h>
@@ -49,7 +50,7 @@ static struct trace_array	max_tr;
 static DEFINE_PER_CPU(struct trace_array_cpu, max_data);
 
 static int                      tracer_enabled;
-static unsigned long            trace_nr_entries = 4096UL;
+static unsigned long            trace_nr_entries = 16384UL;
 
 static struct tracer            *trace_types __read_mostly;
 static struct tracer            *current_trace __read_mostly;
@@ -57,6 +58,8 @@ static int			max_tracer_type_len;
 
 static DEFINE_MUTEX(trace_types_lock);
 
+#define ENTRIES_PER_PAGE (PAGE_SIZE / sizeof(struct trace_entry))
+
 static int __init set_nr_entries(char *str)
 {
         if (!str)
@@ -103,6 +106,7 @@ static const char *trace_options[] = {
 
 static unsigned trace_flags;
 
+static DEFINE_SPINLOCK(ftrace_max_lock);
 
 /*
  * Copy the new maximum trace into the separate maximum-trace
@@ -136,17 +140,23 @@ update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
 {
         struct trace_array_cpu *data;
         void *save_trace;
+        struct list_head save_pages;
         int i;
 
+        WARN_ON_ONCE(!irqs_disabled());
+        spin_lock(&ftrace_max_lock);
         /* clear out all the previous traces */
         for_each_possible_cpu(i) {
                 data = tr->data[i];
                 save_trace = max_tr.data[i]->trace;
+                save_pages = max_tr.data[i]->trace_pages;
                 memcpy(max_tr.data[i], data, sizeof(*data));
                 data->trace = save_trace;
+                data->trace_pages = save_pages;
         }
 
         __update_max_tr(tr, tsk, cpu);
+        spin_unlock(&ftrace_max_lock);
 }
 
 /**
@@ -160,16 +170,22 @@ update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
 {
         struct trace_array_cpu *data = tr->data[cpu];
         void *save_trace;
+        struct list_head save_pages;
         int i;
 
+        WARN_ON_ONCE(!irqs_disabled());
+        spin_lock(&ftrace_max_lock);
         for_each_possible_cpu(i)
                 tracing_reset(max_tr.data[i]);
 
         save_trace = max_tr.data[cpu]->trace;
+        save_pages = max_tr.data[cpu]->trace_pages;
         memcpy(max_tr.data[cpu], data, sizeof(*data));
         data->trace = save_trace;
+        data->trace_pages = save_pages;
 
         __update_max_tr(tr, tsk, cpu);
+        spin_unlock(&ftrace_max_lock);
 }
 
 int register_tracer(struct tracer *type)
@@ -236,7 +252,8 @@ void unregister_tracer(struct tracer *type)
 void notrace tracing_reset(struct trace_array_cpu *data)
 {
         data->trace_idx = 0;
-        atomic_set(&data->underrun, 0);
+        data->trace_current = data->trace;
+        data->trace_current_idx = 0;
 }
 
 #ifdef CONFIG_FTRACE
@@ -367,21 +384,27 @@ tracing_get_trace_entry(struct trace_array *tr,
 {
         unsigned long idx, idx_next;
         struct trace_entry *entry;
+        struct page *page;
+        struct list_head *next;
 
-        idx = data->trace_idx;
+        data->trace_idx++;
+        idx = data->trace_current_idx;
         idx_next = idx + 1;
 
-        if (unlikely(idx_next >= tr->entries)) {
-                atomic_inc(&data->underrun);
+        entry = data->trace_current + idx * TRACE_ENTRY_SIZE;
+
+        if (unlikely(idx_next >= ENTRIES_PER_PAGE)) {
+                page = virt_to_page(data->trace_current);
+                if (unlikely(&page->lru == data->trace_pages.prev))
+                        next = data->trace_pages.next;
+                else
+                        next = page->lru.next;
+                page = list_entry(next, struct page, lru);
+                data->trace_current = page_address(page);
                 idx_next = 0;
         }
 
-        data->trace_idx = idx_next;
-
-        if (unlikely(idx_next != 0 && atomic_read(&data->underrun)))
-                atomic_inc(&data->underrun);
-
-        entry = data->trace + idx * TRACE_ENTRY_SIZE;
+        data->trace_current_idx = idx_next;
 
         return entry;
 }
@@ -442,21 +465,38 @@ enum trace_file_type {
 };
 
 static struct trace_entry *
-trace_entry_idx(struct trace_array *tr, unsigned long idx, int cpu)
+trace_entry_idx(struct trace_array *tr, struct trace_array_cpu *data,
+                struct trace_iterator *iter, int cpu)
 {
-        struct trace_entry *array = tr->data[cpu]->trace;
-        unsigned long underrun;
+        struct page *page;
+        struct trace_entry *array;
 
-        if (idx >= tr->entries)
+        if (iter->next_idx[cpu] >= tr->entries ||
+            iter->next_idx[cpu] >= data->trace_idx)
                 return NULL;
 
-        underrun = atomic_read(&tr->data[cpu]->underrun);
-        if (underrun)
-                idx = ((underrun - 1) + idx) % tr->entries;
-        else if (idx >= tr->data[cpu]->trace_idx)
-                return NULL;
+        if (!iter->next_page[cpu]) {
+                /*
+                 * Initialize. If the count of elements in
+                 * this buffer is greater than the max entries
+                 * we had an underrun. Which means we looped around.
+                 * We can simply use the current pointer as our
+                 * starting point.
+                 */
+                if (data->trace_idx >= tr->entries) {
+                        page = virt_to_page(data->trace_current);
+                        iter->next_page[cpu] = &page->lru;
+                        iter->next_page_idx[cpu] = data->trace_current_idx;
+                } else {
+                        iter->next_page[cpu] = data->trace_pages.next;
+                        iter->next_page_idx[cpu] = 0;
+                }
+        }
 
-        return &array[idx];
+        page = list_entry(iter->next_page[cpu], struct page, lru);
+        array = page_address(page);
+
+        return &array[iter->next_page_idx[cpu]];
 }
 
 static struct notrace trace_entry *
@@ -470,7 +510,7 @@ find_next_entry(struct trace_iterator *iter, int *ent_cpu)
         for_each_possible_cpu(cpu) {
                 if (!tr->data[cpu]->trace)
                         continue;
-                ent = trace_entry_idx(tr, iter->next_idx[cpu], cpu);
+                ent = trace_entry_idx(tr, tr->data[cpu], iter, cpu);
                 if (ent &&
                     (!next || (long)(next->idx - ent->idx) > 0)) {
                         next = ent;
@@ -492,8 +532,19 @@ static void *find_next_entry_inc(struct trace_iterator *iter)
         next = find_next_entry(iter, &next_cpu);
 
         if (next) {
-                iter->next_idx[next_cpu]++;
                 iter->idx++;
+                iter->next_idx[next_cpu]++;
+                iter->next_page_idx[next_cpu]++;
+                if (iter->next_page_idx[next_cpu] >= ENTRIES_PER_PAGE) {
+                        struct trace_array_cpu *data = iter->tr->data[next_cpu];
+
+                        iter->next_page_idx[next_cpu] = 0;
+                        iter->next_page[next_cpu] =
+                                iter->next_page[next_cpu]->next;
+                        if (iter->next_page[next_cpu] == &data->trace_pages)
+                                iter->next_page[next_cpu] =
+                                        data->trace_pages.next;
+                }
         }
         iter->ent = next;
         iter->cpu = next_cpu;
@@ -554,14 +605,16 @@ static void *s_start(struct seq_file *m, loff_t *pos)
                 iter->cpu = 0;
                 iter->idx = -1;
 
-                for (i = 0; i < NR_CPUS; i++)
+                for_each_possible_cpu(i) {
                         iter->next_idx[i] = 0;
+                        iter->next_page[i] = NULL;
+                }
 
                 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
                         ;
 
         } else {
-                l = *pos;
+                l = *pos - 1;
                 p = s_next(m, p, &l);
         }
 
@@ -654,9 +707,8 @@ print_trace_header(struct seq_file *m, struct trace_iterator *iter)
         struct trace_array *tr = iter->tr;
         struct trace_array_cpu *data = tr->data[tr->cpu];
         struct tracer *type = current_trace;
-        unsigned long underruns = 0;
-        unsigned long underrun;
-        unsigned long entries   = 0;
+        unsigned long total   = 0;
+        unsigned long entries = 0;
         int cpu;
         const char *name = "preemption";
 
@@ -665,11 +717,10 @@ print_trace_header(struct seq_file *m, struct trace_iterator *iter)
 
         for_each_possible_cpu(cpu) {
                 if (tr->data[cpu]->trace) {
-                        underrun = atomic_read(&tr->data[cpu]->underrun);
-                        if (underrun) {
-                                underruns += underrun;
+                        total += tr->data[cpu]->trace_idx;
+                        if (tr->data[cpu]->trace_idx > tr->entries)
                                 entries += tr->entries;
-                        } else
+                        else
                                 entries += tr->data[cpu]->trace_idx;
                 }
         }
@@ -682,7 +733,7 @@ print_trace_header(struct seq_file *m, struct trace_iterator *iter)
                    " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
                    data->saved_latency,
                    entries,
-                   (entries + underruns),
+                   total,
                    tr->cpu,
 #if defined(CONFIG_PREEMPT_NONE)
                    "server",
@@ -882,8 +933,7 @@ static int trace_empty(struct trace_iterator *iter)
                 data = iter->tr->data[cpu];
 
                 if (data->trace &&
-                    (data->trace_idx ||
-                     atomic_read(&data->underrun)))
+                    data->trace_idx)
                         return 0;
         }
         return 1;
@@ -1464,42 +1514,109 @@ static struct tracer no_tracer __read_mostly =
         .name = "none",
 };
 
-static inline notrace int page_order(const unsigned long size)
+static int trace_alloc_page(void)
 {
-        const unsigned long nr_pages = DIV_ROUND_UP(size, PAGE_SIZE);
-        return ilog2(roundup_pow_of_two(nr_pages));
+        struct trace_array_cpu *data;
+        void *array;
+        struct page *page, *tmp;
+        LIST_HEAD(pages);
+        int i;
+
+        /* first allocate a page for each CPU */
+        for_each_possible_cpu(i) {
+                array = (void *)__get_free_page(GFP_KERNEL);
+                if (array == NULL) {
+                        printk(KERN_ERR "tracer: failed to allocate page"
+                               "for trace buffer!\n");
+                        goto free_pages;
+                }
+
+                page = virt_to_page(array);
+                list_add(&page->lru, &pages);
+
+/* Only allocate if we are actually using the max trace */
+#ifdef CONFIG_TRACER_MAX_TRACE
+                array = (void *)__get_free_page(GFP_KERNEL);
+                if (array == NULL) {
+                        printk(KERN_ERR "tracer: failed to allocate page"
+                               "for trace buffer!\n");
+                        goto free_pages;
+                }
+                page = virt_to_page(array);
+                list_add(&page->lru, &pages);
+#endif
+        }
+
+        /* Now that we successfully allocate a page per CPU, add them */
+        for_each_possible_cpu(i) {
+                data = global_trace.data[i];
+                page = list_entry(pages.next, struct page, lru);
+                list_del(&page->lru);
+                list_add_tail(&page->lru, &data->trace_pages);
+                ClearPageLRU(page);
+
+#ifdef CONFIG_TRACER_MAX_TRACE
+                data = max_tr.data[i];
+                page = list_entry(pages.next, struct page, lru);
+                list_del(&page->lru);
+                list_add_tail(&page->lru, &data->trace_pages);
+                SetPageLRU(page);
+#endif
+        }
+        global_trace.entries += ENTRIES_PER_PAGE;
+
+        return 0;
+
+ free_pages:
+        list_for_each_entry_safe(page, tmp, &pages, lru) {
+                list_del(&page->lru);
+                __free_page(page);
+        }
+        return -ENOMEM;
 }
 
 __init static int tracer_alloc_buffers(void)
 {
-        const int order = page_order(trace_nr_entries * TRACE_ENTRY_SIZE);
-        const unsigned long size = (1UL << order) << PAGE_SHIFT;
-        struct trace_entry *array;
+        struct trace_array_cpu *data;
+        void *array;
+        struct page *page;
+        int pages = 0;
         int i;
 
+        /* Allocate the first page for all buffers */
         for_each_possible_cpu(i) {
-                global_trace.data[i] = &per_cpu(global_trace_cpu, i);
+                data = global_trace.data[i] = &per_cpu(global_trace_cpu, i);
                 max_tr.data[i] = &per_cpu(max_data, i);
 
-                array = (struct trace_entry *)
-                          __get_free_pages(GFP_KERNEL, order);
+                array = (void *)__get_free_page(GFP_KERNEL);
                 if (array == NULL) {
-                        printk(KERN_ERR "tracer: failed to allocate"
-                               " %ld bytes for trace buffer!\n", size);
+                        printk(KERN_ERR "tracer: failed to allocate page"
+                               "for trace buffer!\n");
                         goto free_buffers;
                 }
-                global_trace.data[i]->trace = array;
+                data->trace = array;
+
+                /* set the array to the list */
+                INIT_LIST_HEAD(&data->trace_pages);
+                page = virt_to_page(array);
+                list_add(&page->lru, &data->trace_pages);
+                /* use the LRU flag to differentiate the two buffers */
+                ClearPageLRU(page);
 
 /* Only allocate if we are actually using the max trace */
 #ifdef CONFIG_TRACER_MAX_TRACE
-                array = (struct trace_entry *)
-                          __get_free_pages(GFP_KERNEL, order);
+                array = (void *)__get_free_page(GFP_KERNEL);
                 if (array == NULL) {
-                        printk(KERN_ERR "wakeup tracer: failed to allocate"
-                               " %ld bytes for trace buffer!\n", size);
+                        printk(KERN_ERR "tracer: failed to allocate page"
+                               "for trace buffer!\n");
                         goto free_buffers;
                 }
                 max_tr.data[i]->trace = array;
+
+                INIT_LIST_HEAD(&max_tr.data[i]->trace_pages);
+                page = virt_to_page(array);
+                list_add(&page->lru, &max_tr.data[i]->trace_pages);
+                SetPageLRU(page);
 #endif
         }
 
@@ -1507,11 +1624,18 @@ __init static int tracer_alloc_buffers(void)
          * Since we allocate by orders of pages, we may be able to
          * round up a bit.
          */
-        global_trace.entries = size / TRACE_ENTRY_SIZE;
+        global_trace.entries = ENTRIES_PER_PAGE;
         max_tr.entries = global_trace.entries;
+        pages++;
+
+        while (global_trace.entries < trace_nr_entries) {
+                if (trace_alloc_page())
+                        break;
+                pages++;
+        }
 
-        pr_info("tracer: %ld bytes allocated for %ld",
-                size, trace_nr_entries);
+        pr_info("tracer: %d pages allocated for %ld",
+                pages, trace_nr_entries);
         pr_info(" entries of %ld bytes\n", (long)TRACE_ENTRY_SIZE);
         pr_info("   actual entries %ld\n", global_trace.entries);
 
@@ -1526,17 +1650,26 @@ __init static int tracer_alloc_buffers(void)
 
  free_buffers:
         for (i-- ; i >= 0; i--) {
+                struct page *page, *tmp;
                 struct trace_array_cpu *data = global_trace.data[i];
 
                 if (data && data->trace) {
-                        free_pages((unsigned long)data->trace, order);
+                        list_for_each_entry_safe(page, tmp,
+                                                 &data->trace_pages, lru) {
+                                list_del(&page->lru);
+                                __free_page(page);
+                        }
                         data->trace = NULL;
                 }
 
 #ifdef CONFIG_TRACER_MAX_TRACE
                 data = max_tr.data[i];
                 if (data && data->trace) {
-                        free_pages((unsigned long)data->trace, order);
+                        list_for_each_entry_safe(page, tmp,
+                                                 &data->trace_pages, lru) {
+                                list_del(&page->lru);
+                                __free_page(page);
+                        }
                         data->trace = NULL;
                 }
 #endif
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index 3173a93561d4..83e257e38084 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -54,9 +54,11 @@ struct trace_entry {
  */
 struct trace_array_cpu {
         void                    *trace;
+        void                    *trace_current;
+        unsigned                trace_current_idx;
+        struct list_head        trace_pages;
         unsigned long           trace_idx;
         atomic_t                disabled;
-        atomic_t                underrun;
         unsigned long           saved_latency;
         unsigned long           critical_start;
         unsigned long           critical_end;
@@ -112,8 +114,10 @@ struct trace_iterator {
         unsigned long           iter_flags;
         loff_t                  pos;
         unsigned long           next_idx[NR_CPUS];
+        struct list_head        *next_page[NR_CPUS];
+        unsigned                next_page_idx[NR_CPUS];
+        long                    idx;
         int                     cpu;
-        int                     idx;
 };
 
 void notrace tracing_reset(struct trace_array_cpu *data);