ftrace: change buffers to producer consumer

This patch changes the way the CPU trace buffers are handled.
Instead of always starting from the trace page head, the logic
is changed to a producer consumer logic. This allows for the
buffers to be drained while they are alive.

Signed-off-by: Steven Rostedt <srostedt@redhat.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

2 files changed, 65 insertions, 44 deletions

diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 6580e7ed04be..777b859e1c2e 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -176,10 +176,9 @@ flip_trace(struct trace_array_cpu *tr1, struct trace_array_cpu *tr2)
 
         INIT_LIST_HEAD(&flip_pages);
 
-        tr1->trace_current = NULL;
-        memcpy(&tr1->trace_current_idx, &tr2->trace_current_idx,
+        memcpy(&tr1->trace_head_idx, &tr2->trace_head_idx,
                 sizeof(struct trace_array_cpu) -
-                offsetof(struct trace_array_cpu, trace_current_idx));
+                offsetof(struct trace_array_cpu, trace_head_idx));
 
         check_pages(tr1);
         check_pages(tr2);
@@ -228,7 +227,6 @@ update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
                 tracing_reset(max_tr.data[i]);
 
         flip_trace(max_tr.data[cpu], data);
-
         tracing_reset(data);
 
         __update_max_tr(tr, tsk, cpu);
@@ -343,9 +341,9 @@ void unregister_tracer(struct tracer *type)
 notrace void tracing_reset(struct trace_array_cpu *data)
 {
         data->trace_idx = 0;
-        data->trace_current = head_page(data);
-        data->trace_current_idx = 0;
-        data->time_offset = 0;
+        data->trace_head = data->trace_tail = head_page(data);
+        data->trace_head_idx = 0;
+        data->trace_tail_idx = 0;
 }
 
 #ifdef CONFIG_FTRACE
@@ -470,38 +468,65 @@ notrace void tracing_record_cmdline(struct task_struct *tsk)
         trace_save_cmdline(tsk);
 }
 
+static inline notrace struct list_head *
+trace_next_list(struct trace_array_cpu *data, struct list_head *next)
+{
+        /*
+         * Roundrobin - but skip the head (which is not a real page):
+         */
+        next = next->next;
+        if (unlikely(next == &data->trace_pages))
+                next = next->next;
+        BUG_ON(next == &data->trace_pages);
+
+        return next;
+}
+
+static inline notrace void *
+trace_next_page(struct trace_array_cpu *data, void *addr)
+{
+        struct list_head *next;
+        struct page *page;
+
+        page = virt_to_page(addr);
+
+        next = trace_next_list(data, &page->lru);
+        page = list_entry(next, struct page, lru);
+
+        return page_address(page);
+}
+
 static inline notrace struct trace_entry *
 tracing_get_trace_entry(struct trace_array *tr, struct trace_array_cpu *data)
 {
         unsigned long idx, idx_next;
         struct trace_entry *entry;
-        struct list_head *next;
-        struct page *page;
 
         data->trace_idx++;
-        idx = data->trace_current_idx;
+        idx = data->trace_head_idx;
         idx_next = idx + 1;
 
         BUG_ON(idx * TRACE_ENTRY_SIZE >= PAGE_SIZE);
 
-        entry = data->trace_current + idx * TRACE_ENTRY_SIZE;
+        entry = data->trace_head + idx * TRACE_ENTRY_SIZE;
 
         if (unlikely(idx_next >= ENTRIES_PER_PAGE)) {
-                page = virt_to_page(data->trace_current);
-                /*
-                 * Roundrobin - but skip the head (which is not a real page):
-                 */
-                next = page->lru.next;
-                if (unlikely(next == &data->trace_pages))
-                        next = next->next;
-                BUG_ON(next == &data->trace_pages);
-
-                page = list_entry(next, struct page, lru);
-                data->trace_current = page_address(page);
+                data->trace_head = trace_next_page(data, data->trace_head);
                 idx_next = 0;
         }
 
-        data->trace_current_idx = idx_next;
+        if (data->trace_head == data->trace_tail &&
+            idx_next == data->trace_tail_idx) {
+                /* overrun */
+                data->trace_tail_idx++;
+                if (data->trace_tail_idx >= ENTRIES_PER_PAGE) {
+                        data->trace_tail =
+                                trace_next_page(data, data->trace_tail);
+                        data->trace_tail_idx = 0;
+                }
+        }
+
+        data->trace_head_idx = idx_next;
 
         return entry;
 }
@@ -571,21 +596,11 @@ trace_entry_idx(struct trace_array *tr, struct trace_array_cpu *data,
                 return NULL;
 
         if (!iter->next_page[cpu]) {
-                /*
-                 * Initialize. If the count of elements in
-                 * this buffer is greater than the max entries
-                 * we had an underrun. Which means we looped around.
-                 * We can simply use the current pointer as our
-                 * starting point.
-                 */
-                if (data->trace_idx >= tr->entries) {
-                        page = virt_to_page(data->trace_current);
-                        iter->next_page[cpu] = &page->lru;
-                        iter->next_page_idx[cpu] = data->trace_current_idx;
-                } else {
-                        iter->next_page[cpu] = data->trace_pages.next;
-                        iter->next_page_idx[cpu] = 0;
-                }
+                /* Initialize the iterator for this cpu trace buffer */
+                WARN_ON(!data->trace_tail);
+                page = virt_to_page(data->trace_tail);
+                iter->next_page[cpu] = &page->lru;
+                iter->next_page_idx[cpu] = data->trace_tail_idx;
         }
 
         page = list_entry(iter->next_page[cpu], struct page, lru);
@@ -593,6 +608,12 @@ trace_entry_idx(struct trace_array *tr, struct trace_array_cpu *data,
 
         array = page_address(page);
 
+        /* Still possible to catch up to the tail */
+        if (iter->next_idx[cpu] && array == data->trace_tail &&
+            iter->next_page_idx[cpu] == data->trace_tail_idx)
+                return NULL;
+
+        WARN_ON(iter->next_page_idx[cpu] >= ENTRIES_PER_PAGE);
         return &array[iter->next_page_idx[cpu]];
 }
 
@@ -638,10 +659,8 @@ static void *find_next_entry_inc(struct trace_iterator *iter)
 
                         iter->next_page_idx[next_cpu] = 0;
                         iter->next_page[next_cpu] =
-                                iter->next_page[next_cpu]->next;
-                        if (iter->next_page[next_cpu] == &data->trace_pages)
-                                iter->next_page[next_cpu] =
-                                        data->trace_pages.next;
+                             trace_next_list(data, iter->next_page[next_cpu]);
+
                 }
         }
         iter->prev_ent = iter->ent;
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index 5df8ff2b84a7..0ce127455b4b 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -53,13 +53,15 @@ struct trace_entry {
  * the trace, etc.)
  */
 struct trace_array_cpu {
-        void                    *trace_current;
         struct list_head        trace_pages;
         atomic_t                disabled;
         cycle_t                 time_offset;
 
         /* these fields get copied into max-trace: */
-        unsigned                trace_current_idx;
+        unsigned                trace_head_idx;
+        unsigned                trace_tail_idx;
+        void                    *trace_head; /* producer */
+        void                    *trace_tail; /* consumer */
         unsigned long           trace_idx;
         unsigned long           saved_latency;
         unsigned long           critical_start;