tracing: Allocate the ftrace event profile buffer dynamically

Currently the trace event profile buffer is allocated in the stack. But
this may be too much for the stack, as the events can have large
statically defined field size and can also grow with dynamic arrays.

Allocate two per cpu buffer for all profiled events. The first cpu
buffer is used to host every non-nmi context traces. It is protected
by disabling the interrupts while writing and committing the trace.

The second buffer is reserved for nmi. So that there is no race between
them and the first buffer.

The whole write/commit section is rcu protected because we release
these buffers while deactivating the last profiling trace event.

v2: Move the buffers from trace_event to be global, as pointed by
    Steven Rostedt.

v3: Fix the syscall events to handle the profiling buffer races
    by disabling interrupts, now that the buffers are globals.

Suggested-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Li Zefan <lizf@cn.fujitsu.com>
Cc: Jason Baron <jbaron@redhat.com>
Cc: Masami Hiramatsu <mhiramat@redhat.com>
Cc: Ingo Molnar <mingo@elte.hu>

2 files changed, 61 insertions, 28 deletions

diff --git a/include/linux/ftrace_event.h b/include/linux/ftrace_event.h
index bc103d7b1ca8..4ec5e67e18cf 100644
--- a/include/linux/ftrace_event.h
+++ b/include/linux/ftrace_event.h
@@ -4,6 +4,7 @@
 #include <linux/ring_buffer.h>
 #include <linux/trace_seq.h>
 #include <linux/percpu.h>
+#include <linux/hardirq.h>
 
 struct trace_array;
 struct tracer;
@@ -134,6 +135,11 @@ struct ftrace_event_call {
         void                    (*profile_disable)(void);
 };
 
+#define FTRACE_MAX_PROFILE_SIZE 2048
+
+extern char                     *trace_profile_buf;
+extern char                     *trace_profile_buf_nmi;
+
 #define MAX_FILTER_PRED         32
 #define MAX_FILTER_STR_VAL      256     /* Should handle KSYM_SYMBOL_LEN */
 
diff --git a/include/trace/ftrace.h b/include/trace/ftrace.h
index a822087857e9..a0361cb69769 100644
--- a/include/trace/ftrace.h
+++ b/include/trace/ftrace.h
@@ -648,11 +648,12 @@ __attribute__((section("_ftrace_events"))) event_##call = {		\
  *      struct ftrace_raw_##call *entry;
  *      u64 __addr = 0, __count = 1;
  *      unsigned long irq_flags;
+ *      struct trace_entry *ent;
  *      int __entry_size;
  *      int __data_size;
+ *      int __cpu
  *      int pc;
  *
- *      local_save_flags(irq_flags);
  *      pc = preempt_count();
  *
  *      __data_size = ftrace_get_offsets_<call>(&__data_offsets, args);
@@ -663,25 +664,34 @@ __attribute__((section("_ftrace_events"))) event_##call = {		\
  *                           sizeof(u64));
  *      __entry_size -= sizeof(u32);
  *
- *      do {
- *              char raw_data[__entry_size]; <- allocate our sample in the stack
- *              struct trace_entry *ent;
+ *      // Protect the non nmi buffer
+ *      // This also protects the rcu read side
+ *      local_irq_save(irq_flags);
+ *      __cpu = smp_processor_id();
+ *
+ *      if (in_nmi())
+ *              raw_data = rcu_dereference(trace_profile_buf_nmi);
+ *      else
+ *              raw_data = rcu_dereference(trace_profile_buf);
+ *
+ *      if (!raw_data)
+ *              goto end;
  *
- *              zero dead bytes from alignment to avoid stack leak to userspace:
+ *      raw_data = per_cpu_ptr(raw_data, __cpu);
  *
- *              *(u64 *)(&raw_data[__entry_size - sizeof(u64)]) = 0ULL;
- *              entry = (struct ftrace_raw_<call> *)raw_data;
- *              ent = &entry->ent;
- *              tracing_generic_entry_update(ent, irq_flags, pc);
- *              ent->type = event_call->id;
+ *      //zero dead bytes from alignment to avoid stack leak to userspace:
+ *      *(u64 *)(&raw_data[__entry_size - sizeof(u64)]) = 0ULL;
+ *      entry = (struct ftrace_raw_<call> *)raw_data;
+ *      ent = &entry->ent;
+ *      tracing_generic_entry_update(ent, irq_flags, pc);
+ *      ent->type = event_call->id;
  *
- *              <tstruct> <- do some jobs with dynamic arrays
+ *      <tstruct> <- do some jobs with dynamic arrays
  *
- *              <assign>  <- affect our values
+ *      <assign>  <- affect our values
  *
- *              perf_tpcounter_event(event_call->id, __addr, __count, entry,
- *                           __entry_size);  <- submit them to perf counter
- *      } while (0);
+ *      perf_tpcounter_event(event_call->id, __addr, __count, entry,
+ *                   __entry_size);  <- submit them to perf counter
  *
  * }
  */
@@ -704,11 +714,13 @@ static void ftrace_profile_##call(proto)				\
         struct ftrace_raw_##call *entry;                                \
         u64 __addr = 0, __count = 1;                                    \
         unsigned long irq_flags;                                        \
+        struct trace_entry *ent;                                        \
         int __entry_size;                                               \
         int __data_size;                                                \
+        char *raw_data;                                                 \
+        int __cpu;                                                      \
         int pc;                                                         \
                                                                         \
-        local_save_flags(irq_flags);                                    \
         pc = preempt_count();                                           \
                                                                         \
         __data_size = ftrace_get_offsets_##call(&__data_offsets, args); \
@@ -716,23 +728,38 @@ static void ftrace_profile_##call(proto)				\
                              sizeof(u64));                              \
         __entry_size -= sizeof(u32);                                    \
                                                                         \
-        do {                                                            \
-                char raw_data[__entry_size];                            \
-                struct trace_entry *ent;                                \
+        if (WARN_ONCE(__entry_size > FTRACE_MAX_PROFILE_SIZE,           \
+                      "profile buffer not large enough"))               \
+                return;                                                 \
+                                                                        \
+        local_irq_save(irq_flags);                                      \
+        __cpu = smp_processor_id();                                     \
                                                                         \
-                *(u64 *)(&raw_data[__entry_size - sizeof(u64)]) = 0ULL; \
-                entry = (struct ftrace_raw_##call *)raw_data;           \
-                ent = &entry->ent;                                      \
-                tracing_generic_entry_update(ent, irq_flags, pc);       \
-                ent->type = event_call->id;                             \
+        if (in_nmi())                                                   \
+                raw_data = rcu_dereference(trace_profile_buf_nmi);              \
+        else                                                            \
+                raw_data = rcu_dereference(trace_profile_buf);          \
                                                                         \
-                tstruct                                                 \
+        if (!raw_data)                                                  \
+                goto end;                                               \
                                                                         \
-                { assign; }                                             \
+        raw_data = per_cpu_ptr(raw_data, __cpu);                        \
                                                                         \
-                perf_tpcounter_event(event_call->id, __addr, __count, entry,\
+        *(u64 *)(&raw_data[__entry_size - sizeof(u64)]) = 0ULL;         \
+        entry = (struct ftrace_raw_##call *)raw_data;                   \
+        ent = &entry->ent;                                              \
+        tracing_generic_entry_update(ent, irq_flags, pc);               \
+        ent->type = event_call->id;                                     \
+                                                                        \
+        tstruct                                                         \
+                                                                        \
+        { assign; }                                                     \
+                                                                        \
+        perf_tpcounter_event(event_call->id, __addr, __count, entry,    \
                              __entry_size);                             \
-        } while (0);                                                    \
+                                                                        \
+end:                                                                    \
+        local_irq_restore(irq_flags);                                   \
                                                                         \
 }