1 files changed, 70 insertions, 74 deletions

diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 21153e64bf1c..ec439999f387 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -159,6 +159,8 @@ static union trace_eval_map_item *trace_eval_maps;
 #endif /* CONFIG_TRACE_EVAL_MAP_FILE */
 
 static int tracing_set_tracer(struct trace_array *tr, const char *buf);
+static void ftrace_trace_userstack(struct ring_buffer *buffer,
+                                   unsigned long flags, int pc);
 
 #define MAX_TRACER_SIZE         100
 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
@@ -496,8 +498,10 @@ int trace_pid_write(struct trace_pid_list *filtered_pids,
          * not modified.
          */
         pid_list = kmalloc(sizeof(*pid_list), GFP_KERNEL);
-        if (!pid_list)
+        if (!pid_list) {
+                trace_parser_put(&parser);
                 return -ENOMEM;
+        }
 
         pid_list->pid_max = READ_ONCE(pid_max);
 
@@ -507,6 +511,7 @@ int trace_pid_write(struct trace_pid_list *filtered_pids,
 
         pid_list->pids = vzalloc((pid_list->pid_max + 7) >> 3);
         if (!pid_list->pids) {
+                trace_parser_put(&parser);
                 kfree(pid_list);
                 return -ENOMEM;
         }
@@ -2749,12 +2754,21 @@ trace_function(struct trace_array *tr,
 
 #ifdef CONFIG_STACKTRACE
 
-#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
+/* Allow 4 levels of nesting: normal, softirq, irq, NMI */
+#define FTRACE_KSTACK_NESTING   4
+
+#define FTRACE_KSTACK_ENTRIES   (PAGE_SIZE / FTRACE_KSTACK_NESTING)
+
 struct ftrace_stack {
-        unsigned long           calls[FTRACE_STACK_MAX_ENTRIES];
+        unsigned long           calls[FTRACE_KSTACK_ENTRIES];
 };
 
-static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
+
+struct ftrace_stacks {
+        struct ftrace_stack     stacks[FTRACE_KSTACK_NESTING];
+};
+
+static DEFINE_PER_CPU(struct ftrace_stacks, ftrace_stacks);
 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
 
 static void __ftrace_trace_stack(struct ring_buffer *buffer,
@@ -2763,13 +2777,10 @@ static void __ftrace_trace_stack(struct ring_buffer *buffer,
 {
         struct trace_event_call *call = &event_kernel_stack;
         struct ring_buffer_event *event;
+        unsigned int size, nr_entries;
+        struct ftrace_stack *fstack;
         struct stack_entry *entry;
-        struct stack_trace trace;
-        int use_stack;
-        int size = FTRACE_STACK_ENTRIES;
-
-        trace.nr_entries        = 0;
-        trace.skip              = skip;
+        int stackidx;
 
         /*
          * Add one, for this function and the call to save_stack_trace()
@@ -2777,7 +2788,7 @@ static void __ftrace_trace_stack(struct ring_buffer *buffer,
          */
 #ifndef CONFIG_UNWINDER_ORC
         if (!regs)
-                trace.skip++;
+                skip++;
 #endif
 
         /*
@@ -2788,53 +2799,40 @@ static void __ftrace_trace_stack(struct ring_buffer *buffer,
          */
         preempt_disable_notrace();
 
-        use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
+        stackidx = __this_cpu_inc_return(ftrace_stack_reserve) - 1;
+
+        /* This should never happen. If it does, yell once and skip */
+        if (WARN_ON_ONCE(stackidx > FTRACE_KSTACK_NESTING))
+                goto out;
+
         /*
-         * We don't need any atomic variables, just a barrier.
-         * If an interrupt comes in, we don't care, because it would
-         * have exited and put the counter back to what we want.
-         * We just need a barrier to keep gcc from moving things
-         * around.
+         * The above __this_cpu_inc_return() is 'atomic' cpu local. An
+         * interrupt will either see the value pre increment or post
+         * increment. If the interrupt happens pre increment it will have
+         * restored the counter when it returns.  We just need a barrier to
+         * keep gcc from moving things around.
          */
         barrier();
-        if (use_stack == 1) {
-                trace.entries           = this_cpu_ptr(ftrace_stack.calls);
-                trace.max_entries       = FTRACE_STACK_MAX_ENTRIES;
 
-                if (regs)
-                        save_stack_trace_regs(regs, &trace);
-                else
-                        save_stack_trace(&trace);
-
-                if (trace.nr_entries > size)
-                        size = trace.nr_entries;
-        } else
-                /* From now on, use_stack is a boolean */
-                use_stack = 0;
+        fstack = this_cpu_ptr(ftrace_stacks.stacks) + stackidx;
+        size = ARRAY_SIZE(fstack->calls);
 
-        size *= sizeof(unsigned long);
+        if (regs) {
+                nr_entries = stack_trace_save_regs(regs, fstack->calls,
+                                                   size, skip);
+        } else {
+                nr_entries = stack_trace_save(fstack->calls, size, skip);
+        }
 
+        size = nr_entries * sizeof(unsigned long);
         event = __trace_buffer_lock_reserve(buffer, TRACE_STACK,
                                             sizeof(*entry) + size, flags, pc);
         if (!event)
                 goto out;
         entry = ring_buffer_event_data(event);
 
-        memset(&entry->caller, 0, size);
-
-        if (use_stack)
-                memcpy(&entry->caller, trace.entries,
-                       trace.nr_entries * sizeof(unsigned long));
-        else {
-                trace.max_entries       = FTRACE_STACK_ENTRIES;
-                trace.entries           = entry->caller;
-                if (regs)
-                        save_stack_trace_regs(regs, &trace);
-                else
-                        save_stack_trace(&trace);
-        }
-
-        entry->size = trace.nr_entries;
+        memcpy(&entry->caller, fstack->calls, size);
+        entry->size = nr_entries;
 
         if (!call_filter_check_discard(call, entry, buffer, event))
                 __buffer_unlock_commit(buffer, event);
@@ -2904,15 +2902,15 @@ void trace_dump_stack(int skip)
 }
 EXPORT_SYMBOL_GPL(trace_dump_stack);
 
+#ifdef CONFIG_USER_STACKTRACE_SUPPORT
 static DEFINE_PER_CPU(int, user_stack_count);
 
-void
+static void
 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
 {
         struct trace_event_call *call = &event_user_stack;
         struct ring_buffer_event *event;
         struct userstack_entry *entry;
-        struct stack_trace trace;
 
         if (!(global_trace.trace_flags & TRACE_ITER_USERSTACKTRACE))
                 return;
@@ -2943,12 +2941,7 @@ ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
         entry->tgid             = current->tgid;
         memset(&entry->caller, 0, sizeof(entry->caller));
 
-        trace.nr_entries        = 0;
-        trace.max_entries       = FTRACE_STACK_ENTRIES;
-        trace.skip              = 0;
-        trace.entries           = entry->caller;
-
-        save_stack_trace_user(&trace);
+        stack_trace_save_user(entry->caller, FTRACE_STACK_ENTRIES);
         if (!call_filter_check_discard(call, entry, buffer, event))
                 __buffer_unlock_commit(buffer, event);
 
@@ -2957,13 +2950,12 @@ ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
  out:
         preempt_enable();
 }
-
-#ifdef UNUSED
-static void __trace_userstack(struct trace_array *tr, unsigned long flags)
+#else /* CONFIG_USER_STACKTRACE_SUPPORT */
+static void ftrace_trace_userstack(struct ring_buffer *buffer,
+                                   unsigned long flags, int pc)
 {
-        ftrace_trace_userstack(tr, flags, preempt_count());
 }
-#endif /* UNUSED */
+#endif /* !CONFIG_USER_STACKTRACE_SUPPORT */
 
 #endif /* CONFIG_STACKTRACE */
 
@@ -7025,35 +7017,43 @@ struct buffer_ref {
         struct ring_buffer      *buffer;
         void                    *page;
         int                     cpu;
-        int                     ref;
+        refcount_t              refcount;
 };
 
+static void buffer_ref_release(struct buffer_ref *ref)
+{
+        if (!refcount_dec_and_test(&ref->refcount))
+                return;
+        ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page);
+        kfree(ref);
+}
+
 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
                                     struct pipe_buffer *buf)
 {
         struct buffer_ref *ref = (struct buffer_ref *)buf->private;
 
-        if (--ref->ref)
-                return;
-
-        ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page);
-        kfree(ref);
+        buffer_ref_release(ref);
         buf->private = 0;
 }
 
-static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
+static bool buffer_pipe_buf_get(struct pipe_inode_info *pipe,
                                 struct pipe_buffer *buf)
 {
         struct buffer_ref *ref = (struct buffer_ref *)buf->private;
 
-        ref->ref++;
+        if (refcount_read(&ref->refcount) > INT_MAX/2)
+                return false;
+
+        refcount_inc(&ref->refcount);
+        return true;
 }
 
 /* Pipe buffer operations for a buffer. */
 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
         .confirm                = generic_pipe_buf_confirm,
         .release                = buffer_pipe_buf_release,
-        .steal                  = generic_pipe_buf_steal,
+        .steal                  = generic_pipe_buf_nosteal,
         .get                    = buffer_pipe_buf_get,
 };
 
@@ -7066,11 +7066,7 @@ static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
         struct buffer_ref *ref =
                 (struct buffer_ref *)spd->partial[i].private;
 
-        if (--ref->ref)
-                return;
-
-        ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page);
-        kfree(ref);
+        buffer_ref_release(ref);
         spd->partial[i].private = 0;
 }
 
@@ -7125,7 +7121,7 @@ tracing_buffers_splice_read(struct file *file, loff_t *ppos,
                         break;
                 }
 
-                ref->ref = 1;
+                refcount_set(&ref->refcount, 1);
                 ref->buffer = iter->trace_buffer->buffer;
                 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
                 if (IS_ERR(ref->page)) {