1 files changed, 352 insertions, 135 deletions

diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c
index b6dcc42ef7f5..79e52d93860b 100644
--- a/kernel/trace/trace_uprobe.c
+++ b/kernel/trace/trace_uprobe.c
@@ -51,22 +51,17 @@ struct trace_uprobe_filter {
  */
 struct trace_uprobe {
         struct list_head                list;
-        struct ftrace_event_class       class;
-        struct ftrace_event_call        call;
         struct trace_uprobe_filter      filter;
         struct uprobe_consumer          consumer;
         struct inode                    *inode;
         char                            *filename;
         unsigned long                   offset;
         unsigned long                   nhit;
-        unsigned int                    flags;  /* For TP_FLAG_* */
-        ssize_t                         size;   /* trace entry size */
-        unsigned int                    nr_args;
-        struct probe_arg                args[];
+        struct trace_probe              tp;
 };
 
-#define SIZEOF_TRACE_UPROBE(n)                  \
-        (offsetof(struct trace_uprobe, args) +  \
+#define SIZEOF_TRACE_UPROBE(n)                          \
+        (offsetof(struct trace_uprobe, tp.args) +       \
         (sizeof(struct probe_arg) * (n)))
 
 static int register_uprobe_event(struct trace_uprobe *tu);
@@ -75,10 +70,151 @@ static int unregister_uprobe_event(struct trace_uprobe *tu);
 static DEFINE_MUTEX(uprobe_lock);
 static LIST_HEAD(uprobe_list);
 
+struct uprobe_dispatch_data {
+        struct trace_uprobe     *tu;
+        unsigned long           bp_addr;
+};
+
 static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs);
 static int uretprobe_dispatcher(struct uprobe_consumer *con,
                                 unsigned long func, struct pt_regs *regs);
 
+#ifdef CONFIG_STACK_GROWSUP
+static unsigned long adjust_stack_addr(unsigned long addr, unsigned int n)
+{
+        return addr - (n * sizeof(long));
+}
+#else
+static unsigned long adjust_stack_addr(unsigned long addr, unsigned int n)
+{
+        return addr + (n * sizeof(long));
+}
+#endif
+
+static unsigned long get_user_stack_nth(struct pt_regs *regs, unsigned int n)
+{
+        unsigned long ret;
+        unsigned long addr = user_stack_pointer(regs);
+
+        addr = adjust_stack_addr(addr, n);
+
+        if (copy_from_user(&ret, (void __force __user *) addr, sizeof(ret)))
+                return 0;
+
+        return ret;
+}
+
+/*
+ * Uprobes-specific fetch functions
+ */
+#define DEFINE_FETCH_stack(type)                                        \
+static __kprobes void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs,\
+                                          void *offset, void *dest)     \
+{                                                                       \
+        *(type *)dest = (type)get_user_stack_nth(regs,                  \
+                                              ((unsigned long)offset)); \
+}
+DEFINE_BASIC_FETCH_FUNCS(stack)
+/* No string on the stack entry */
+#define fetch_stack_string      NULL
+#define fetch_stack_string_size NULL
+
+#define DEFINE_FETCH_memory(type)                                       \
+static __kprobes void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs,\
+                                                void *addr, void *dest) \
+{                                                                       \
+        type retval;                                                    \
+        void __user *vaddr = (void __force __user *) addr;              \
+                                                                        \
+        if (copy_from_user(&retval, vaddr, sizeof(type)))               \
+                *(type *)dest = 0;                                      \
+        else                                                            \
+                *(type *) dest = retval;                                \
+}
+DEFINE_BASIC_FETCH_FUNCS(memory)
+/*
+ * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max
+ * length and relative data location.
+ */
+static __kprobes void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs,
+                                                      void *addr, void *dest)
+{
+        long ret;
+        u32 rloc = *(u32 *)dest;
+        int maxlen  = get_rloc_len(rloc);
+        u8 *dst = get_rloc_data(dest);
+        void __user *src = (void __force __user *) addr;
+
+        if (!maxlen)
+                return;
+
+        ret = strncpy_from_user(dst, src, maxlen);
+
+        if (ret < 0) {  /* Failed to fetch string */
+                ((u8 *)get_rloc_data(dest))[0] = '\0';
+                *(u32 *)dest = make_data_rloc(0, get_rloc_offs(rloc));
+        } else {
+                *(u32 *)dest = make_data_rloc(ret, get_rloc_offs(rloc));
+        }
+}
+
+static __kprobes void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs,
+                                                      void *addr, void *dest)
+{
+        int len;
+        void __user *vaddr = (void __force __user *) addr;
+
+        len = strnlen_user(vaddr, MAX_STRING_SIZE);
+
+        if (len == 0 || len > MAX_STRING_SIZE)  /* Failed to check length */
+                *(u32 *)dest = 0;
+        else
+                *(u32 *)dest = len;
+}
+
+static unsigned long translate_user_vaddr(void *file_offset)
+{
+        unsigned long base_addr;
+        struct uprobe_dispatch_data *udd;
+
+        udd = (void *) current->utask->vaddr;
+
+        base_addr = udd->bp_addr - udd->tu->offset;
+        return base_addr + (unsigned long)file_offset;
+}
+
+#define DEFINE_FETCH_file_offset(type)                                  \
+static __kprobes void FETCH_FUNC_NAME(file_offset, type)(struct pt_regs *regs,\
+                                        void *offset, void *dest)       \
+{                                                                       \
+        void *vaddr = (void *)translate_user_vaddr(offset);             \
+                                                                        \
+        FETCH_FUNC_NAME(memory, type)(regs, vaddr, dest);               \
+}
+DEFINE_BASIC_FETCH_FUNCS(file_offset)
+DEFINE_FETCH_file_offset(string)
+DEFINE_FETCH_file_offset(string_size)
+
+/* Fetch type information table */
+const struct fetch_type uprobes_fetch_type_table[] = {
+        /* Special types */
+        [FETCH_TYPE_STRING] = __ASSIGN_FETCH_TYPE("string", string, string,
+                                        sizeof(u32), 1, "__data_loc char[]"),
+        [FETCH_TYPE_STRSIZE] = __ASSIGN_FETCH_TYPE("string_size", u32,
+                                        string_size, sizeof(u32), 0, "u32"),
+        /* Basic types */
+        ASSIGN_FETCH_TYPE(u8,  u8,  0),
+        ASSIGN_FETCH_TYPE(u16, u16, 0),
+        ASSIGN_FETCH_TYPE(u32, u32, 0),
+        ASSIGN_FETCH_TYPE(u64, u64, 0),
+        ASSIGN_FETCH_TYPE(s8,  u8,  1),
+        ASSIGN_FETCH_TYPE(s16, u16, 1),
+        ASSIGN_FETCH_TYPE(s32, u32, 1),
+        ASSIGN_FETCH_TYPE(s64, u64, 1),
+
+        ASSIGN_FETCH_TYPE_END
+};
+
 static inline void init_trace_uprobe_filter(struct trace_uprobe_filter *filter)
 {
         rwlock_init(&filter->rwlock);
@@ -114,13 +250,13 @@ alloc_trace_uprobe(const char *group, const char *event, int nargs, bool is_ret)
         if (!tu)
                 return ERR_PTR(-ENOMEM);
 
-        tu->call.class = &tu->class;
-        tu->call.name = kstrdup(event, GFP_KERNEL);
-        if (!tu->call.name)
+        tu->tp.call.class = &tu->tp.class;
+        tu->tp.call.name = kstrdup(event, GFP_KERNEL);
+        if (!tu->tp.call.name)
                 goto error;
 
-        tu->class.system = kstrdup(group, GFP_KERNEL);
-        if (!tu->class.system)
+        tu->tp.class.system = kstrdup(group, GFP_KERNEL);
+        if (!tu->tp.class.system)
                 goto error;
 
         INIT_LIST_HEAD(&tu->list);
@@ -128,11 +264,11 @@ alloc_trace_uprobe(const char *group, const char *event, int nargs, bool is_ret)
         if (is_ret)
                 tu->consumer.ret_handler = uretprobe_dispatcher;
         init_trace_uprobe_filter(&tu->filter);
-        tu->call.flags |= TRACE_EVENT_FL_USE_CALL_FILTER;
+        tu->tp.call.flags |= TRACE_EVENT_FL_USE_CALL_FILTER;
         return tu;
 
 error:
-        kfree(tu->call.name);
+        kfree(tu->tp.call.name);
         kfree(tu);
 
         return ERR_PTR(-ENOMEM);
@@ -142,12 +278,12 @@ static void free_trace_uprobe(struct trace_uprobe *tu)
 {
         int i;
 
-        for (i = 0; i < tu->nr_args; i++)
-                traceprobe_free_probe_arg(&tu->args[i]);
+        for (i = 0; i < tu->tp.nr_args; i++)
+                traceprobe_free_probe_arg(&tu->tp.args[i]);
 
         iput(tu->inode);
-        kfree(tu->call.class->system);
-        kfree(tu->call.name);
+        kfree(tu->tp.call.class->system);
+        kfree(tu->tp.call.name);
         kfree(tu->filename);
         kfree(tu);
 }
@@ -157,8 +293,8 @@ static struct trace_uprobe *find_probe_event(const char *event, const char *grou
         struct trace_uprobe *tu;
 
         list_for_each_entry(tu, &uprobe_list, list)
-                if (strcmp(tu->call.name, event) == 0 &&
-                    strcmp(tu->call.class->system, group) == 0)
+                if (strcmp(tu->tp.call.name, event) == 0 &&
+                    strcmp(tu->tp.call.class->system, group) == 0)
                         return tu;
 
         return NULL;
@@ -181,16 +317,16 @@ static int unregister_trace_uprobe(struct trace_uprobe *tu)
 /* Register a trace_uprobe and probe_event */
 static int register_trace_uprobe(struct trace_uprobe *tu)
 {
-        struct trace_uprobe *old_tp;
+        struct trace_uprobe *old_tu;
         int ret;
 
         mutex_lock(&uprobe_lock);
 
         /* register as an event */
-        old_tp = find_probe_event(tu->call.name, tu->call.class->system);
-        if (old_tp) {
+        old_tu = find_probe_event(tu->tp.call.name, tu->tp.call.class->system);
+        if (old_tu) {
                 /* delete old event */
-                ret = unregister_trace_uprobe(old_tp);
+                ret = unregister_trace_uprobe(old_tu);
                 if (ret)
                         goto end;
         }
@@ -211,7 +347,7 @@ end:
 
 /*
  * Argument syntax:
- *  - Add uprobe: p|r[:[GRP/]EVENT] PATH:SYMBOL [FETCHARGS]
+ *  - Add uprobe: p|r[:[GRP/]EVENT] PATH:OFFSET [FETCHARGS]
  *
  *  - Remove uprobe: -:[GRP/]EVENT
  */
@@ -360,34 +496,36 @@ static int create_trace_uprobe(int argc, char **argv)
         /* parse arguments */
         ret = 0;
         for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {
+                struct probe_arg *parg = &tu->tp.args[i];
+
                 /* Increment count for freeing args in error case */
-                tu->nr_args++;
+                tu->tp.nr_args++;
 
                 /* Parse argument name */
                 arg = strchr(argv[i], '=');
                 if (arg) {
                         *arg++ = '\0';
-                        tu->args[i].name = kstrdup(argv[i], GFP_KERNEL);
+                        parg->name = kstrdup(argv[i], GFP_KERNEL);
                 } else {
                         arg = argv[i];
                         /* If argument name is omitted, set "argN" */
                         snprintf(buf, MAX_EVENT_NAME_LEN, "arg%d", i + 1);
-                        tu->args[i].name = kstrdup(buf, GFP_KERNEL);
+                        parg->name = kstrdup(buf, GFP_KERNEL);
                 }
 
-                if (!tu->args[i].name) {
+                if (!parg->name) {
                         pr_info("Failed to allocate argument[%d] name.\n", i);
                         ret = -ENOMEM;
                         goto error;
                 }
 
-                if (!is_good_name(tu->args[i].name)) {
-                        pr_info("Invalid argument[%d] name: %s\n", i, tu->args[i].name);
+                if (!is_good_name(parg->name)) {
+                        pr_info("Invalid argument[%d] name: %s\n", i, parg->name);
                         ret = -EINVAL;
                         goto error;
                 }
 
-                if (traceprobe_conflict_field_name(tu->args[i].name, tu->args, i)) {
+                if (traceprobe_conflict_field_name(parg->name, tu->tp.args, i)) {
                         pr_info("Argument[%d] name '%s' conflicts with "
                                 "another field.\n", i, argv[i]);
                         ret = -EINVAL;
@@ -395,7 +533,8 @@ static int create_trace_uprobe(int argc, char **argv)
                 }
 
                 /* Parse fetch argument */
-                ret = traceprobe_parse_probe_arg(arg, &tu->size, &tu->args[i], false, false);
+                ret = traceprobe_parse_probe_arg(arg, &tu->tp.size, parg,
+                                                 is_return, false);
                 if (ret) {
                         pr_info("Parse error at argument[%d]. (%d)\n", i, ret);
                         goto error;
@@ -459,11 +598,11 @@ static int probes_seq_show(struct seq_file *m, void *v)
         char c = is_ret_probe(tu) ? 'r' : 'p';
         int i;
 
-        seq_printf(m, "%c:%s/%s", c, tu->call.class->system, tu->call.name);
+        seq_printf(m, "%c:%s/%s", c, tu->tp.call.class->system, tu->tp.call.name);
         seq_printf(m, " %s:0x%p", tu->filename, (void *)tu->offset);
 
-        for (i = 0; i < tu->nr_args; i++)
-                seq_printf(m, " %s=%s", tu->args[i].name, tu->args[i].comm);
+        for (i = 0; i < tu->tp.nr_args; i++)
+                seq_printf(m, " %s=%s", tu->tp.args[i].name, tu->tp.args[i].comm);
 
         seq_printf(m, "\n");
         return 0;
@@ -509,7 +648,7 @@ static int probes_profile_seq_show(struct seq_file *m, void *v)
 {
         struct trace_uprobe *tu = v;
 
-        seq_printf(m, "  %s %-44s %15lu\n", tu->filename, tu->call.name, tu->nhit);
+        seq_printf(m, "  %s %-44s %15lu\n", tu->filename, tu->tp.call.name, tu->nhit);
         return 0;
 }
 
@@ -533,21 +672,117 @@ static const struct file_operations uprobe_profile_ops = {
         .release        = seq_release,
 };
 
+struct uprobe_cpu_buffer {
+        struct mutex mutex;
+        void *buf;
+};
+static struct uprobe_cpu_buffer __percpu *uprobe_cpu_buffer;
+static int uprobe_buffer_refcnt;
+
+static int uprobe_buffer_init(void)
+{
+        int cpu, err_cpu;
+
+        uprobe_cpu_buffer = alloc_percpu(struct uprobe_cpu_buffer);
+        if (uprobe_cpu_buffer == NULL)
+                return -ENOMEM;
+
+        for_each_possible_cpu(cpu) {
+                struct page *p = alloc_pages_node(cpu_to_node(cpu),
+                                                  GFP_KERNEL, 0);
+                if (p == NULL) {
+                        err_cpu = cpu;
+                        goto err;
+                }
+                per_cpu_ptr(uprobe_cpu_buffer, cpu)->buf = page_address(p);
+                mutex_init(&per_cpu_ptr(uprobe_cpu_buffer, cpu)->mutex);
+        }
+
+        return 0;
+
+err:
+        for_each_possible_cpu(cpu) {
+                if (cpu == err_cpu)
+                        break;
+                free_page((unsigned long)per_cpu_ptr(uprobe_cpu_buffer, cpu)->buf);
+        }
+
+        free_percpu(uprobe_cpu_buffer);
+        return -ENOMEM;
+}
+
+static int uprobe_buffer_enable(void)
+{
+        int ret = 0;
+
+        BUG_ON(!mutex_is_locked(&event_mutex));
+
+        if (uprobe_buffer_refcnt++ == 0) {
+                ret = uprobe_buffer_init();
+                if (ret < 0)
+                        uprobe_buffer_refcnt--;
+        }
+
+        return ret;
+}
+
+static void uprobe_buffer_disable(void)
+{
+        BUG_ON(!mutex_is_locked(&event_mutex));
+
+        if (--uprobe_buffer_refcnt == 0) {
+                free_percpu(uprobe_cpu_buffer);
+                uprobe_cpu_buffer = NULL;
+        }
+}
+
+static struct uprobe_cpu_buffer *uprobe_buffer_get(void)
+{
+        struct uprobe_cpu_buffer *ucb;
+        int cpu;
+
+        cpu = raw_smp_processor_id();
+        ucb = per_cpu_ptr(uprobe_cpu_buffer, cpu);
+
+        /*
+         * Use per-cpu buffers for fastest access, but we might migrate
+         * so the mutex makes sure we have sole access to it.
+         */
+        mutex_lock(&ucb->mutex);
+
+        return ucb;
+}
+
+static void uprobe_buffer_put(struct uprobe_cpu_buffer *ucb)
+{
+        mutex_unlock(&ucb->mutex);
+}
+
 static void uprobe_trace_print(struct trace_uprobe *tu,
                                 unsigned long func, struct pt_regs *regs)
 {
         struct uprobe_trace_entry_head *entry;
         struct ring_buffer_event *event;
         struct ring_buffer *buffer;
+        struct uprobe_cpu_buffer *ucb;
         void *data;
-        int size, i;
-        struct ftrace_event_call *call = &tu->call;
+        int size, dsize, esize;
+        struct ftrace_event_call *call = &tu->tp.call;
+
+        dsize = __get_data_size(&tu->tp, regs);
+        esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
 
-        size = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
+        if (WARN_ON_ONCE(!uprobe_cpu_buffer || tu->tp.size + dsize > PAGE_SIZE))
+                return;
+
+        ucb = uprobe_buffer_get();
+        store_trace_args(esize, &tu->tp, regs, ucb->buf, dsize);
+
+        size = esize + tu->tp.size + dsize;
         event = trace_current_buffer_lock_reserve(&buffer, call->event.type,
-                                                  size + tu->size, 0, 0);
+                                                  size, 0, 0);
         if (!event)
-                return;
+                goto out;
 
         entry = ring_buffer_event_data(event);
         if (is_ret_probe(tu)) {
@@ -559,11 +794,13 @@ static void uprobe_trace_print(struct trace_uprobe *tu,
                 data = DATAOF_TRACE_ENTRY(entry, false);
         }
 
-        for (i = 0; i < tu->nr_args; i++)
-                call_fetch(&tu->args[i].fetch, regs, data + tu->args[i].offset);
+        memcpy(data, ucb->buf, tu->tp.size + dsize);
 
         if (!call_filter_check_discard(call, entry, buffer, event))
                 trace_buffer_unlock_commit(buffer, event, 0, 0);
+
+out:
+        uprobe_buffer_put(ucb);
 }
 
 /* uprobe handler */
@@ -591,23 +828,24 @@ print_uprobe_event(struct trace_iterator *iter, int flags, struct trace_event *e
         int i;
 
         entry = (struct uprobe_trace_entry_head *)iter->ent;
-        tu = container_of(event, struct trace_uprobe, call.event);
+        tu = container_of(event, struct trace_uprobe, tp.call.event);
 
         if (is_ret_probe(tu)) {
-                if (!trace_seq_printf(s, "%s: (0x%lx <- 0x%lx)", tu->call.name,
+                if (!trace_seq_printf(s, "%s: (0x%lx <- 0x%lx)", tu->tp.call.name,
                                         entry->vaddr[1], entry->vaddr[0]))
                         goto partial;
                 data = DATAOF_TRACE_ENTRY(entry, true);
         } else {
-                if (!trace_seq_printf(s, "%s: (0x%lx)", tu->call.name,
+                if (!trace_seq_printf(s, "%s: (0x%lx)", tu->tp.call.name,
                                         entry->vaddr[0]))
                         goto partial;
                 data = DATAOF_TRACE_ENTRY(entry, false);
         }
 
-        for (i = 0; i < tu->nr_args; i++) {
-                if (!tu->args[i].type->print(s, tu->args[i].name,
-                                             data + tu->args[i].offset, entry))
+        for (i = 0; i < tu->tp.nr_args; i++) {
+                struct probe_arg *parg = &tu->tp.args[i];
+
+                if (!parg->type->print(s, parg->name, data + parg->offset, entry))
                         goto partial;
         }
 
@@ -618,11 +856,6 @@ partial:
         return TRACE_TYPE_PARTIAL_LINE;
 }
 
-static inline bool is_trace_uprobe_enabled(struct trace_uprobe *tu)
-{
-        return tu->flags & (TP_FLAG_TRACE | TP_FLAG_PROFILE);
-}
-
 typedef bool (*filter_func_t)(struct uprobe_consumer *self,
                                 enum uprobe_filter_ctx ctx,
                                 struct mm_struct *mm);
@@ -632,29 +865,35 @@ probe_event_enable(struct trace_uprobe *tu, int flag, filter_func_t filter)
 {
         int ret = 0;
 
-        if (is_trace_uprobe_enabled(tu))
+        if (trace_probe_is_enabled(&tu->tp))
                 return -EINTR;
 
+        ret = uprobe_buffer_enable();
+        if (ret < 0)
+                return ret;
+
         WARN_ON(!uprobe_filter_is_empty(&tu->filter));
 
-        tu->flags |= flag;
+        tu->tp.flags |= flag;
         tu->consumer.filter = filter;
         ret = uprobe_register(tu->inode, tu->offset, &tu->consumer);
         if (ret)
-                tu->flags &= ~flag;
+                tu->tp.flags &= ~flag;
 
         return ret;
 }
 
 static void probe_event_disable(struct trace_uprobe *tu, int flag)
 {
-        if (!is_trace_uprobe_enabled(tu))
+        if (!trace_probe_is_enabled(&tu->tp))
                 return;
 
         WARN_ON(!uprobe_filter_is_empty(&tu->filter));
 
         uprobe_unregister(tu->inode, tu->offset, &tu->consumer);
-        tu->flags &= ~flag;
+        tu->tp.flags &= ~flag;
+
+        uprobe_buffer_disable();
 }
 
 static int uprobe_event_define_fields(struct ftrace_event_call *event_call)
@@ -672,12 +911,12 @@ static int uprobe_event_define_fields(struct ftrace_event_call *event_call)
                 size = SIZEOF_TRACE_ENTRY(false);
         }
         /* Set argument names as fields */
-        for (i = 0; i < tu->nr_args; i++) {
-                ret = trace_define_field(event_call, tu->args[i].type->fmttype,
-                                         tu->args[i].name,
-                                         size + tu->args[i].offset,
-                                         tu->args[i].type->size,
-                                         tu->args[i].type->is_signed,
+        for (i = 0; i < tu->tp.nr_args; i++) {
+                struct probe_arg *parg = &tu->tp.args[i];
+
+                ret = trace_define_field(event_call, parg->type->fmttype,
+                                         parg->name, size + parg->offset,
+                                         parg->type->size, parg->type->is_signed,
                                          FILTER_OTHER);
 
                 if (ret)
@@ -686,59 +925,6 @@ static int uprobe_event_define_fields(struct ftrace_event_call *event_call)
         return 0;
 }
 
-#define LEN_OR_ZERO             (len ? len - pos : 0)
-static int __set_print_fmt(struct trace_uprobe *tu, char *buf, int len)
-{
-        const char *fmt, *arg;
-        int i;
-        int pos = 0;
-
-        if (is_ret_probe(tu)) {
-                fmt = "(%lx <- %lx)";
-                arg = "REC->" FIELD_STRING_FUNC ", REC->" FIELD_STRING_RETIP;
-        } else {
-                fmt = "(%lx)";
-                arg = "REC->" FIELD_STRING_IP;
-        }
-
-        /* When len=0, we just calculate the needed length */
-
-        pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", fmt);
-
-        for (i = 0; i < tu->nr_args; i++) {
-                pos += snprintf(buf + pos, LEN_OR_ZERO, " %s=%s",
-                                tu->args[i].name, tu->args[i].type->fmt);
-        }
-
-        pos += snprintf(buf + pos, LEN_OR_ZERO, "\", %s", arg);
-
-        for (i = 0; i < tu->nr_args; i++) {
-                pos += snprintf(buf + pos, LEN_OR_ZERO, ", REC->%s",
-                                tu->args[i].name);
-        }
-
-        return pos;     /* return the length of print_fmt */
-}
-#undef LEN_OR_ZERO
-
-static int set_print_fmt(struct trace_uprobe *tu)
-{
-        char *print_fmt;
-        int len;
-
-        /* First: called with 0 length to calculate the needed length */
-        len = __set_print_fmt(tu, NULL, 0);
-        print_fmt = kmalloc(len + 1, GFP_KERNEL);
-        if (!print_fmt)
-                return -ENOMEM;
-
-        /* Second: actually write the @print_fmt */
-        __set_print_fmt(tu, print_fmt, len + 1);
-        tu->call.print_fmt = print_fmt;
-
-        return 0;
-}
-
 #ifdef CONFIG_PERF_EVENTS
 static bool
 __uprobe_perf_filter(struct trace_uprobe_filter *filter, struct mm_struct *mm)
@@ -831,14 +1017,27 @@ static bool uprobe_perf_filter(struct uprobe_consumer *uc,
 static void uprobe_perf_print(struct trace_uprobe *tu,
                                 unsigned long func, struct pt_regs *regs)
 {
-        struct ftrace_event_call *call = &tu->call;
+        struct ftrace_event_call *call = &tu->tp.call;
         struct uprobe_trace_entry_head *entry;
         struct hlist_head *head;
+        struct uprobe_cpu_buffer *ucb;
         void *data;
-        int size, rctx, i;
+        int size, dsize, esize;
+        int rctx;
 
-        size = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
-        size = ALIGN(size + tu->size + sizeof(u32), sizeof(u64)) - sizeof(u32);
+        dsize = __get_data_size(&tu->tp, regs);
+        esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
+
+        if (WARN_ON_ONCE(!uprobe_cpu_buffer))
+                return;
+
+        size = esize + tu->tp.size + dsize;
+        size = ALIGN(size + sizeof(u32), sizeof(u64)) - sizeof(u32);
+        if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE, "profile buffer not large enough"))
+                return;
+
+        ucb = uprobe_buffer_get();
+        store_trace_args(esize, &tu->tp, regs, ucb->buf, dsize);
 
         preempt_disable();
         head = this_cpu_ptr(call->perf_events);
@@ -858,12 +1057,18 @@ static void uprobe_perf_print(struct trace_uprobe *tu,
                 data = DATAOF_TRACE_ENTRY(entry, false);
         }
 
-        for (i = 0; i < tu->nr_args; i++)
-                call_fetch(&tu->args[i].fetch, regs, data + tu->args[i].offset);
+        memcpy(data, ucb->buf, tu->tp.size + dsize);
+
+        if (size - esize > tu->tp.size + dsize) {
+                int len = tu->tp.size + dsize;
+
+                memset(data + len, 0, size - esize - len);
+        }
 
         perf_trace_buf_submit(entry, size, rctx, 0, 1, regs, head, NULL);
  out:
         preempt_enable();
+        uprobe_buffer_put(ucb);
 }
 
 /* uprobe profile handler */
@@ -921,16 +1126,22 @@ int trace_uprobe_register(struct ftrace_event_call *event, enum trace_reg type,
 static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs)
 {
         struct trace_uprobe *tu;
+        struct uprobe_dispatch_data udd;
         int ret = 0;
 
         tu = container_of(con, struct trace_uprobe, consumer);
         tu->nhit++;
 
-        if (tu->flags & TP_FLAG_TRACE)
+        udd.tu = tu;
+        udd.bp_addr = instruction_pointer(regs);
+
+        current->utask->vaddr = (unsigned long) &udd;
+
+        if (tu->tp.flags & TP_FLAG_TRACE)
                 ret |= uprobe_trace_func(tu, regs);
 
 #ifdef CONFIG_PERF_EVENTS
-        if (tu->flags & TP_FLAG_PROFILE)
+        if (tu->tp.flags & TP_FLAG_PROFILE)
                 ret |= uprobe_perf_func(tu, regs);
 #endif
         return ret;
@@ -940,14 +1151,20 @@ static int uretprobe_dispatcher(struct uprobe_consumer *con,
                                 unsigned long func, struct pt_regs *regs)
 {
         struct trace_uprobe *tu;
+        struct uprobe_dispatch_data udd;
 
         tu = container_of(con, struct trace_uprobe, consumer);
 
-        if (tu->flags & TP_FLAG_TRACE)
+        udd.tu = tu;
+        udd.bp_addr = func;
+
+        current->utask->vaddr = (unsigned long) &udd;
+
+        if (tu->tp.flags & TP_FLAG_TRACE)
                 uretprobe_trace_func(tu, func, regs);
 
 #ifdef CONFIG_PERF_EVENTS
-        if (tu->flags & TP_FLAG_PROFILE)
+        if (tu->tp.flags & TP_FLAG_PROFILE)
                 uretprobe_perf_func(tu, func, regs);
 #endif
         return 0;
@@ -959,7 +1176,7 @@ static struct trace_event_functions uprobe_funcs = {
 
 static int register_uprobe_event(struct trace_uprobe *tu)
 {
-        struct ftrace_event_call *call = &tu->call;
+        struct ftrace_event_call *call = &tu->tp.call;
         int ret;
 
         /* Initialize ftrace_event_call */
@@ -967,7 +1184,7 @@ static int register_uprobe_event(struct trace_uprobe *tu)
         call->event.funcs = &uprobe_funcs;
         call->class->define_fields = uprobe_event_define_fields;
 
-        if (set_print_fmt(tu) < 0)
+        if (set_print_fmt(&tu->tp, is_ret_probe(tu)) < 0)
                 return -ENOMEM;
 
         ret = register_ftrace_event(&call->event);
@@ -994,11 +1211,11 @@ static int unregister_uprobe_event(struct trace_uprobe *tu)
         int ret;
 
         /* tu->event is unregistered in trace_remove_event_call() */
-        ret = trace_remove_event_call(&tu->call);
+        ret = trace_remove_event_call(&tu->tp.call);
         if (ret)
                 return ret;
-        kfree(tu->call.print_fmt);
-        tu->call.print_fmt = NULL;
+        kfree(tu->tp.call.print_fmt);
+        tu->tp.call.print_fmt = NULL;
         return 0;
 }