1 files changed, 89 insertions, 141 deletions

diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c
index 9f46e98ba8f2..243f6834d026 100644
--- a/kernel/trace/trace_kprobe.c
+++ b/kernel/trace/trace_kprobe.c
@@ -35,12 +35,17 @@ struct trace_probe {
         const char              *symbol;        /* symbol name */
         struct ftrace_event_class       class;
         struct ftrace_event_call        call;
-        struct ftrace_event_file * __rcu *files;
+        struct list_head        files;
         ssize_t                 size;           /* trace entry size */
         unsigned int            nr_args;
         struct probe_arg        args[];
 };
 
+struct event_file_link {
+        struct ftrace_event_file        *file;
+        struct list_head                list;
+};
+
 #define SIZEOF_TRACE_PROBE(n)                   \
         (offsetof(struct trace_probe, args) +   \
         (sizeof(struct probe_arg) * (n)))
@@ -90,7 +95,7 @@ static __kprobes bool trace_probe_is_on_module(struct trace_probe *tp)
 }
 
 static int register_probe_event(struct trace_probe *tp);
-static void unregister_probe_event(struct trace_probe *tp);
+static int unregister_probe_event(struct trace_probe *tp);
 
 static DEFINE_MUTEX(probe_lock);
 static LIST_HEAD(probe_list);
@@ -150,6 +155,7 @@ static struct trace_probe *alloc_trace_probe(const char *group,
                 goto error;
 
         INIT_LIST_HEAD(&tp->list);
+        INIT_LIST_HEAD(&tp->files);
         return tp;
 error:
         kfree(tp->call.name);
@@ -183,25 +189,6 @@ static struct trace_probe *find_trace_probe(const char *event,
         return NULL;
 }
 
-static int trace_probe_nr_files(struct trace_probe *tp)
-{
-        struct ftrace_event_file **file;
-        int ret = 0;
-
-        /*
-         * Since all tp->files updater is protected by probe_enable_lock,
-         * we don't need to lock an rcu_read_lock.
-         */
-        file = rcu_dereference_raw(tp->files);
-        if (file)
-                while (*(file++))
-                        ret++;
-
-        return ret;
-}
-
-static DEFINE_MUTEX(probe_enable_lock);
-
 /*
  * Enable trace_probe
  * if the file is NULL, enable "perf" handler, or enable "trace" handler.
@@ -211,67 +198,42 @@ enable_trace_probe(struct trace_probe *tp, struct ftrace_event_file *file)
 {
         int ret = 0;
 
-        mutex_lock(&probe_enable_lock);
-
         if (file) {
-                struct ftrace_event_file **new, **old;
-                int n = trace_probe_nr_files(tp);
-
-                old = rcu_dereference_raw(tp->files);
-                /* 1 is for new one and 1 is for stopper */
-                new = kzalloc((n + 2) * sizeof(struct ftrace_event_file *),
-                              GFP_KERNEL);
-                if (!new) {
+                struct event_file_link *link;
+
+                link = kmalloc(sizeof(*link), GFP_KERNEL);
+                if (!link) {
                         ret = -ENOMEM;
-                        goto out_unlock;
+                        goto out;
                 }
-                memcpy(new, old, n * sizeof(struct ftrace_event_file *));
-                new[n] = file;
-                /* The last one keeps a NULL */
 
-                rcu_assign_pointer(tp->files, new);
-                tp->flags |= TP_FLAG_TRACE;
+                link->file = file;
+                list_add_tail_rcu(&link->list, &tp->files);
 
-                if (old) {
-                        /* Make sure the probe is done with old files */
-                        synchronize_sched();
-                        kfree(old);
-                }
+                tp->flags |= TP_FLAG_TRACE;
         } else
                 tp->flags |= TP_FLAG_PROFILE;
 
-        if (trace_probe_is_enabled(tp) && trace_probe_is_registered(tp) &&
-            !trace_probe_has_gone(tp)) {
+        if (trace_probe_is_registered(tp) && !trace_probe_has_gone(tp)) {
                 if (trace_probe_is_return(tp))
                         ret = enable_kretprobe(&tp->rp);
                 else
                         ret = enable_kprobe(&tp->rp.kp);
         }
-
- out_unlock:
-        mutex_unlock(&probe_enable_lock);
-
+ out:
         return ret;
 }
 
-static int
-trace_probe_file_index(struct trace_probe *tp, struct ftrace_event_file *file)
+static struct event_file_link *
+find_event_file_link(struct trace_probe *tp, struct ftrace_event_file *file)
 {
-        struct ftrace_event_file **files;
-        int i;
+        struct event_file_link *link;
 
-        /*
-         * Since all tp->files updater is protected by probe_enable_lock,
-         * we don't need to lock an rcu_read_lock.
-         */
-        files = rcu_dereference_raw(tp->files);
-        if (files) {
-                for (i = 0; files[i]; i++)
-                        if (files[i] == file)
-                                return i;
-        }
+        list_for_each_entry(link, &tp->files, list)
+                if (link->file == file)
+                        return link;
 
-        return -1;
+        return NULL;
 }
 
 /*
@@ -281,43 +243,23 @@ trace_probe_file_index(struct trace_probe *tp, struct ftrace_event_file *file)
 static int
 disable_trace_probe(struct trace_probe *tp, struct ftrace_event_file *file)
 {
+        struct event_file_link *link = NULL;
+        int wait = 0;
         int ret = 0;
 
-        mutex_lock(&probe_enable_lock);
-
         if (file) {
-                struct ftrace_event_file **new, **old;
-                int n = trace_probe_nr_files(tp);
-                int i, j;
-
-                old = rcu_dereference_raw(tp->files);
-                if (n == 0 || trace_probe_file_index(tp, file) < 0) {
+                link = find_event_file_link(tp, file);
+                if (!link) {
                         ret = -EINVAL;
-                        goto out_unlock;
+                        goto out;
                 }
 
-                if (n == 1) {   /* Remove the last file */
-                        tp->flags &= ~TP_FLAG_TRACE;
-                        new = NULL;
-                } else {
-                        new = kzalloc(n * sizeof(struct ftrace_event_file *),
-                                      GFP_KERNEL);
-                        if (!new) {
-                                ret = -ENOMEM;
-                                goto out_unlock;
-                        }
-
-                        /* This copy & check loop copies the NULL stopper too */
-                        for (i = 0, j = 0; j < n && i < n + 1; i++)
-                                if (old[i] != file)
-                                        new[j++] = old[i];
-                }
-
-                rcu_assign_pointer(tp->files, new);
+                list_del_rcu(&link->list);
+                wait = 1;
+                if (!list_empty(&tp->files))
+                        goto out;
 
-                /* Make sure the probe is done with old files */
-                synchronize_sched();
-                kfree(old);
+                tp->flags &= ~TP_FLAG_TRACE;
         } else
                 tp->flags &= ~TP_FLAG_PROFILE;
 
@@ -326,10 +268,21 @@ disable_trace_probe(struct trace_probe *tp, struct ftrace_event_file *file)
                         disable_kretprobe(&tp->rp);
                 else
                         disable_kprobe(&tp->rp.kp);
+                wait = 1;
+        }
+ out:
+        if (wait) {
+                /*
+                 * Synchronize with kprobe_trace_func/kretprobe_trace_func
+                 * to ensure disabled (all running handlers are finished).
+                 * This is not only for kfree(), but also the caller,
+                 * trace_remove_event_call() supposes it for releasing
+                 * event_call related objects, which will be accessed in
+                 * the kprobe_trace_func/kretprobe_trace_func.
+                 */
+                synchronize_sched();
+                kfree(link);    /* Ignored if link == NULL */
         }
-
- out_unlock:
-        mutex_unlock(&probe_enable_lock);
 
         return ret;
 }
@@ -398,9 +351,12 @@ static int unregister_trace_probe(struct trace_probe *tp)
         if (trace_probe_is_enabled(tp))
                 return -EBUSY;
 
+        /* Will fail if probe is being used by ftrace or perf */
+        if (unregister_probe_event(tp))
+                return -EBUSY;
+
         __unregister_trace_probe(tp);
         list_del(&tp->list);
-        unregister_probe_event(tp);
 
         return 0;
 }
@@ -679,7 +635,9 @@ static int release_all_trace_probes(void)
         /* TODO: Use batch unregistration */
         while (!list_empty(&probe_list)) {
                 tp = list_entry(probe_list.next, struct trace_probe, list);
-                unregister_trace_probe(tp);
+                ret = unregister_trace_probe(tp);
+                if (ret)
+                        goto end;
                 free_trace_probe(tp);
         }
 
@@ -885,20 +843,10 @@ __kprobe_trace_func(struct trace_probe *tp, struct pt_regs *regs,
 static __kprobes void
 kprobe_trace_func(struct trace_probe *tp, struct pt_regs *regs)
 {
-        /*
-         * Note: preempt is already disabled around the kprobe handler.
-         * However, we still need an smp_read_barrier_depends() corresponding
-         * to smp_wmb() in rcu_assign_pointer() to access the pointer.
-         */
-        struct ftrace_event_file **file = rcu_dereference_raw(tp->files);
-
-        if (unlikely(!file))
-                return;
+        struct event_file_link *link;
 
-        while (*file) {
-                __kprobe_trace_func(tp, regs, *file);
-                file++;
-        }
+        list_for_each_entry_rcu(link, &tp->files, list)
+                __kprobe_trace_func(tp, regs, link->file);
 }
 
 /* Kretprobe handler */
@@ -945,20 +893,10 @@ static __kprobes void
 kretprobe_trace_func(struct trace_probe *tp, struct kretprobe_instance *ri,
                      struct pt_regs *regs)
 {
-        /*
-         * Note: preempt is already disabled around the kprobe handler.
-         * However, we still need an smp_read_barrier_depends() corresponding
-         * to smp_wmb() in rcu_assign_pointer() to access the pointer.
-         */
-        struct ftrace_event_file **file = rcu_dereference_raw(tp->files);
-
-        if (unlikely(!file))
-                return;
+        struct event_file_link *link;
 
-        while (*file) {
-                __kretprobe_trace_func(tp, ri, regs, *file);
-                file++;
-        }
+        list_for_each_entry_rcu(link, &tp->files, list)
+                __kretprobe_trace_func(tp, ri, regs, link->file);
 }
 
 /* Event entry printers */
@@ -1157,13 +1095,14 @@ kprobe_perf_func(struct trace_probe *tp, struct pt_regs *regs)
         int size, __size, dsize;
         int rctx;
 
+        head = this_cpu_ptr(call->perf_events);
+        if (hlist_empty(head))
+                return;
+
         dsize = __get_data_size(tp, regs);
         __size = sizeof(*entry) + tp->size + dsize;
         size = ALIGN(__size + sizeof(u32), sizeof(u64));
         size -= sizeof(u32);
-        if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
-                     "profile buffer not large enough"))
-                return;
 
         entry = perf_trace_buf_prepare(size, call->event.type, regs, &rctx);
         if (!entry)
@@ -1172,10 +1111,7 @@ kprobe_perf_func(struct trace_probe *tp, struct pt_regs *regs)
         entry->ip = (unsigned long)tp->rp.kp.addr;
         memset(&entry[1], 0, dsize);
         store_trace_args(sizeof(*entry), tp, regs, (u8 *)&entry[1], dsize);
-
-        head = this_cpu_ptr(call->perf_events);
-        perf_trace_buf_submit(entry, size, rctx,
-                                        entry->ip, 1, regs, head, NULL);
+        perf_trace_buf_submit(entry, size, rctx, 0, 1, regs, head, NULL);
 }
 
 /* Kretprobe profile handler */
@@ -1189,13 +1125,14 @@ kretprobe_perf_func(struct trace_probe *tp, struct kretprobe_instance *ri,
         int size, __size, dsize;
         int rctx;
 
+        head = this_cpu_ptr(call->perf_events);
+        if (hlist_empty(head))
+                return;
+
         dsize = __get_data_size(tp, regs);
         __size = sizeof(*entry) + tp->size + dsize;
         size = ALIGN(__size + sizeof(u32), sizeof(u64));
         size -= sizeof(u32);
-        if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
-                     "profile buffer not large enough"))
-                return;
 
         entry = perf_trace_buf_prepare(size, call->event.type, regs, &rctx);
         if (!entry)
@@ -1204,13 +1141,16 @@ kretprobe_perf_func(struct trace_probe *tp, struct kretprobe_instance *ri,
         entry->func = (unsigned long)tp->rp.kp.addr;
         entry->ret_ip = (unsigned long)ri->ret_addr;
         store_trace_args(sizeof(*entry), tp, regs, (u8 *)&entry[1], dsize);
-
-        head = this_cpu_ptr(call->perf_events);
-        perf_trace_buf_submit(entry, size, rctx,
-                                        entry->ret_ip, 1, regs, head, NULL);
+        perf_trace_buf_submit(entry, size, rctx, 0, 1, regs, head, NULL);
 }
 #endif  /* CONFIG_PERF_EVENTS */
 
+/*
+ * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex.
+ *
+ * kprobe_trace_self_tests_init() does enable_trace_probe/disable_trace_probe
+ * lockless, but we can't race with this __init function.
+ */
 static __kprobes
 int kprobe_register(struct ftrace_event_call *event,
                     enum trace_reg type, void *data)
@@ -1312,11 +1252,15 @@ static int register_probe_event(struct trace_probe *tp)
         return ret;
 }
 
-static void unregister_probe_event(struct trace_probe *tp)
+static int unregister_probe_event(struct trace_probe *tp)
 {
+        int ret;
+
         /* tp->event is unregistered in trace_remove_event_call() */
-        trace_remove_event_call(&tp->call);
-        kfree(tp->call.print_fmt);
+        ret = trace_remove_event_call(&tp->call);
+        if (!ret)
+                kfree(tp->call.print_fmt);
+        return ret;
 }
 
 /* Make a debugfs interface for controlling probe points */
@@ -1376,6 +1320,10 @@ find_trace_probe_file(struct trace_probe *tp, struct trace_array *tr)
         return NULL;
 }
 
+/*
+ * Nobody but us can call enable_trace_probe/disable_trace_probe at this
+ * stage, we can do this lockless.
+ */
 static __init int kprobe_trace_self_tests_init(void)
 {
         int ret, warn = 0;