1 files changed, 274 insertions, 104 deletions

diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c
index 27d13b36b8b..00d527c945a 100644
--- a/kernel/trace/trace_kprobe.c
+++ b/kernel/trace/trace_kprobe.c
@@ -343,6 +343,14 @@ DEFINE_BASIC_FETCH_FUNCS(deref)
 DEFINE_FETCH_deref(string)
 DEFINE_FETCH_deref(string_size)
 
+static __kprobes void update_deref_fetch_param(struct deref_fetch_param *data)
+{
+        if (CHECK_FETCH_FUNCS(deref, data->orig.fn))
+                update_deref_fetch_param(data->orig.data);
+        else if (CHECK_FETCH_FUNCS(symbol, data->orig.fn))
+                update_symbol_cache(data->orig.data);
+}
+
 static __kprobes void free_deref_fetch_param(struct deref_fetch_param *data)
 {
         if (CHECK_FETCH_FUNCS(deref, data->orig.fn))
@@ -377,6 +385,19 @@ DEFINE_BASIC_FETCH_FUNCS(bitfield)
 #define fetch_bitfield_string_size NULL
 
 static __kprobes void
+update_bitfield_fetch_param(struct bitfield_fetch_param *data)
+{
+        /*
+         * Don't check the bitfield itself, because this must be the
+         * last fetch function.
+         */
+        if (CHECK_FETCH_FUNCS(deref, data->orig.fn))
+                update_deref_fetch_param(data->orig.data);
+        else if (CHECK_FETCH_FUNCS(symbol, data->orig.fn))
+                update_symbol_cache(data->orig.data);
+}
+
+static __kprobes void
 free_bitfield_fetch_param(struct bitfield_fetch_param *data)
 {
         /*
@@ -389,6 +410,7 @@ free_bitfield_fetch_param(struct bitfield_fetch_param *data)
                 free_symbol_cache(data->orig.data);
         kfree(data);
 }
+
 /* Default (unsigned long) fetch type */
 #define __DEFAULT_FETCH_TYPE(t) u##t
 #define _DEFAULT_FETCH_TYPE(t) __DEFAULT_FETCH_TYPE(t)
@@ -536,6 +558,7 @@ struct probe_arg {
 /* Flags for trace_probe */
 #define TP_FLAG_TRACE   1
 #define TP_FLAG_PROFILE 2
+#define TP_FLAG_REGISTERED 4
 
 struct trace_probe {
         struct list_head        list;
@@ -555,16 +578,49 @@ struct trace_probe {
         (sizeof(struct probe_arg) * (n)))
 
 
-static __kprobes int probe_is_return(struct trace_probe *tp)
+static __kprobes int trace_probe_is_return(struct trace_probe *tp)
 {
         return tp->rp.handler != NULL;
 }
 
-static __kprobes const char *probe_symbol(struct trace_probe *tp)
+static __kprobes const char *trace_probe_symbol(struct trace_probe *tp)
 {
         return tp->symbol ? tp->symbol : "unknown";
 }
 
+static __kprobes unsigned long trace_probe_offset(struct trace_probe *tp)
+{
+        return tp->rp.kp.offset;
+}
+
+static __kprobes bool trace_probe_is_enabled(struct trace_probe *tp)
+{
+        return !!(tp->flags & (TP_FLAG_TRACE | TP_FLAG_PROFILE));
+}
+
+static __kprobes bool trace_probe_is_registered(struct trace_probe *tp)
+{
+        return !!(tp->flags & TP_FLAG_REGISTERED);
+}
+
+static __kprobes bool trace_probe_has_gone(struct trace_probe *tp)
+{
+        return !!(kprobe_gone(&tp->rp.kp));
+}
+
+static __kprobes bool trace_probe_within_module(struct trace_probe *tp,
+                                                struct module *mod)
+{
+        int len = strlen(mod->name);
+        const char *name = trace_probe_symbol(tp);
+        return strncmp(mod->name, name, len) == 0 && name[len] == ':';
+}
+
+static __kprobes bool trace_probe_is_on_module(struct trace_probe *tp)
+{
+        return !!strchr(trace_probe_symbol(tp), ':');
+}
+
 static int register_probe_event(struct trace_probe *tp);
 static void unregister_probe_event(struct trace_probe *tp);
 
@@ -646,6 +702,16 @@ error:
         return ERR_PTR(ret);
 }
 
+static void update_probe_arg(struct probe_arg *arg)
+{
+        if (CHECK_FETCH_FUNCS(bitfield, arg->fetch.fn))
+                update_bitfield_fetch_param(arg->fetch.data);
+        else if (CHECK_FETCH_FUNCS(deref, arg->fetch.fn))
+                update_deref_fetch_param(arg->fetch.data);
+        else if (CHECK_FETCH_FUNCS(symbol, arg->fetch.fn))
+                update_symbol_cache(arg->fetch.data);
+}
+
 static void free_probe_arg(struct probe_arg *arg)
 {
         if (CHECK_FETCH_FUNCS(bitfield, arg->fetch.fn))
@@ -671,7 +737,7 @@ static void free_trace_probe(struct trace_probe *tp)
         kfree(tp);
 }
 
-static struct trace_probe *find_probe_event(const char *event,
+static struct trace_probe *find_trace_probe(const char *event,
                                             const char *group)
 {
         struct trace_probe *tp;
@@ -683,15 +749,104 @@ static struct trace_probe *find_probe_event(const char *event,
         return NULL;
 }
 
-/* Unregister a trace_probe and probe_event: call with locking probe_lock */
-static void unregister_trace_probe(struct trace_probe *tp)
+/* Enable trace_probe - @flag must be TP_FLAG_TRACE or TP_FLAG_PROFILE */
+static int enable_trace_probe(struct trace_probe *tp, int flag)
 {
-        if (probe_is_return(tp))
-                unregister_kretprobe(&tp->rp);
+        int ret = 0;
+
+        tp->flags |= flag;
+        if (trace_probe_is_enabled(tp) && trace_probe_is_registered(tp) &&
+            !trace_probe_has_gone(tp)) {
+                if (trace_probe_is_return(tp))
+                        ret = enable_kretprobe(&tp->rp);
+                else
+                        ret = enable_kprobe(&tp->rp.kp);
+        }
+
+        return ret;
+}
+
+/* Disable trace_probe - @flag must be TP_FLAG_TRACE or TP_FLAG_PROFILE */
+static void disable_trace_probe(struct trace_probe *tp, int flag)
+{
+        tp->flags &= ~flag;
+        if (!trace_probe_is_enabled(tp) && trace_probe_is_registered(tp)) {
+                if (trace_probe_is_return(tp))
+                        disable_kretprobe(&tp->rp);
+                else
+                        disable_kprobe(&tp->rp.kp);
+        }
+}
+
+/* Internal register function - just handle k*probes and flags */
+static int __register_trace_probe(struct trace_probe *tp)
+{
+        int i, ret;
+
+        if (trace_probe_is_registered(tp))
+                return -EINVAL;
+
+        for (i = 0; i < tp->nr_args; i++)
+                update_probe_arg(&tp->args[i]);
+
+        /* Set/clear disabled flag according to tp->flag */
+        if (trace_probe_is_enabled(tp))
+                tp->rp.kp.flags &= ~KPROBE_FLAG_DISABLED;
+        else
+                tp->rp.kp.flags |= KPROBE_FLAG_DISABLED;
+
+        if (trace_probe_is_return(tp))
+                ret = register_kretprobe(&tp->rp);
         else
-                unregister_kprobe(&tp->rp.kp);
+                ret = register_kprobe(&tp->rp.kp);
+
+        if (ret == 0)
+                tp->flags |= TP_FLAG_REGISTERED;
+        else {
+                pr_warning("Could not insert probe at %s+%lu: %d\n",
+                           trace_probe_symbol(tp), trace_probe_offset(tp), ret);
+                if (ret == -ENOENT && trace_probe_is_on_module(tp)) {
+                        pr_warning("This probe might be able to register after"
+                                   "target module is loaded. Continue.\n");
+                        ret = 0;
+                } else if (ret == -EILSEQ) {
+                        pr_warning("Probing address(0x%p) is not an "
+                                   "instruction boundary.\n",
+                                   tp->rp.kp.addr);
+                        ret = -EINVAL;
+                }
+        }
+
+        return ret;
+}
+
+/* Internal unregister function - just handle k*probes and flags */
+static void __unregister_trace_probe(struct trace_probe *tp)
+{
+        if (trace_probe_is_registered(tp)) {
+                if (trace_probe_is_return(tp))
+                        unregister_kretprobe(&tp->rp);
+                else
+                        unregister_kprobe(&tp->rp.kp);
+                tp->flags &= ~TP_FLAG_REGISTERED;
+                /* Cleanup kprobe for reuse */
+                if (tp->rp.kp.symbol_name)
+                        tp->rp.kp.addr = NULL;
+        }
+}
+
+/* Unregister a trace_probe and probe_event: call with locking probe_lock */
+static int unregister_trace_probe(struct trace_probe *tp)
+{
+        /* Enabled event can not be unregistered */
+        if (trace_probe_is_enabled(tp))
+                return -EBUSY;
+
+        __unregister_trace_probe(tp);
         list_del(&tp->list);
         unregister_probe_event(tp);
+
+        return 0;
 }
 
 /* Register a trace_probe and probe_event */
@@ -702,41 +857,68 @@ static int register_trace_probe(struct trace_probe *tp)
 
         mutex_lock(&probe_lock);
 
-        /* register as an event */
-        old_tp = find_probe_event(tp->call.name, tp->call.class->system);
+        /* Delete old (same name) event if exist */
+        old_tp = find_trace_probe(tp->call.name, tp->call.class->system);
         if (old_tp) {
-                /* delete old event */
-                unregister_trace_probe(old_tp);
+                ret = unregister_trace_probe(old_tp);
+                if (ret < 0)
+                        goto end;
                 free_trace_probe(old_tp);
         }
+
+        /* Register new event */
         ret = register_probe_event(tp);
         if (ret) {
                 pr_warning("Failed to register probe event(%d)\n", ret);
                 goto end;
         }
 
-        tp->rp.kp.flags |= KPROBE_FLAG_DISABLED;
-        if (probe_is_return(tp))
-                ret = register_kretprobe(&tp->rp);
-        else
-                ret = register_kprobe(&tp->rp.kp);
-
-        if (ret) {
-                pr_warning("Could not insert probe(%d)\n", ret);
-                if (ret == -EILSEQ) {
-                        pr_warning("Probing address(0x%p) is not an "
-                                   "instruction boundary.\n",
-                                   tp->rp.kp.addr);
-                        ret = -EINVAL;
-                }
+        /* Register k*probe */
+        ret = __register_trace_probe(tp);
+        if (ret < 0)
                 unregister_probe_event(tp);
-        } else
+        else
                 list_add_tail(&tp->list, &probe_list);
+
 end:
         mutex_unlock(&probe_lock);
         return ret;
 }
 
+/* Module notifier call back, checking event on the module */
+static int trace_probe_module_callback(struct notifier_block *nb,
+                                       unsigned long val, void *data)
+{
+        struct module *mod = data;
+        struct trace_probe *tp;
+        int ret;
+
+        if (val != MODULE_STATE_COMING)
+                return NOTIFY_DONE;
+
+        /* Update probes on coming module */
+        mutex_lock(&probe_lock);
+        list_for_each_entry(tp, &probe_list, list) {
+                if (trace_probe_within_module(tp, mod)) {
+                        /* Don't need to check busy - this should have gone. */
+                        __unregister_trace_probe(tp);
+                        ret = __register_trace_probe(tp);
+                        if (ret)
+                                pr_warning("Failed to re-register probe %s on"
+                                           "%s: %d\n",
+                                           tp->call.name, mod->name, ret);
+                }
+        }
+        mutex_unlock(&probe_lock);
+
+        return NOTIFY_DONE;
+}
+
+static struct notifier_block trace_probe_module_nb = {
+        .notifier_call = trace_probe_module_callback,
+        .priority = 1   /* Invoked after kprobe module callback */
+};
+
 /* Split symbol and offset. */
 static int split_symbol_offset(char *symbol, unsigned long *offset)
 {
@@ -962,8 +1144,8 @@ static int create_trace_probe(int argc, char **argv)
 {
         /*
          * Argument syntax:
-         *  - Add kprobe: p[:[GRP/]EVENT] KSYM[+OFFS]|KADDR [FETCHARGS]
-         *  - Add kretprobe: r[:[GRP/]EVENT] KSYM[+0] [FETCHARGS]
+         *  - Add kprobe: p[:[GRP/]EVENT] [MOD:]KSYM[+OFFS]|KADDR [FETCHARGS]
+         *  - Add kretprobe: r[:[GRP/]EVENT] [MOD:]KSYM[+0] [FETCHARGS]
          * Fetch args:
          *  $retval     : fetch return value
          *  $stack      : fetch stack address
@@ -1025,17 +1207,18 @@ static int create_trace_probe(int argc, char **argv)
                         return -EINVAL;
                 }
                 mutex_lock(&probe_lock);
-                tp = find_probe_event(event, group);
+                tp = find_trace_probe(event, group);
                 if (!tp) {
                         mutex_unlock(&probe_lock);
                         pr_info("Event %s/%s doesn't exist.\n", group, event);
                         return -ENOENT;
                 }
                 /* delete an event */
-                unregister_trace_probe(tp);
-                free_trace_probe(tp);
+                ret = unregister_trace_probe(tp);
+                if (ret == 0)
+                        free_trace_probe(tp);
                 mutex_unlock(&probe_lock);
-                return 0;
+                return ret;
         }
 
         if (argc < 2) {
@@ -1144,20 +1327,30 @@ error:
         return ret;
 }
 
-static void cleanup_all_probes(void)
+static int release_all_trace_probes(void)
 {
         struct trace_probe *tp;
+        int ret = 0;
 
         mutex_lock(&probe_lock);
+        /* Ensure no probe is in use. */
+        list_for_each_entry(tp, &probe_list, list)
+                if (trace_probe_is_enabled(tp)) {
+                        ret = -EBUSY;
+                        goto end;
+                }
         /* TODO: Use batch unregistration */
         while (!list_empty(&probe_list)) {
                 tp = list_entry(probe_list.next, struct trace_probe, list);
                 unregister_trace_probe(tp);
                 free_trace_probe(tp);
         }
+
+end:
         mutex_unlock(&probe_lock);
-}
 
+        return ret;
+}
 
 /* Probes listing interfaces */
 static void *probes_seq_start(struct seq_file *m, loff_t *pos)
@@ -1181,15 +1374,16 @@ static int probes_seq_show(struct seq_file *m, void *v)
         struct trace_probe *tp = v;
         int i;
 
-        seq_printf(m, "%c", probe_is_return(tp) ? 'r' : 'p');
+        seq_printf(m, "%c", trace_probe_is_return(tp) ? 'r' : 'p');
         seq_printf(m, ":%s/%s", tp->call.class->system, tp->call.name);
 
         if (!tp->symbol)
                 seq_printf(m, " 0x%p", tp->rp.kp.addr);
         else if (tp->rp.kp.offset)
-                seq_printf(m, " %s+%u", probe_symbol(tp), tp->rp.kp.offset);
+                seq_printf(m, " %s+%u", trace_probe_symbol(tp),
+                           tp->rp.kp.offset);
         else
-                seq_printf(m, " %s", probe_symbol(tp));
+                seq_printf(m, " %s", trace_probe_symbol(tp));
 
         for (i = 0; i < tp->nr_args; i++)
                 seq_printf(m, " %s=%s", tp->args[i].name, tp->args[i].comm);
@@ -1207,9 +1401,13 @@ static const struct seq_operations probes_seq_op = {
 
 static int probes_open(struct inode *inode, struct file *file)
 {
-        if ((file->f_mode & FMODE_WRITE) &&
-            (file->f_flags & O_TRUNC))
-                cleanup_all_probes();
+        int ret;
+
+        if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
+                ret = release_all_trace_probes();
+                if (ret < 0)
+                        return ret;
+        }
 
         return seq_open(file, &probes_seq_op);
 }
@@ -1397,7 +1595,8 @@ static __kprobes void kprobe_trace_func(struct kprobe *kp, struct pt_regs *regs)
         store_trace_args(sizeof(*entry), tp, regs, (u8 *)&entry[1], dsize);
 
         if (!filter_current_check_discard(buffer, call, entry, event))
-                trace_nowake_buffer_unlock_commit(buffer, event, irq_flags, pc);
+                trace_nowake_buffer_unlock_commit_regs(buffer, event,
+                                                       irq_flags, pc, regs);
 }
 
 /* Kretprobe handler */
@@ -1429,7 +1628,8 @@ static __kprobes void kretprobe_trace_func(struct kretprobe_instance *ri,
         store_trace_args(sizeof(*entry), tp, regs, (u8 *)&entry[1], dsize);
 
         if (!filter_current_check_discard(buffer, call, entry, event))
-                trace_nowake_buffer_unlock_commit(buffer, event, irq_flags, pc);
+                trace_nowake_buffer_unlock_commit_regs(buffer, event,
+                                                       irq_flags, pc, regs);
 }
 
 /* Event entry printers */
@@ -1511,30 +1711,6 @@ partial:
         return TRACE_TYPE_PARTIAL_LINE;
 }
 
-static int probe_event_enable(struct ftrace_event_call *call)
-{
-        struct trace_probe *tp = (struct trace_probe *)call->data;
-
-        tp->flags |= TP_FLAG_TRACE;
-        if (probe_is_return(tp))
-                return enable_kretprobe(&tp->rp);
-        else
-                return enable_kprobe(&tp->rp.kp);
-}
-
-static void probe_event_disable(struct ftrace_event_call *call)
-{
-        struct trace_probe *tp = (struct trace_probe *)call->data;
-
-        tp->flags &= ~TP_FLAG_TRACE;
-        if (!(tp->flags & (TP_FLAG_TRACE | TP_FLAG_PROFILE))) {
-                if (probe_is_return(tp))
-                        disable_kretprobe(&tp->rp);
-                else
-                        disable_kprobe(&tp->rp.kp);
-        }
-}
-
 #undef DEFINE_FIELD
 #define DEFINE_FIELD(type, item, name, is_signed)                       \
         do {                                                            \
@@ -1596,7 +1772,7 @@ static int __set_print_fmt(struct trace_probe *tp, char *buf, int len)
 
         const char *fmt, *arg;
 
-        if (!probe_is_return(tp)) {
+        if (!trace_probe_is_return(tp)) {
                 fmt = "(%lx)";
                 arg = "REC->" FIELD_STRING_IP;
         } else {
@@ -1713,49 +1889,25 @@ static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri,
         head = this_cpu_ptr(call->perf_events);
         perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1, regs, head);
 }
-
-static int probe_perf_enable(struct ftrace_event_call *call)
-{
-        struct trace_probe *tp = (struct trace_probe *)call->data;
-
-        tp->flags |= TP_FLAG_PROFILE;
-
-        if (probe_is_return(tp))
-                return enable_kretprobe(&tp->rp);
-        else
-                return enable_kprobe(&tp->rp.kp);
-}
-
-static void probe_perf_disable(struct ftrace_event_call *call)
-{
-        struct trace_probe *tp = (struct trace_probe *)call->data;
-
-        tp->flags &= ~TP_FLAG_PROFILE;
-
-        if (!(tp->flags & TP_FLAG_TRACE)) {
-                if (probe_is_return(tp))
-                        disable_kretprobe(&tp->rp);
-                else
-                        disable_kprobe(&tp->rp.kp);
-        }
-}
 #endif  /* CONFIG_PERF_EVENTS */
 
 static __kprobes
 int kprobe_register(struct ftrace_event_call *event, enum trace_reg type)
 {
+        struct trace_probe *tp = (struct trace_probe *)event->data;
+
         switch (type) {
         case TRACE_REG_REGISTER:
-                return probe_event_enable(event);
+                return enable_trace_probe(tp, TP_FLAG_TRACE);
         case TRACE_REG_UNREGISTER:
-                probe_event_disable(event);
+                disable_trace_probe(tp, TP_FLAG_TRACE);
                 return 0;
 
 #ifdef CONFIG_PERF_EVENTS
         case TRACE_REG_PERF_REGISTER:
-                return probe_perf_enable(event);
+                return enable_trace_probe(tp, TP_FLAG_PROFILE);
         case TRACE_REG_PERF_UNREGISTER:
-                probe_perf_disable(event);
+                disable_trace_probe(tp, TP_FLAG_PROFILE);
                 return 0;
 #endif
         }
@@ -1805,7 +1957,7 @@ static int register_probe_event(struct trace_probe *tp)
 
         /* Initialize ftrace_event_call */
         INIT_LIST_HEAD(&call->class->fields);
-        if (probe_is_return(tp)) {
+        if (trace_probe_is_return(tp)) {
                 call->event.funcs = &kretprobe_funcs;
                 call->class->define_fields = kretprobe_event_define_fields;
         } else {
@@ -1844,6 +1996,9 @@ static __init int init_kprobe_trace(void)
         struct dentry *d_tracer;
         struct dentry *entry;
 
+        if (register_module_notifier(&trace_probe_module_nb))
+                return -EINVAL;
+
         d_tracer = tracing_init_dentry();
         if (!d_tracer)
                 return 0;
@@ -1897,12 +2052,12 @@ static __init int kprobe_trace_self_tests_init(void)
                 warn++;
         } else {
                 /* Enable trace point */
-                tp = find_probe_event("testprobe", KPROBE_EVENT_SYSTEM);
+                tp = find_trace_probe("testprobe", KPROBE_EVENT_SYSTEM);
                 if (WARN_ON_ONCE(tp == NULL)) {
                         pr_warning("error on getting new probe.\n");
                         warn++;
                 } else
-                        probe_event_enable(&tp->call);
+                        enable_trace_probe(tp, TP_FLAG_TRACE);
         }
 
         ret = command_trace_probe("r:testprobe2 kprobe_trace_selftest_target "
@@ -1912,12 +2067,12 @@ static __init int kprobe_trace_self_tests_init(void)
                 warn++;
         } else {
                 /* Enable trace point */
-                tp = find_probe_event("testprobe2", KPROBE_EVENT_SYSTEM);
+                tp = find_trace_probe("testprobe2", KPROBE_EVENT_SYSTEM);
                 if (WARN_ON_ONCE(tp == NULL)) {
                         pr_warning("error on getting new probe.\n");
                         warn++;
                 } else
-                        probe_event_enable(&tp->call);
+                        enable_trace_probe(tp, TP_FLAG_TRACE);
         }
 
         if (warn)
@@ -1925,6 +2080,21 @@ static __init int kprobe_trace_self_tests_init(void)
 
         ret = target(1, 2, 3, 4, 5, 6);
 
+        /* Disable trace points before removing it */
+        tp = find_trace_probe("testprobe", KPROBE_EVENT_SYSTEM);
+        if (WARN_ON_ONCE(tp == NULL)) {
+                pr_warning("error on getting test probe.\n");
+                warn++;
+        } else
+                disable_trace_probe(tp, TP_FLAG_TRACE);
+
+        tp = find_trace_probe("testprobe2", KPROBE_EVENT_SYSTEM);
+        if (WARN_ON_ONCE(tp == NULL)) {
+                pr_warning("error on getting 2nd test probe.\n");
+                warn++;
+        } else
+                disable_trace_probe(tp, TP_FLAG_TRACE);
+
         ret = command_trace_probe("-:testprobe");
         if (WARN_ON_ONCE(ret)) {
                 pr_warning("error on deleting a probe.\n");
@@ -1938,7 +2108,7 @@ static __init int kprobe_trace_self_tests_init(void)
         }
 
 end:
-        cleanup_all_probes();
+        release_all_trace_probes();
         if (warn)
                 pr_cont("NG: Some tests are failed. Please check them.\n");
         else