1 files changed, 119 insertions, 60 deletions

diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 908038f5744..798b16cd40f 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -32,7 +32,6 @@
 
 #include <trace/events/sched.h>
 
-#include <asm/ftrace.h>
 #include <asm/setup.h>
 
 #include "trace_output.h"
@@ -82,14 +81,14 @@ static int ftrace_disabled __read_mostly;
 
 static DEFINE_MUTEX(ftrace_lock);
 
-static struct ftrace_ops ftrace_list_end __read_mostly =
-{
+static struct ftrace_ops ftrace_list_end __read_mostly = {
         .func           = ftrace_stub,
 };
 
 static struct ftrace_ops *ftrace_global_list __read_mostly = &ftrace_list_end;
 static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end;
 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
+static ftrace_func_t __ftrace_trace_function_delay __read_mostly = ftrace_stub;
 ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
 ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
 static struct ftrace_ops global_ops;
@@ -148,6 +147,7 @@ void clear_ftrace_function(void)
 {
         ftrace_trace_function = ftrace_stub;
         __ftrace_trace_function = ftrace_stub;
+        __ftrace_trace_function_delay = ftrace_stub;
         ftrace_pid_function = ftrace_stub;
 }
 
@@ -210,7 +210,12 @@ static void update_ftrace_function(void)
 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
         ftrace_trace_function = func;
 #else
+#ifdef CONFIG_DYNAMIC_FTRACE
+        /* do not update till all functions have been modified */
+        __ftrace_trace_function_delay = func;
+#else
         __ftrace_trace_function = func;
+#endif
         ftrace_trace_function = ftrace_test_stop_func;
 #endif
 }
@@ -785,8 +790,7 @@ static void unregister_ftrace_profiler(void)
         unregister_ftrace_graph();
 }
 #else
-static struct ftrace_ops ftrace_profile_ops __read_mostly =
-{
+static struct ftrace_ops ftrace_profile_ops __read_mostly = {
         .func           = function_profile_call,
 };
 
@@ -806,19 +810,10 @@ ftrace_profile_write(struct file *filp, const char __user *ubuf,
                      size_t cnt, loff_t *ppos)
 {
         unsigned long val;
-        char buf[64];           /* big enough to hold a number */
         int ret;
 
-        if (cnt >= sizeof(buf))
-                return -EINVAL;
-
-        if (copy_from_user(&buf, ubuf, cnt))
-                return -EFAULT;
-
-        buf[cnt] = 0;
-
-        ret = strict_strtoul(buf, 10, &val);
-        if (ret < 0)
+        ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
+        if (ret)
                 return ret;
 
         val = !!val;
@@ -952,7 +947,7 @@ struct ftrace_func_probe {
 };
 
 enum {
-        FTRACE_ENABLE_CALLS             = (1 << 0),
+        FTRACE_UPDATE_CALLS             = (1 << 0),
         FTRACE_DISABLE_CALLS            = (1 << 1),
         FTRACE_UPDATE_TRACE_FUNC        = (1 << 2),
         FTRACE_START_FUNC_RET           = (1 << 3),
@@ -1182,8 +1177,14 @@ alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
         return NULL;
 }
 
+static void
+ftrace_hash_rec_disable(struct ftrace_ops *ops, int filter_hash);
+static void
+ftrace_hash_rec_enable(struct ftrace_ops *ops, int filter_hash);
+
 static int
-ftrace_hash_move(struct ftrace_hash **dst, struct ftrace_hash *src)
+ftrace_hash_move(struct ftrace_ops *ops, int enable,
+                 struct ftrace_hash **dst, struct ftrace_hash *src)
 {
         struct ftrace_func_entry *entry;
         struct hlist_node *tp, *tn;
@@ -1193,9 +1194,16 @@ ftrace_hash_move(struct ftrace_hash **dst, struct ftrace_hash *src)
         unsigned long key;
         int size = src->count;
         int bits = 0;
+        int ret;
         int i;
 
         /*
+         * Remove the current set, update the hash and add
+         * them back.
+         */
+        ftrace_hash_rec_disable(ops, enable);
+
+        /*
          * If the new source is empty, just free dst and assign it
          * the empty_hash.
          */
@@ -1215,9 +1223,10 @@ ftrace_hash_move(struct ftrace_hash **dst, struct ftrace_hash *src)
         if (bits > FTRACE_HASH_MAX_BITS)
                 bits = FTRACE_HASH_MAX_BITS;
 
+        ret = -ENOMEM;
         new_hash = alloc_ftrace_hash(bits);
         if (!new_hash)
-                return -ENOMEM;
+                goto out;
 
         size = 1 << src->size_bits;
         for (i = 0; i < size; i++) {
@@ -1236,7 +1245,16 @@ ftrace_hash_move(struct ftrace_hash **dst, struct ftrace_hash *src)
         rcu_assign_pointer(*dst, new_hash);
         free_ftrace_hash_rcu(old_hash);
 
-        return 0;
+        ret = 0;
+ out:
+        /*
+         * Enable regardless of ret:
+         *  On success, we enable the new hash.
+         *  On failure, we re-enable the original hash.
+         */
+        ftrace_hash_rec_enable(ops, enable);
+
+        return ret;
 }
 
 /*
@@ -1498,7 +1516,7 @@ int ftrace_text_reserved(void *start, void *end)
 
 
 static int
-__ftrace_replace_code(struct dyn_ftrace *rec, int enable)
+__ftrace_replace_code(struct dyn_ftrace *rec, int update)
 {
         unsigned long ftrace_addr;
         unsigned long flag = 0UL;
@@ -1506,17 +1524,17 @@ __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
         ftrace_addr = (unsigned long)FTRACE_ADDR;
 
         /*
-         * If we are enabling tracing:
+         * If we are updating calls:
          *
          *   If the record has a ref count, then we need to enable it
          *   because someone is using it.
          *
          *   Otherwise we make sure its disabled.
          *
-         * If we are disabling tracing, then disable all records that
+         * If we are disabling calls, then disable all records that
          * are enabled.
          */
-        if (enable && (rec->flags & ~FTRACE_FL_MASK))
+        if (update && (rec->flags & ~FTRACE_FL_MASK))
                 flag = FTRACE_FL_ENABLED;
 
         /* If the state of this record hasn't changed, then do nothing */
@@ -1532,7 +1550,7 @@ __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
         return ftrace_make_nop(NULL, rec, ftrace_addr);
 }
 
-static void ftrace_replace_code(int enable)
+static void ftrace_replace_code(int update)
 {
         struct dyn_ftrace *rec;
         struct ftrace_page *pg;
@@ -1546,7 +1564,7 @@ static void ftrace_replace_code(int enable)
                 if (rec->flags & FTRACE_FL_FREE)
                         continue;
 
-                failed = __ftrace_replace_code(rec, enable);
+                failed = __ftrace_replace_code(rec, update);
                 if (failed) {
                         ftrace_bug(failed, rec->ip);
                         /* Stop processing */
@@ -1596,7 +1614,13 @@ static int __ftrace_modify_code(void *data)
 {
         int *command = data;
 
-        if (*command & FTRACE_ENABLE_CALLS)
+        /*
+         * Do not call function tracer while we update the code.
+         * We are in stop machine, no worrying about races.
+         */
+        function_trace_stop++;
+
+        if (*command & FTRACE_UPDATE_CALLS)
                 ftrace_replace_code(1);
         else if (*command & FTRACE_DISABLE_CALLS)
                 ftrace_replace_code(0);
@@ -1609,6 +1633,18 @@ static int __ftrace_modify_code(void *data)
         else if (*command & FTRACE_STOP_FUNC_RET)
                 ftrace_disable_ftrace_graph_caller();
 
+#ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
+        /*
+         * For archs that call ftrace_test_stop_func(), we must
+         * wait till after we update all the function callers
+         * before we update the callback. This keeps different
+         * ops that record different functions from corrupting
+         * each other.
+         */
+        __ftrace_trace_function = __ftrace_trace_function_delay;
+#endif
+        function_trace_stop--;
+
         return 0;
 }
 
@@ -1652,7 +1688,7 @@ static int ftrace_startup(struct ftrace_ops *ops, int command)
                 return -ENODEV;
 
         ftrace_start_up++;
-        command |= FTRACE_ENABLE_CALLS;
+        command |= FTRACE_UPDATE_CALLS;
 
         /* ops marked global share the filter hashes */
         if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
@@ -1704,8 +1740,7 @@ static void ftrace_shutdown(struct ftrace_ops *ops, int command)
         if (ops != &global_ops || !global_start_up)
                 ops->flags &= ~FTRACE_OPS_FL_ENABLED;
 
-        if (!ftrace_start_up)
-                command |= FTRACE_DISABLE_CALLS;
+        command |= FTRACE_UPDATE_CALLS;
 
         if (saved_ftrace_func != ftrace_trace_function) {
                 saved_ftrace_func = ftrace_trace_function;
@@ -1727,7 +1762,7 @@ static void ftrace_startup_sysctl(void)
         saved_ftrace_func = NULL;
         /* ftrace_start_up is true if we want ftrace running */
         if (ftrace_start_up)
-                ftrace_run_update_code(FTRACE_ENABLE_CALLS);
+                ftrace_run_update_code(FTRACE_UPDATE_CALLS);
 }
 
 static void ftrace_shutdown_sysctl(void)
@@ -1744,10 +1779,36 @@ static cycle_t		ftrace_update_time;
 static unsigned long    ftrace_update_cnt;
 unsigned long           ftrace_update_tot_cnt;
 
+static int ops_traces_mod(struct ftrace_ops *ops)
+{
+        struct ftrace_hash *hash;
+
+        hash = ops->filter_hash;
+        return !!(!hash || !hash->count);
+}
+
 static int ftrace_update_code(struct module *mod)
 {
         struct dyn_ftrace *p;
         cycle_t start, stop;
+        unsigned long ref = 0;
+
+        /*
+         * When adding a module, we need to check if tracers are
+         * currently enabled and if they are set to trace all functions.
+         * If they are, we need to enable the module functions as well
+         * as update the reference counts for those function records.
+         */
+        if (mod) {
+                struct ftrace_ops *ops;
+
+                for (ops = ftrace_ops_list;
+                     ops != &ftrace_list_end; ops = ops->next) {
+                        if (ops->flags & FTRACE_OPS_FL_ENABLED &&
+                            ops_traces_mod(ops))
+                                ref++;
+                }
+        }
 
         start = ftrace_now(raw_smp_processor_id());
         ftrace_update_cnt = 0;
@@ -1760,7 +1821,7 @@ static int ftrace_update_code(struct module *mod)
 
                 p = ftrace_new_addrs;
                 ftrace_new_addrs = p->newlist;
-                p->flags = 0L;
+                p->flags = ref;
 
                 /*
                  * Do the initial record conversion from mcount jump
@@ -1783,7 +1844,7 @@ static int ftrace_update_code(struct module *mod)
                  * conversion puts the module to the correct state, thus
                  * passing the ftrace_make_call check.
                  */
-                if (ftrace_start_up) {
+                if (ftrace_start_up && ref) {
                         int failed = __ftrace_replace_code(p, 1);
                         if (failed) {
                                 ftrace_bug(failed, p->ip);
@@ -2407,10 +2468,9 @@ ftrace_match_module_records(struct ftrace_hash *hash, char *buff, char *mod)
  */
 
 static int
-ftrace_mod_callback(char *func, char *cmd, char *param, int enable)
+ftrace_mod_callback(struct ftrace_hash *hash,
+                    char *func, char *cmd, char *param, int enable)
 {
-        struct ftrace_ops *ops = &global_ops;
-        struct ftrace_hash *hash;
         char *mod;
         int ret = -EINVAL;
 
@@ -2430,11 +2490,6 @@ ftrace_mod_callback(char *func, char *cmd, char *param, int enable)
         if (!strlen(mod))
                 return ret;
 
-        if (enable)
-                hash = ops->filter_hash;
-        else
-                hash = ops->notrace_hash;
-
         ret = ftrace_match_module_records(hash, func, mod);
         if (!ret)
                 ret = -EINVAL;
@@ -2760,7 +2815,7 @@ static int ftrace_process_regex(struct ftrace_hash *hash,
         mutex_lock(&ftrace_cmd_mutex);
         list_for_each_entry(p, &ftrace_commands, list) {
                 if (strcmp(p->name, command) == 0) {
-                        ret = p->func(func, command, next, enable);
+                        ret = p->func(hash, func, command, next, enable);
                         goto out_unlock;
                 }
         }
@@ -2857,7 +2912,11 @@ ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
                 ftrace_match_records(hash, buf, len);
 
         mutex_lock(&ftrace_lock);
-        ret = ftrace_hash_move(orig_hash, hash);
+        ret = ftrace_hash_move(ops, enable, orig_hash, hash);
+        if (!ret && ops->flags & FTRACE_OPS_FL_ENABLED
+            && ftrace_enabled)
+                ftrace_run_update_code(FTRACE_UPDATE_CALLS);
+
         mutex_unlock(&ftrace_lock);
 
         mutex_unlock(&ftrace_regex_lock);
@@ -3040,18 +3099,12 @@ ftrace_regex_release(struct inode *inode, struct file *file)
                         orig_hash = &iter->ops->notrace_hash;
 
                 mutex_lock(&ftrace_lock);
-                /*
-                 * Remove the current set, update the hash and add
-                 * them back.
-                 */
-                ftrace_hash_rec_disable(iter->ops, filter_hash);
-                ret = ftrace_hash_move(orig_hash, iter->hash);
-                if (!ret) {
-                        ftrace_hash_rec_enable(iter->ops, filter_hash);
-                        if (iter->ops->flags & FTRACE_OPS_FL_ENABLED
-                            && ftrace_enabled)
-                                ftrace_run_update_code(FTRACE_ENABLE_CALLS);
-                }
+                ret = ftrace_hash_move(iter->ops, filter_hash,
+                                       orig_hash, iter->hash);
+                if (!ret && (iter->ops->flags & FTRACE_OPS_FL_ENABLED)
+                    && ftrace_enabled)
+                        ftrace_run_update_code(FTRACE_UPDATE_CALLS);
+
                 mutex_unlock(&ftrace_lock);
         }
         free_ftrace_hash(iter->hash);
@@ -3330,7 +3383,7 @@ static int ftrace_process_locs(struct module *mod,
 {
         unsigned long *p;
         unsigned long addr;
-        unsigned long flags;
+        unsigned long flags = 0; /* Shut up gcc */
 
         mutex_lock(&ftrace_lock);
         p = start;
@@ -3348,12 +3401,18 @@ static int ftrace_process_locs(struct module *mod,
         }
 
         /*
-         * Disable interrupts to prevent interrupts from executing
-         * code that is being modified.
+         * We only need to disable interrupts on start up
+         * because we are modifying code that an interrupt
+         * may execute, and the modification is not atomic.
+         * But for modules, nothing runs the code we modify
+         * until we are finished with it, and there's no
+         * reason to cause large interrupt latencies while we do it.
          */
-        local_irq_save(flags);
+        if (!mod)
+                local_irq_save(flags);
         ftrace_update_code(mod);
-        local_irq_restore(flags);
+        if (!mod)
+                local_irq_restore(flags);
         mutex_unlock(&ftrace_lock);
 
         return 0;