Merge Linus' tree to be be to apply submitted patches to newer code than

current trivial.git base

8 files changed, 540 insertions, 336 deletions

diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 1654b12c891a..31c90fec4158 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -65,15 +65,21 @@
 #define FL_GLOBAL_CONTROL_MASK (FTRACE_OPS_FL_CONTROL)
 
 #ifdef CONFIG_DYNAMIC_FTRACE
-#define INIT_REGEX_LOCK(opsname)        \
-        .regex_lock     = __MUTEX_INITIALIZER(opsname.regex_lock),
+#define INIT_OPS_HASH(opsname)  \
+        .func_hash              = &opsname.local_hash,                  \
+        .local_hash.regex_lock  = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
+#define ASSIGN_OPS_HASH(opsname, val) \
+        .func_hash              = val, \
+        .local_hash.regex_lock  = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
 #else
-#define INIT_REGEX_LOCK(opsname)
+#define INIT_OPS_HASH(opsname)
+#define ASSIGN_OPS_HASH(opsname, val)
 #endif
 
 static struct ftrace_ops ftrace_list_end __read_mostly = {
         .func           = ftrace_stub,
         .flags          = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_STUB,
+        INIT_OPS_HASH(ftrace_list_end)
 };
 
 /* ftrace_enabled is a method to turn ftrace on or off */
@@ -107,6 +113,9 @@ ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
 static struct ftrace_ops global_ops;
 static struct ftrace_ops control_ops;
 
+static void ftrace_ops_recurs_func(unsigned long ip, unsigned long parent_ip,
+                                   struct ftrace_ops *op, struct pt_regs *regs);
+
 #if ARCH_SUPPORTS_FTRACE_OPS
 static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
                                  struct ftrace_ops *op, struct pt_regs *regs);
@@ -140,7 +149,8 @@ static inline void ftrace_ops_init(struct ftrace_ops *ops)
 {
 #ifdef CONFIG_DYNAMIC_FTRACE
         if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) {
-                mutex_init(&ops->regex_lock);
+                mutex_init(&ops->local_hash.regex_lock);
+                ops->func_hash = &ops->local_hash;
                 ops->flags |= FTRACE_OPS_FL_INITIALIZED;
         }
 #endif
@@ -244,18 +254,24 @@ static void update_ftrace_function(void)
         ftrace_func_t func;
 
         /*
+         * Prepare the ftrace_ops that the arch callback will use.
+         * If there's only one ftrace_ops registered, the ftrace_ops_list
+         * will point to the ops we want.
+         */
+        set_function_trace_op = ftrace_ops_list;
+
+        /* If there's no ftrace_ops registered, just call the stub function */
+        if (ftrace_ops_list == &ftrace_list_end) {
+                func = ftrace_stub;
+
+        /*
          * If we are at the end of the list and this ops is
          * recursion safe and not dynamic and the arch supports passing ops,
          * then have the mcount trampoline call the function directly.
          */
-        if (ftrace_ops_list == &ftrace_list_end ||
-            (ftrace_ops_list->next == &ftrace_list_end &&
-             !(ftrace_ops_list->flags & FTRACE_OPS_FL_DYNAMIC) &&
-             (ftrace_ops_list->flags & FTRACE_OPS_FL_RECURSION_SAFE) &&
-             !FTRACE_FORCE_LIST_FUNC)) {
-                /* Set the ftrace_ops that the arch callback uses */
-                set_function_trace_op = ftrace_ops_list;
-                func = ftrace_ops_list->func;
+        } else if (ftrace_ops_list->next == &ftrace_list_end) {
+                func = ftrace_ops_get_func(ftrace_ops_list);
+
         } else {
                 /* Just use the default ftrace_ops */
                 set_function_trace_op = &ftrace_list_end;
@@ -899,7 +915,7 @@ static void unregister_ftrace_profiler(void)
 static struct ftrace_ops ftrace_profile_ops __read_mostly = {
         .func           = function_profile_call,
         .flags          = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
-        INIT_REGEX_LOCK(ftrace_profile_ops)
+        INIT_OPS_HASH(ftrace_profile_ops)
 };
 
 static int register_ftrace_profiler(void)
@@ -1041,6 +1057,12 @@ static struct pid * const ftrace_swapper_pid = &init_struct_pid;
 
 static struct ftrace_ops *removed_ops;
 
+/*
+ * Set when doing a global update, like enabling all recs or disabling them.
+ * It is not set when just updating a single ftrace_ops.
+ */
+static bool update_all_ops;
+
 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
 # error Dynamic ftrace depends on MCOUNT_RECORD
 #endif
@@ -1081,11 +1103,12 @@ static const struct ftrace_hash empty_hash = {
 #define EMPTY_HASH      ((struct ftrace_hash *)&empty_hash)
 
 static struct ftrace_ops global_ops = {
-        .func                   = ftrace_stub,
-        .notrace_hash           = EMPTY_HASH,
-        .filter_hash            = EMPTY_HASH,
-        .flags                  = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
-        INIT_REGEX_LOCK(global_ops)
+        .func                           = ftrace_stub,
+        .local_hash.notrace_hash        = EMPTY_HASH,
+        .local_hash.filter_hash         = EMPTY_HASH,
+        INIT_OPS_HASH(global_ops)
+        .flags                          = FTRACE_OPS_FL_RECURSION_SAFE |
+                                          FTRACE_OPS_FL_INITIALIZED,
 };
 
 struct ftrace_page {
@@ -1226,8 +1249,8 @@ static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
 void ftrace_free_filter(struct ftrace_ops *ops)
 {
         ftrace_ops_init(ops);
-        free_ftrace_hash(ops->filter_hash);
-        free_ftrace_hash(ops->notrace_hash);
+        free_ftrace_hash(ops->func_hash->filter_hash);
+        free_ftrace_hash(ops->func_hash->notrace_hash);
 }
 
 static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
@@ -1288,9 +1311,9 @@ alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
 }
 
 static void
-ftrace_hash_rec_disable(struct ftrace_ops *ops, int filter_hash);
+ftrace_hash_rec_disable_modify(struct ftrace_ops *ops, int filter_hash);
 static void
-ftrace_hash_rec_enable(struct ftrace_ops *ops, int filter_hash);
+ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, int filter_hash);
 
 static int
 ftrace_hash_move(struct ftrace_ops *ops, int enable,
@@ -1299,7 +1322,6 @@ ftrace_hash_move(struct ftrace_ops *ops, int enable,
         struct ftrace_func_entry *entry;
         struct hlist_node *tn;
         struct hlist_head *hhd;
-        struct ftrace_hash *old_hash;
         struct ftrace_hash *new_hash;
         int size = src->count;
         int bits = 0;
@@ -1342,17 +1364,30 @@ update:
          * Remove the current set, update the hash and add
          * them back.
          */
-        ftrace_hash_rec_disable(ops, enable);
+        ftrace_hash_rec_disable_modify(ops, enable);
 
-        old_hash = *dst;
         rcu_assign_pointer(*dst, new_hash);
-        free_ftrace_hash_rcu(old_hash);
 
-        ftrace_hash_rec_enable(ops, enable);
+        ftrace_hash_rec_enable_modify(ops, enable);
 
         return 0;
 }
 
+static bool hash_contains_ip(unsigned long ip,
+                             struct ftrace_ops_hash *hash)
+{
+        /*
+         * The function record is a match if it exists in the filter
+         * hash and not in the notrace hash. Note, an emty hash is
+         * considered a match for the filter hash, but an empty
+         * notrace hash is considered not in the notrace hash.
+         */
+        return (ftrace_hash_empty(hash->filter_hash) ||
+                ftrace_lookup_ip(hash->filter_hash, ip)) &&
+                (ftrace_hash_empty(hash->notrace_hash) ||
+                 !ftrace_lookup_ip(hash->notrace_hash, ip));
+}
+
 /*
  * Test the hashes for this ops to see if we want to call
  * the ops->func or not.
@@ -1368,8 +1403,7 @@ update:
 static int
 ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
 {
-        struct ftrace_hash *filter_hash;
-        struct ftrace_hash *notrace_hash;
+        struct ftrace_ops_hash hash;
         int ret;
 
 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
@@ -1382,13 +1416,10 @@ ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
                 return 0;
 #endif
 
-        filter_hash = rcu_dereference_raw_notrace(ops->filter_hash);
-        notrace_hash = rcu_dereference_raw_notrace(ops->notrace_hash);
+        hash.filter_hash = rcu_dereference_raw_notrace(ops->func_hash->filter_hash);
+        hash.notrace_hash = rcu_dereference_raw_notrace(ops->func_hash->notrace_hash);
 
-        if ((ftrace_hash_empty(filter_hash) ||
-             ftrace_lookup_ip(filter_hash, ip)) &&
-            (ftrace_hash_empty(notrace_hash) ||
-             !ftrace_lookup_ip(notrace_hash, ip)))
+        if (hash_contains_ip(ip, &hash))
                 ret = 1;
         else
                 ret = 0;
@@ -1500,33 +1531,6 @@ static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec)
         return  keep_regs;
 }
 
-static void ftrace_remove_tramp(struct ftrace_ops *ops,
-                                struct dyn_ftrace *rec)
-{
-        struct ftrace_func_entry *entry;
-
-        entry = ftrace_lookup_ip(ops->tramp_hash, rec->ip);
-        if (!entry)
-                return;
-
-        /*
-         * The tramp_hash entry will be removed at time
-         * of update.
-         */
-        ops->nr_trampolines--;
-        rec->flags &= ~FTRACE_FL_TRAMP;
-}
-
-static void ftrace_clear_tramps(struct dyn_ftrace *rec)
-{
-        struct ftrace_ops *op;
-
-        do_for_each_ftrace_op(op, ftrace_ops_list) {
-                if (op->nr_trampolines)
-                        ftrace_remove_tramp(op, rec);
-        } while_for_each_ftrace_op(op);
-}
-
 static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
                                      int filter_hash,
                                      bool inc)
@@ -1554,14 +1558,14 @@ static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
          *   gets inversed.
          */
         if (filter_hash) {
-                hash = ops->filter_hash;
-                other_hash = ops->notrace_hash;
+                hash = ops->func_hash->filter_hash;
+                other_hash = ops->func_hash->notrace_hash;
                 if (ftrace_hash_empty(hash))
                         all = 1;
         } else {
                 inc = !inc;
-                hash = ops->notrace_hash;
-                other_hash = ops->filter_hash;
+                hash = ops->func_hash->notrace_hash;
+                other_hash = ops->func_hash->filter_hash;
                 /*
                  * If the notrace hash has no items,
                  * then there's nothing to do.
@@ -1615,22 +1619,17 @@ static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
                          * function, and the ops has a trampoline registered
                          * for it, then we can call it directly.
                          */
-                        if (ftrace_rec_count(rec) == 1 && ops->trampoline) {
+                        if (ftrace_rec_count(rec) == 1 && ops->trampoline)
                                 rec->flags |= FTRACE_FL_TRAMP;
-                                ops->nr_trampolines++;
-                        } else {
+                        else
                                 /*
                                  * If we are adding another function callback
                                  * to this function, and the previous had a
-                                 * trampoline used, then we need to go back to
-                                 * the default trampoline.
+                                 * custom trampoline in use, then we need to go
+                                 * back to the default trampoline.
                                  */
                                 rec->flags &= ~FTRACE_FL_TRAMP;
 
-                                /* remove trampolines from any ops for this rec */
-                                ftrace_clear_tramps(rec);
-                        }
-
                         /*
                          * If any ops wants regs saved for this function
                          * then all ops will get saved regs.
@@ -1642,9 +1641,6 @@ static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
                                 return;
                         rec->flags--;
 
-                        if (ops->trampoline && !ftrace_rec_count(rec))
-                                ftrace_remove_tramp(ops, rec);
-
                         /*
                          * If the rec had REGS enabled and the ops that is
                          * being removed had REGS set, then see if there is
@@ -1659,6 +1655,17 @@ static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
                         }
 
                         /*
+                         * If the rec had TRAMP enabled, then it needs to
+                         * be cleared. As TRAMP can only be enabled iff
+                         * there is only a single ops attached to it.
+                         * In otherwords, always disable it on decrementing.
+                         * In the future, we may set it if rec count is
+                         * decremented to one, and the ops that is left
+                         * has a trampoline.
+                         */
+                        rec->flags &= ~FTRACE_FL_TRAMP;
+
+                        /*
                          * flags will be cleared in ftrace_check_record()
                          * if rec count is zero.
                          */
@@ -1682,6 +1689,41 @@ static void ftrace_hash_rec_enable(struct ftrace_ops *ops,
         __ftrace_hash_rec_update(ops, filter_hash, 1);
 }
 
+static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops,
+                                          int filter_hash, int inc)
+{
+        struct ftrace_ops *op;
+
+        __ftrace_hash_rec_update(ops, filter_hash, inc);
+
+        if (ops->func_hash != &global_ops.local_hash)
+                return;
+
+        /*
+         * If the ops shares the global_ops hash, then we need to update
+         * all ops that are enabled and use this hash.
+         */
+        do_for_each_ftrace_op(op, ftrace_ops_list) {
+                /* Already done */
+                if (op == ops)
+                        continue;
+                if (op->func_hash == &global_ops.local_hash)
+                        __ftrace_hash_rec_update(op, filter_hash, inc);
+        } while_for_each_ftrace_op(op);
+}
+
+static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops,
+                                           int filter_hash)
+{
+        ftrace_hash_rec_update_modify(ops, filter_hash, 0);
+}
+
+static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops,
+                                          int filter_hash)
+{
+        ftrace_hash_rec_update_modify(ops, filter_hash, 1);
+}
+
 static void print_ip_ins(const char *fmt, unsigned char *p)
 {
         int i;
@@ -1842,21 +1884,86 @@ int ftrace_test_record(struct dyn_ftrace *rec, int enable)
 }
 
 static struct ftrace_ops *
+ftrace_find_tramp_ops_any(struct dyn_ftrace *rec)
+{
+        struct ftrace_ops *op;
+        unsigned long ip = rec->ip;
+
+        do_for_each_ftrace_op(op, ftrace_ops_list) {
+
+                if (!op->trampoline)
+                        continue;
+
+                if (hash_contains_ip(ip, op->func_hash))
+                        return op;
+        } while_for_each_ftrace_op(op);
+
+        return NULL;
+}
+
+static struct ftrace_ops *
 ftrace_find_tramp_ops_curr(struct dyn_ftrace *rec)
 {
         struct ftrace_ops *op;
+        unsigned long ip = rec->ip;
 
-        /* Removed ops need to be tested first */
-        if (removed_ops && removed_ops->tramp_hash) {
-                if (ftrace_lookup_ip(removed_ops->tramp_hash, rec->ip))
+        /*
+         * Need to check removed ops first.
+         * If they are being removed, and this rec has a tramp,
+         * and this rec is in the ops list, then it would be the
+         * one with the tramp.
+         */
+        if (removed_ops) {
+                if (hash_contains_ip(ip, &removed_ops->old_hash))
                         return removed_ops;
         }
 
+        /*
+         * Need to find the current trampoline for a rec.
+         * Now, a trampoline is only attached to a rec if there
+         * was a single 'ops' attached to it. But this can be called
+         * when we are adding another op to the rec or removing the
+         * current one. Thus, if the op is being added, we can
+         * ignore it because it hasn't attached itself to the rec
+         * yet.
+         *
+         * If an ops is being modified (hooking to different functions)
+         * then we don't care about the new functions that are being
+         * added, just the old ones (that are probably being removed).
+         *
+         * If we are adding an ops to a function that already is using
+         * a trampoline, it needs to be removed (trampolines are only
+         * for single ops connected), then an ops that is not being
+         * modified also needs to be checked.
+         */
         do_for_each_ftrace_op(op, ftrace_ops_list) {
-                if (!op->tramp_hash)
+
+                if (!op->trampoline)
                         continue;
 
-                if (ftrace_lookup_ip(op->tramp_hash, rec->ip))
+                /*
+                 * If the ops is being added, it hasn't gotten to
+                 * the point to be removed from this tree yet.
+                 */
+                if (op->flags & FTRACE_OPS_FL_ADDING)
+                        continue;
+
+
+                /*
+                 * If the ops is being modified and is in the old
+                 * hash, then it is probably being removed from this
+                 * function.
+                 */
+                if ((op->flags & FTRACE_OPS_FL_MODIFYING) &&
+                    hash_contains_ip(ip, &op->old_hash))
+                        return op;
+                /*
+                 * If the ops is not being added or modified, and it's
+                 * in its normal filter hash, then this must be the one
+                 * we want!
+                 */
+                if (!(op->flags & FTRACE_OPS_FL_MODIFYING) &&
+                    hash_contains_ip(ip, op->func_hash))
                         return op;
 
         } while_for_each_ftrace_op(op);
@@ -1868,10 +1975,11 @@ static struct ftrace_ops *
 ftrace_find_tramp_ops_new(struct dyn_ftrace *rec)
 {
         struct ftrace_ops *op;
+        unsigned long ip = rec->ip;
 
         do_for_each_ftrace_op(op, ftrace_ops_list) {
                 /* pass rec in as regs to have non-NULL val */
-                if (ftrace_ops_test(op, rec->ip, rec))
+                if (hash_contains_ip(ip, op->func_hash))
                         return op;
         } while_for_each_ftrace_op(op);
 
@@ -1896,8 +2004,8 @@ unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec)
         if (rec->flags & FTRACE_FL_TRAMP) {
                 ops = ftrace_find_tramp_ops_new(rec);
                 if (FTRACE_WARN_ON(!ops || !ops->trampoline)) {
-                        pr_warning("Bad trampoline accounting at: %p (%pS)\n",
-                                    (void *)rec->ip, (void *)rec->ip);
+                        pr_warn("Bad trampoline accounting at: %p (%pS) (%lx)\n",
+                                (void *)rec->ip, (void *)rec->ip, rec->flags);
                         /* Ftrace is shutting down, return anything */
                         return (unsigned long)FTRACE_ADDR;
                 }
@@ -1964,7 +2072,7 @@ __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
                 return ftrace_make_call(rec, ftrace_addr);
 
         case FTRACE_UPDATE_MAKE_NOP:
-                return ftrace_make_nop(NULL, rec, ftrace_addr);
+                return ftrace_make_nop(NULL, rec, ftrace_old_addr);
 
         case FTRACE_UPDATE_MODIFY_CALL:
                 return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr);
@@ -2178,89 +2286,6 @@ void __weak arch_ftrace_update_code(int command)
         ftrace_run_stop_machine(command);
 }
 
-static int ftrace_save_ops_tramp_hash(struct ftrace_ops *ops)
-{
-        struct ftrace_page *pg;
-        struct dyn_ftrace *rec;
-        int size, bits;
-        int ret;
-
-        size = ops->nr_trampolines;
-        bits = 0;
-        /*
-         * Make the hash size about 1/2 the # found
-         */
-        for (size /= 2; size; size >>= 1)
-                bits++;
-
-        ops->tramp_hash = alloc_ftrace_hash(bits);
-        /*
-         * TODO: a failed allocation is going to screw up
-         * the accounting of what needs to be modified
-         * and not. For now, we kill ftrace if we fail
-         * to allocate here. But there are ways around this,
-         * but that will take a little more work.
-         */
-        if (!ops->tramp_hash)
-                return -ENOMEM;
-
-        do_for_each_ftrace_rec(pg, rec) {
-                if (ftrace_rec_count(rec) == 1 &&
-                    ftrace_ops_test(ops, rec->ip, rec)) {
-
-                        /*
-                         * If another ops adds to a rec, the rec will
-                         * lose its trampoline and never get it back
-                         * until all ops are off of it.
-                         */
-                        if (!(rec->flags & FTRACE_FL_TRAMP))
-                                continue;
-
-                        /* This record had better have a trampoline */
-                        if (FTRACE_WARN_ON(!(rec->flags & FTRACE_FL_TRAMP_EN)))
-                                return -1;
-
-                        ret = add_hash_entry(ops->tramp_hash, rec->ip);
-                        if (ret < 0)
-                                return ret;
-                }
-        } while_for_each_ftrace_rec();
-
-        /* The number of recs in the hash must match nr_trampolines */
-        FTRACE_WARN_ON(ops->tramp_hash->count != ops->nr_trampolines);
-
-        return 0;
-}
-
-static int ftrace_save_tramp_hashes(void)
-{
-        struct ftrace_ops *op;
-        int ret;
-
-        /*
-         * Now that any trampoline is being used, we need to save the
-         * hashes for the ops that have them. This allows the mapping
-         * back from the record to the ops that has the trampoline to
-         * know what code is being replaced. Modifying code must always
-         * verify what it is changing.
-         */
-        do_for_each_ftrace_op(op, ftrace_ops_list) {
-
-                /* The tramp_hash is recreated each time. */
-                free_ftrace_hash(op->tramp_hash);
-                op->tramp_hash = NULL;
-
-                if (op->nr_trampolines) {
-                        ret = ftrace_save_ops_tramp_hash(op);
-                        if (ret)
-                                return ret;
-                }
-
-        } while_for_each_ftrace_op(op);
-
-        return 0;
-}
-
 static void ftrace_run_update_code(int command)
 {
         int ret;
@@ -2280,9 +2305,16 @@ static void ftrace_run_update_code(int command)
 
         ret = ftrace_arch_code_modify_post_process();
         FTRACE_WARN_ON(ret);
+}
 
-        ret = ftrace_save_tramp_hashes();
-        FTRACE_WARN_ON(ret);
+static void ftrace_run_modify_code(struct ftrace_ops *ops, int command,
+                                   struct ftrace_hash *old_hash)
+{
+        ops->flags |= FTRACE_OPS_FL_MODIFYING;
+        ops->old_hash.filter_hash = old_hash;
+        ftrace_run_update_code(command);
+        ops->old_hash.filter_hash = NULL;
+        ops->flags &= ~FTRACE_OPS_FL_MODIFYING;
 }
 
 static ftrace_func_t saved_ftrace_func;
@@ -2306,6 +2338,13 @@ static void ftrace_startup_enable(int command)
         ftrace_run_update_code(command);
 }
 
+static void ftrace_startup_all(int command)
+{
+        update_all_ops = true;
+        ftrace_startup_enable(command);
+        update_all_ops = false;
+}
+
 static int ftrace_startup(struct ftrace_ops *ops, int command)
 {
         int ret;
@@ -2320,12 +2359,22 @@ static int ftrace_startup(struct ftrace_ops *ops, int command)
         ftrace_start_up++;
         command |= FTRACE_UPDATE_CALLS;
 
-        ops->flags |= FTRACE_OPS_FL_ENABLED;
+        /*
+         * Note that ftrace probes uses this to start up
+         * and modify functions it will probe. But we still
+         * set the ADDING flag for modification, as probes
+         * do not have trampolines. If they add them in the
+         * future, then the probes will need to distinguish
+         * between adding and updating probes.
+         */
+        ops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_ADDING;
 
         ftrace_hash_rec_enable(ops, 1);
 
         ftrace_startup_enable(command);
 
+        ops->flags &= ~FTRACE_OPS_FL_ADDING;
+
         return 0;
 }
 
@@ -2375,11 +2424,35 @@ static int ftrace_shutdown(struct ftrace_ops *ops, int command)
          * If the ops uses a trampoline, then it needs to be
          * tested first on update.
          */
+        ops->flags |= FTRACE_OPS_FL_REMOVING;
         removed_ops = ops;
 
+        /* The trampoline logic checks the old hashes */
+        ops->old_hash.filter_hash = ops->func_hash->filter_hash;
+        ops->old_hash.notrace_hash = ops->func_hash->notrace_hash;
+
         ftrace_run_update_code(command);
 
+        /*
+         * If there's no more ops registered with ftrace, run a
+         * sanity check to make sure all rec flags are cleared.
+         */
+        if (ftrace_ops_list == &ftrace_list_end) {
+                struct ftrace_page *pg;
+                struct dyn_ftrace *rec;
+
+                do_for_each_ftrace_rec(pg, rec) {
+                        if (FTRACE_WARN_ON_ONCE(rec->flags))
+                                pr_warn("  %pS flags:%lx\n",
+                                        (void *)rec->ip, rec->flags);
+                } while_for_each_ftrace_rec();
+        }
+
+        ops->old_hash.filter_hash = NULL;
+        ops->old_hash.notrace_hash = NULL;
+
         removed_ops = NULL;
+        ops->flags &= ~FTRACE_OPS_FL_REMOVING;
 
         /*
          * Dynamic ops may be freed, we must make sure that all
@@ -2436,8 +2509,8 @@ static inline int ops_traces_mod(struct ftrace_ops *ops)
          * Filter_hash being empty will default to trace module.
          * But notrace hash requires a test of individual module functions.
          */
-        return ftrace_hash_empty(ops->filter_hash) &&
-                ftrace_hash_empty(ops->notrace_hash);
+        return ftrace_hash_empty(ops->func_hash->filter_hash) &&
+                ftrace_hash_empty(ops->func_hash->notrace_hash);
 }
 
 /*
@@ -2459,12 +2532,12 @@ ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec)
                 return 0;
 
         /* The function must be in the filter */
-        if (!ftrace_hash_empty(ops->filter_hash) &&
-            !ftrace_lookup_ip(ops->filter_hash, rec->ip))
+        if (!ftrace_hash_empty(ops->func_hash->filter_hash) &&
+            !ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip))
                 return 0;
 
         /* If in notrace hash, we ignore it too */
-        if (ftrace_lookup_ip(ops->notrace_hash, rec->ip))
+        if (ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip))
                 return 0;
 
         return 1;
@@ -2785,10 +2858,10 @@ t_next(struct seq_file *m, void *v, loff_t *pos)
         } else {
                 rec = &iter->pg->records[iter->idx++];
                 if (((iter->flags & FTRACE_ITER_FILTER) &&
-                     !(ftrace_lookup_ip(ops->filter_hash, rec->ip))) ||
+                     !(ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip))) ||
 
                     ((iter->flags & FTRACE_ITER_NOTRACE) &&
-                     !ftrace_lookup_ip(ops->notrace_hash, rec->ip)) ||
+                     !ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip)) ||
 
                     ((iter->flags & FTRACE_ITER_ENABLED) &&
                      !(rec->flags & FTRACE_FL_ENABLED))) {
@@ -2837,9 +2910,9 @@ static void *t_start(struct seq_file *m, loff_t *pos)
          * functions are enabled.
          */
         if ((iter->flags & FTRACE_ITER_FILTER &&
-             ftrace_hash_empty(ops->filter_hash)) ||
+             ftrace_hash_empty(ops->func_hash->filter_hash)) ||
             (iter->flags & FTRACE_ITER_NOTRACE &&
-             ftrace_hash_empty(ops->notrace_hash))) {
+             ftrace_hash_empty(ops->func_hash->notrace_hash))) {
                 if (*pos > 0)
                         return t_hash_start(m, pos);
                 iter->flags |= FTRACE_ITER_PRINTALL;
@@ -2904,8 +2977,8 @@ static int t_show(struct seq_file *m, void *v)
                 if (rec->flags & FTRACE_FL_TRAMP_EN) {
                         struct ftrace_ops *ops;
 
-                        ops = ftrace_find_tramp_ops_curr(rec);
-                        if (ops && ops->trampoline)
+                        ops = ftrace_find_tramp_ops_any(rec);
+                        if (ops)
                                 seq_printf(m, "\ttramp: %pS",
                                            (void *)ops->trampoline);
                         else
@@ -3001,12 +3074,12 @@ ftrace_regex_open(struct ftrace_ops *ops, int flag,
         iter->ops = ops;
         iter->flags = flag;
 
-        mutex_lock(&ops->regex_lock);
+        mutex_lock(&ops->func_hash->regex_lock);
 
         if (flag & FTRACE_ITER_NOTRACE)
-                hash = ops->notrace_hash;
+                hash = ops->func_hash->notrace_hash;
         else
-                hash = ops->filter_hash;
+                hash = ops->func_hash->filter_hash;
 
         if (file->f_mode & FMODE_WRITE) {
                 const int size_bits = FTRACE_HASH_DEFAULT_BITS;
@@ -3041,7 +3114,7 @@ ftrace_regex_open(struct ftrace_ops *ops, int flag,
                 file->private_data = iter;
 
  out_unlock:
-        mutex_unlock(&ops->regex_lock);
+        mutex_unlock(&ops->func_hash->regex_lock);
 
         return ret;
 }
@@ -3279,12 +3352,12 @@ static struct ftrace_ops trace_probe_ops __read_mostly =
 {
         .func           = function_trace_probe_call,
         .flags          = FTRACE_OPS_FL_INITIALIZED,
-        INIT_REGEX_LOCK(trace_probe_ops)
+        INIT_OPS_HASH(trace_probe_ops)
 };
 
 static int ftrace_probe_registered;
 
-static void __enable_ftrace_function_probe(void)
+static void __enable_ftrace_function_probe(struct ftrace_hash *old_hash)
 {
         int ret;
         int i;
@@ -3292,7 +3365,8 @@ static void __enable_ftrace_function_probe(void)
         if (ftrace_probe_registered) {
                 /* still need to update the function call sites */
                 if (ftrace_enabled)
-                        ftrace_run_update_code(FTRACE_UPDATE_CALLS);
+                        ftrace_run_modify_code(&trace_probe_ops, FTRACE_UPDATE_CALLS,
+                                               old_hash);
                 return;
         }
 
@@ -3342,7 +3416,8 @@ register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
                               void *data)
 {
         struct ftrace_func_probe *entry;
-        struct ftrace_hash **orig_hash = &trace_probe_ops.filter_hash;
+        struct ftrace_hash **orig_hash = &trace_probe_ops.func_hash->filter_hash;
+        struct ftrace_hash *old_hash = *orig_hash;
         struct ftrace_hash *hash;
         struct ftrace_page *pg;
         struct dyn_ftrace *rec;
@@ -3359,9 +3434,9 @@ register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
         if (WARN_ON(not))
                 return -EINVAL;
 
-        mutex_lock(&trace_probe_ops.regex_lock);
+        mutex_lock(&trace_probe_ops.func_hash->regex_lock);
 
-        hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
+        hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);
         if (!hash) {
                 count = -ENOMEM;
                 goto out;
@@ -3420,15 +3495,18 @@ register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
         } while_for_each_ftrace_rec();
 
         ret = ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash);
-        if (ret < 0)
-                count = ret;
 
-        __enable_ftrace_function_probe();
+        __enable_ftrace_function_probe(old_hash);
+
+        if (!ret)
+                free_ftrace_hash_rcu(old_hash);
+        else
+                count = ret;
 
  out_unlock:
         mutex_unlock(&ftrace_lock);
  out:
-        mutex_unlock(&trace_probe_ops.regex_lock);
+        mutex_unlock(&trace_probe_ops.func_hash->regex_lock);
         free_ftrace_hash(hash);
 
         return count;
@@ -3446,7 +3524,8 @@ __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
         struct ftrace_func_entry *rec_entry;
         struct ftrace_func_probe *entry;
         struct ftrace_func_probe *p;
-        struct ftrace_hash **orig_hash = &trace_probe_ops.filter_hash;
+        struct ftrace_hash **orig_hash = &trace_probe_ops.func_hash->filter_hash;
+        struct ftrace_hash *old_hash = *orig_hash;
         struct list_head free_list;
         struct ftrace_hash *hash;
         struct hlist_node *tmp;
@@ -3454,6 +3533,7 @@ __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
         int type = MATCH_FULL;
         int i, len = 0;
         char *search;
+        int ret;
 
         if (glob && (strcmp(glob, "*") == 0 || !strlen(glob)))
                 glob = NULL;
@@ -3468,7 +3548,7 @@ __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
                         return;
         }
 
-        mutex_lock(&trace_probe_ops.regex_lock);
+        mutex_lock(&trace_probe_ops.func_hash->regex_lock);
 
         hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
         if (!hash)
@@ -3512,8 +3592,11 @@ __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
          * Remove after the disable is called. Otherwise, if the last
          * probe is removed, a null hash means *all enabled*.
          */
-        ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash);
+        ret = ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash);
         synchronize_sched();
+        if (!ret)
+                free_ftrace_hash_rcu(old_hash);
+
         list_for_each_entry_safe(entry, p, &free_list, free_list) {
                 list_del(&entry->free_list);
                 ftrace_free_entry(entry);
@@ -3521,7 +3604,7 @@ __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
         mutex_unlock(&ftrace_lock);
                 
  out_unlock:
-        mutex_unlock(&trace_probe_ops.regex_lock);
+        mutex_unlock(&trace_probe_ops.func_hash->regex_lock);
         free_ftrace_hash(hash);
 }
 
@@ -3700,10 +3783,11 @@ ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove)
         return add_hash_entry(hash, ip);
 }
 
-static void ftrace_ops_update_code(struct ftrace_ops *ops)
+static void ftrace_ops_update_code(struct ftrace_ops *ops,
+                                   struct ftrace_hash *old_hash)
 {
         if (ops->flags & FTRACE_OPS_FL_ENABLED && ftrace_enabled)
-                ftrace_run_update_code(FTRACE_UPDATE_CALLS);
+                ftrace_run_modify_code(ops, FTRACE_UPDATE_CALLS, old_hash);
 }
 
 static int
@@ -3711,18 +3795,19 @@ ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len,
                 unsigned long ip, int remove, int reset, int enable)
 {
         struct ftrace_hash **orig_hash;
+        struct ftrace_hash *old_hash;
         struct ftrace_hash *hash;
         int ret;
 
         if (unlikely(ftrace_disabled))
                 return -ENODEV;
 
-        mutex_lock(&ops->regex_lock);
+        mutex_lock(&ops->func_hash->regex_lock);
 
         if (enable)
-                orig_hash = &ops->filter_hash;
+                orig_hash = &ops->func_hash->filter_hash;
         else
-                orig_hash = &ops->notrace_hash;
+                orig_hash = &ops->func_hash->notrace_hash;
 
         if (reset)
                 hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
@@ -3745,14 +3830,16 @@ ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len,
         }
 
         mutex_lock(&ftrace_lock);
+        old_hash = *orig_hash;
         ret = ftrace_hash_move(ops, enable, orig_hash, hash);
-        if (!ret)
-                ftrace_ops_update_code(ops);
-
+        if (!ret) {
+                ftrace_ops_update_code(ops, old_hash);
+                free_ftrace_hash_rcu(old_hash);
+        }
         mutex_unlock(&ftrace_lock);
 
  out_regex_unlock:
-        mutex_unlock(&ops->regex_lock);
+        mutex_unlock(&ops->func_hash->regex_lock);
 
         free_ftrace_hash(hash);
         return ret;
@@ -3957,6 +4044,7 @@ int ftrace_regex_release(struct inode *inode, struct file *file)
         struct seq_file *m = (struct seq_file *)file->private_data;
         struct ftrace_iterator *iter;
         struct ftrace_hash **orig_hash;
+        struct ftrace_hash *old_hash;
         struct trace_parser *parser;
         int filter_hash;
         int ret;
@@ -3975,26 +4063,28 @@ int ftrace_regex_release(struct inode *inode, struct file *file)
 
         trace_parser_put(parser);
 
-        mutex_lock(&iter->ops->regex_lock);
+        mutex_lock(&iter->ops->func_hash->regex_lock);
 
         if (file->f_mode & FMODE_WRITE) {
                 filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);
 
                 if (filter_hash)
-                        orig_hash = &iter->ops->filter_hash;
+                        orig_hash = &iter->ops->func_hash->filter_hash;
                 else
-                        orig_hash = &iter->ops->notrace_hash;
+                        orig_hash = &iter->ops->func_hash->notrace_hash;
 
                 mutex_lock(&ftrace_lock);
+                old_hash = *orig_hash;
                 ret = ftrace_hash_move(iter->ops, filter_hash,
                                        orig_hash, iter->hash);
-                if (!ret)
-                        ftrace_ops_update_code(iter->ops);
-
+                if (!ret) {
+                        ftrace_ops_update_code(iter->ops, old_hash);
+                        free_ftrace_hash_rcu(old_hash);
+                }
                 mutex_unlock(&ftrace_lock);
         }
 
-        mutex_unlock(&iter->ops->regex_lock);
+        mutex_unlock(&iter->ops->func_hash->regex_lock);
         free_ftrace_hash(iter->hash);
         kfree(iter);
 
@@ -4611,7 +4701,6 @@ void __init ftrace_init(void)
 static struct ftrace_ops global_ops = {
         .func                   = ftrace_stub,
         .flags                  = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
-        INIT_REGEX_LOCK(global_ops)
 };
 
 static int __init ftrace_nodyn_init(void)
@@ -4623,6 +4712,7 @@ core_initcall(ftrace_nodyn_init);
 
 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
 static inline void ftrace_startup_enable(int command) { }
+static inline void ftrace_startup_all(int command) { }
 /* Keep as macros so we do not need to define the commands */
 # define ftrace_startup(ops, command)                                   \
         ({                                                              \
@@ -4713,7 +4803,7 @@ ftrace_ops_control_func(unsigned long ip, unsigned long parent_ip,
 static struct ftrace_ops control_ops = {
         .func   = ftrace_ops_control_func,
         .flags  = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
-        INIT_REGEX_LOCK(control_ops)
+        INIT_OPS_HASH(control_ops)
 };
 
 static inline void
@@ -4772,6 +4862,56 @@ static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip)
 }
 #endif
 
+/*
+ * If there's only one function registered but it does not support
+ * recursion, this function will be called by the mcount trampoline.
+ * This function will handle recursion protection.
+ */
+static void ftrace_ops_recurs_func(unsigned long ip, unsigned long parent_ip,
+                                   struct ftrace_ops *op, struct pt_regs *regs)
+{
+        int bit;
+
+        bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX);
+        if (bit < 0)
+                return;
+
+        op->func(ip, parent_ip, op, regs);
+
+        trace_clear_recursion(bit);
+}
+
+/**
+ * ftrace_ops_get_func - get the function a trampoline should call
+ * @ops: the ops to get the function for
+ *
+ * Normally the mcount trampoline will call the ops->func, but there
+ * are times that it should not. For example, if the ops does not
+ * have its own recursion protection, then it should call the
+ * ftrace_ops_recurs_func() instead.
+ *
+ * Returns the function that the trampoline should call for @ops.
+ */
+ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops)
+{
+        /*
+         * If this is a dynamic ops or we force list func,
+         * then it needs to call the list anyway.
+         */
+        if (ops->flags & FTRACE_OPS_FL_DYNAMIC || FTRACE_FORCE_LIST_FUNC)
+                return ftrace_ops_list_func;
+
+        /*
+         * If the func handles its own recursion, call it directly.
+         * Otherwise call the recursion protected function that
+         * will call the ftrace ops function.
+         */
+        if (!(ops->flags & FTRACE_OPS_FL_RECURSION_SAFE))
+                return ftrace_ops_recurs_func;
+
+        return ops->func;
+}
+
 static void clear_ftrace_swapper(void)
 {
         struct task_struct *p;
@@ -4872,7 +5012,8 @@ static int ftrace_pid_add(int p)
         set_ftrace_pid_task(pid);
 
         ftrace_update_pid_func();
-        ftrace_startup_enable(0);
+
+        ftrace_startup_all(0);
 
         mutex_unlock(&ftrace_lock);
         return 0;
@@ -4901,7 +5042,7 @@ static void ftrace_pid_reset(void)
         }
 
         ftrace_update_pid_func();
-        ftrace_startup_enable(0);
+        ftrace_startup_all(0);
 
         mutex_unlock(&ftrace_lock);
 }
@@ -5145,6 +5286,17 @@ ftrace_enable_sysctl(struct ctl_table *table, int write,
 
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 
+static struct ftrace_ops graph_ops = {
+        .func                   = ftrace_stub,
+        .flags                  = FTRACE_OPS_FL_RECURSION_SAFE |
+                                   FTRACE_OPS_FL_INITIALIZED |
+                                   FTRACE_OPS_FL_STUB,
+#ifdef FTRACE_GRAPH_TRAMP_ADDR
+        .trampoline             = FTRACE_GRAPH_TRAMP_ADDR,
+#endif
+        ASSIGN_OPS_HASH(graph_ops, &global_ops.local_hash)
+};
+
 static int ftrace_graph_active;
 
 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
@@ -5307,12 +5459,28 @@ static int ftrace_graph_entry_test(struct ftrace_graph_ent *trace)
  */
 static void update_function_graph_func(void)
 {
-        if (ftrace_ops_list == &ftrace_list_end ||
-            (ftrace_ops_list == &global_ops &&
-             global_ops.next == &ftrace_list_end))
-                ftrace_graph_entry = __ftrace_graph_entry;
-        else
+        struct ftrace_ops *op;
+        bool do_test = false;
+
+        /*
+         * The graph and global ops share the same set of functions
+         * to test. If any other ops is on the list, then
+         * the graph tracing needs to test if its the function
+         * it should call.
+         */
+        do_for_each_ftrace_op(op, ftrace_ops_list) {
+                if (op != &global_ops && op != &graph_ops &&
+                    op != &ftrace_list_end) {
+                        do_test = true;
+                        /* in double loop, break out with goto */
+                        goto out;
+                }
+        } while_for_each_ftrace_op(op);
+ out:
+        if (do_test)
                 ftrace_graph_entry = ftrace_graph_entry_test;
+        else
+                ftrace_graph_entry = __ftrace_graph_entry;
 }
 
 static struct notifier_block ftrace_suspend_notifier = {
@@ -5353,16 +5521,7 @@ int register_ftrace_graph(trace_func_graph_ret_t retfunc,
         ftrace_graph_entry = ftrace_graph_entry_test;
         update_function_graph_func();
 
-        /* Function graph doesn't use the .func field of global_ops */
-        global_ops.flags |= FTRACE_OPS_FL_STUB;
-
-#ifdef CONFIG_DYNAMIC_FTRACE
-        /* Optimize function graph calling (if implemented by arch) */
-        if (FTRACE_GRAPH_TRAMP_ADDR != 0)
-                global_ops.trampoline = FTRACE_GRAPH_TRAMP_ADDR;
-#endif
-
-        ret = ftrace_startup(&global_ops, FTRACE_START_FUNC_RET);
+        ret = ftrace_startup(&graph_ops, FTRACE_START_FUNC_RET);
 
 out:
         mutex_unlock(&ftrace_lock);
@@ -5380,12 +5539,7 @@ void unregister_ftrace_graph(void)
         ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
         ftrace_graph_entry = ftrace_graph_entry_stub;
         __ftrace_graph_entry = ftrace_graph_entry_stub;
-        ftrace_shutdown(&global_ops, FTRACE_STOP_FUNC_RET);
-        global_ops.flags &= ~FTRACE_OPS_FL_STUB;
-#ifdef CONFIG_DYNAMIC_FTRACE
-        if (FTRACE_GRAPH_TRAMP_ADDR != 0)
-                global_ops.trampoline = 0;
-#endif
+        ftrace_shutdown(&graph_ops, FTRACE_STOP_FUNC_RET);
         unregister_pm_notifier(&ftrace_suspend_notifier);
         unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
 
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index 925f629658d6..a56e07c8d15b 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -538,16 +538,18 @@ static void rb_wake_up_waiters(struct irq_work *work)
  * ring_buffer_wait - wait for input to the ring buffer
  * @buffer: buffer to wait on
  * @cpu: the cpu buffer to wait on
+ * @full: wait until a full page is available, if @cpu != RING_BUFFER_ALL_CPUS
  *
  * If @cpu == RING_BUFFER_ALL_CPUS then the task will wake up as soon
  * as data is added to any of the @buffer's cpu buffers. Otherwise
  * it will wait for data to be added to a specific cpu buffer.
  */
-int ring_buffer_wait(struct ring_buffer *buffer, int cpu)
+int ring_buffer_wait(struct ring_buffer *buffer, int cpu, bool full)
 {
-        struct ring_buffer_per_cpu *cpu_buffer;
+        struct ring_buffer_per_cpu *uninitialized_var(cpu_buffer);
         DEFINE_WAIT(wait);
         struct rb_irq_work *work;
+        int ret = 0;
 
         /*
          * Depending on what the caller is waiting for, either any
@@ -564,36 +566,61 @@ int ring_buffer_wait(struct ring_buffer *buffer, int cpu)
         }
 
 
-        prepare_to_wait(&work->waiters, &wait, TASK_INTERRUPTIBLE);
+        while (true) {
+                prepare_to_wait(&work->waiters, &wait, TASK_INTERRUPTIBLE);
 
-        /*
-         * The events can happen in critical sections where
-         * checking a work queue can cause deadlocks.
-         * After adding a task to the queue, this flag is set
-         * only to notify events to try to wake up the queue
-         * using irq_work.
-         *
-         * We don't clear it even if the buffer is no longer
-         * empty. The flag only causes the next event to run
-         * irq_work to do the work queue wake up. The worse
-         * that can happen if we race with !trace_empty() is that
-         * an event will cause an irq_work to try to wake up
-         * an empty queue.
-         *
-         * There's no reason to protect this flag either, as
-         * the work queue and irq_work logic will do the necessary
-         * synchronization for the wake ups. The only thing
-         * that is necessary is that the wake up happens after
-         * a task has been queued. It's OK for spurious wake ups.
-         */
-        work->waiters_pending = true;
+                /*
+                 * The events can happen in critical sections where
+                 * checking a work queue can cause deadlocks.
+                 * After adding a task to the queue, this flag is set
+                 * only to notify events to try to wake up the queue
+                 * using irq_work.
+                 *
+                 * We don't clear it even if the buffer is no longer
+                 * empty. The flag only causes the next event to run
+                 * irq_work to do the work queue wake up. The worse
+                 * that can happen if we race with !trace_empty() is that
+                 * an event will cause an irq_work to try to wake up
+                 * an empty queue.
+                 *
+                 * There's no reason to protect this flag either, as
+                 * the work queue and irq_work logic will do the necessary
+                 * synchronization for the wake ups. The only thing
+                 * that is necessary is that the wake up happens after
+                 * a task has been queued. It's OK for spurious wake ups.
+                 */
+                work->waiters_pending = true;
+
+                if (signal_pending(current)) {
+                        ret = -EINTR;
+                        break;
+                }
+
+                if (cpu == RING_BUFFER_ALL_CPUS && !ring_buffer_empty(buffer))
+                        break;
+
+                if (cpu != RING_BUFFER_ALL_CPUS &&
+                    !ring_buffer_empty_cpu(buffer, cpu)) {
+                        unsigned long flags;
+                        bool pagebusy;
+
+                        if (!full)
+                                break;
+
+                        raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
+                        pagebusy = cpu_buffer->reader_page == cpu_buffer->commit_page;
+                        raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
+
+                        if (!pagebusy)
+                                break;
+                }
 
-        if ((cpu == RING_BUFFER_ALL_CPUS && ring_buffer_empty(buffer)) ||
-            (cpu != RING_BUFFER_ALL_CPUS && ring_buffer_empty_cpu(buffer, cpu)))
                 schedule();
+        }
 
         finish_wait(&work->waiters, &wait);
-        return 0;
+
+        return ret;
 }
 
 /**
@@ -626,8 +653,22 @@ int ring_buffer_poll_wait(struct ring_buffer *buffer, int cpu,
                 work = &cpu_buffer->irq_work;
         }
 
-        work->waiters_pending = true;
         poll_wait(filp, &work->waiters, poll_table);
+        work->waiters_pending = true;
+        /*
+         * There's a tight race between setting the waiters_pending and
+         * checking if the ring buffer is empty.  Once the waiters_pending bit
+         * is set, the next event will wake the task up, but we can get stuck
+         * if there's only a single event in.
+         *
+         * FIXME: Ideally, we need a memory barrier on the writer side as well,
+         * but adding a memory barrier to all events will cause too much of a
+         * performance hit in the fast path.  We only need a memory barrier when
+         * the buffer goes from empty to having content.  But as this race is
+         * extremely small, and it's not a problem if another event comes in, we
+         * will fix it later.
+         */
+        smp_mb();
 
         if ((cpu == RING_BUFFER_ALL_CPUS && !ring_buffer_empty(buffer)) ||
             (cpu != RING_BUFFER_ALL_CPUS && !ring_buffer_empty_cpu(buffer, cpu)))
@@ -1968,7 +2009,7 @@ rb_add_time_stamp(struct ring_buffer_event *event, u64 delta)
 
 /**
  * rb_update_event - update event type and data
- * @event: the even to update
+ * @event: the event to update
  * @type: the type of event
  * @length: the size of the event field in the ring buffer
  *
@@ -3341,21 +3382,16 @@ static void rb_iter_reset(struct ring_buffer_iter *iter)
         struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
 
         /* Iterator usage is expected to have record disabled */
-        if (list_empty(&cpu_buffer->reader_page->list)) {
-                iter->head_page = rb_set_head_page(cpu_buffer);
-                if (unlikely(!iter->head_page))
-                        return;
-                iter->head = iter->head_page->read;
-        } else {
-                iter->head_page = cpu_buffer->reader_page;
-                iter->head = cpu_buffer->reader_page->read;
-        }
+        iter->head_page = cpu_buffer->reader_page;
+        iter->head = cpu_buffer->reader_page->read;
+
+        iter->cache_reader_page = iter->head_page;
+        iter->cache_read = cpu_buffer->read;
+
         if (iter->head)
                 iter->read_stamp = cpu_buffer->read_stamp;
         else
                 iter->read_stamp = iter->head_page->page->time_stamp;
-        iter->cache_reader_page = cpu_buffer->reader_page;
-        iter->cache_read = cpu_buffer->read;
 }
 
 /**
@@ -3748,12 +3784,14 @@ rb_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
                 return NULL;
 
         /*
-         * We repeat when a time extend is encountered.
-         * Since the time extend is always attached to a data event,
-         * we should never loop more than once.
-         * (We never hit the following condition more than twice).
+         * We repeat when a time extend is encountered or we hit
+         * the end of the page. Since the time extend is always attached
+         * to a data event, we should never loop more than three times.
+         * Once for going to next page, once on time extend, and
+         * finally once to get the event.
+         * (We never hit the following condition more than thrice).
          */
-        if (RB_WARN_ON(cpu_buffer, ++nr_loops > 2))
+        if (RB_WARN_ON(cpu_buffer, ++nr_loops > 3))
                 return NULL;
 
         if (rb_per_cpu_empty(cpu_buffer))
diff --git a/kernel/trace/ring_buffer_benchmark.c b/kernel/trace/ring_buffer_benchmark.c
index 0434ff1b808e..3f9e328c30b5 100644
--- a/kernel/trace/ring_buffer_benchmark.c
+++ b/kernel/trace/ring_buffer_benchmark.c
@@ -205,7 +205,6 @@ static void ring_buffer_consumer(void)
                         break;
 
                 schedule();
-                __set_current_state(TASK_RUNNING);
         }
         reader_finish = 0;
         complete(&read_done);
@@ -379,7 +378,6 @@ static int ring_buffer_consumer_thread(void *arg)
                         break;
 
                 schedule();
-                __set_current_state(TASK_RUNNING);
         }
         __set_current_state(TASK_RUNNING);
 
@@ -407,7 +405,6 @@ static int ring_buffer_producer_thread(void *arg)
                 trace_printk("Sleeping for 10 secs\n");
                 set_current_state(TASK_INTERRUPTIBLE);
                 schedule_timeout(HZ * SLEEP_TIME);
-                __set_current_state(TASK_RUNNING);
         }
 
         if (kill_test)
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index f3ef80c8914c..0fa2d2070bd4 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -1076,13 +1076,14 @@ update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
 }
 #endif /* CONFIG_TRACER_MAX_TRACE */
 
-static int wait_on_pipe(struct trace_iterator *iter)
+static int wait_on_pipe(struct trace_iterator *iter, bool full)
 {
         /* Iterators are static, they should be filled or empty */
         if (trace_buffer_iter(iter, iter->cpu_file))
                 return 0;
 
-        return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file);
+        return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file,
+                                full);
 }
 
 #ifdef CONFIG_FTRACE_STARTUP_TEST
@@ -4434,15 +4435,12 @@ static int tracing_wait_pipe(struct file *filp)
 
                 mutex_unlock(&iter->mutex);
 
-                ret = wait_on_pipe(iter);
+                ret = wait_on_pipe(iter, false);
 
                 mutex_lock(&iter->mutex);
 
                 if (ret)
                         return ret;
-
-                if (signal_pending(current))
-                        return -EINTR;
         }
 
         return 1;
@@ -5372,16 +5370,12 @@ tracing_buffers_read(struct file *filp, char __user *ubuf,
                                 goto out_unlock;
                         }
                         mutex_unlock(&trace_types_lock);
-                        ret = wait_on_pipe(iter);
+                        ret = wait_on_pipe(iter, false);
                         mutex_lock(&trace_types_lock);
                         if (ret) {
                                 size = ret;
                                 goto out_unlock;
                         }
-                        if (signal_pending(current)) {
-                                size = -EINTR;
-                                goto out_unlock;
-                        }
                         goto again;
                 }
                 size = 0;
@@ -5500,7 +5494,7 @@ tracing_buffers_splice_read(struct file *file, loff_t *ppos,
         };
         struct buffer_ref *ref;
         int entries, size, i;
-        ssize_t ret;
+        ssize_t ret = 0;
 
         mutex_lock(&trace_types_lock);
 
@@ -5538,13 +5532,16 @@ tracing_buffers_splice_read(struct file *file, loff_t *ppos,
                 int r;
 
                 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
-                if (!ref)
+                if (!ref) {
+                        ret = -ENOMEM;
                         break;
+                }
 
                 ref->ref = 1;
                 ref->buffer = iter->trace_buffer->buffer;
                 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
                 if (!ref->page) {
+                        ret = -ENOMEM;
                         kfree(ref);
                         break;
                 }
@@ -5582,19 +5579,19 @@ tracing_buffers_splice_read(struct file *file, loff_t *ppos,
 
         /* did we read anything? */
         if (!spd.nr_pages) {
+                if (ret)
+                        goto out;
+
                 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) {
                         ret = -EAGAIN;
                         goto out;
                 }
                 mutex_unlock(&trace_types_lock);
-                ret = wait_on_pipe(iter);
+                ret = wait_on_pipe(iter, true);
                 mutex_lock(&trace_types_lock);
                 if (ret)
                         goto out;
-                if (signal_pending(current)) {
-                        ret = -EINTR;
-                        goto out;
-                }
+
                 goto again;
         }
 
diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c
index ef06ce7e9cf8..0cc51edde3a8 100644
--- a/kernel/trace/trace_events.c
+++ b/kernel/trace/trace_events.c
@@ -2513,8 +2513,11 @@ static __init int event_test_thread(void *unused)
         kfree(test_malloc);
 
         set_current_state(TASK_INTERRUPTIBLE);
-        while (!kthread_should_stop())
+        while (!kthread_should_stop()) {
                 schedule();
+                set_current_state(TASK_INTERRUPTIBLE);
+        }
+        __set_current_state(TASK_RUNNING);
 
         return 0;
 }
diff --git a/kernel/trace/trace_selftest.c b/kernel/trace/trace_selftest.c
index 5ef60499dc8e..b0f86ea77881 100644
--- a/kernel/trace/trace_selftest.c
+++ b/kernel/trace/trace_selftest.c
@@ -382,6 +382,8 @@ static int trace_selftest_startup_dynamic_tracing(struct tracer *trace,
 
         /* check the trace buffer */
         ret = trace_test_buffer(&tr->trace_buffer, &count);
+
+        ftrace_enabled = 1;
         tracing_start();
 
         /* we should only have one item */
@@ -679,6 +681,8 @@ trace_selftest_startup_function(struct tracer *trace, struct trace_array *tr)
 
         /* check the trace buffer */
         ret = trace_test_buffer(&tr->trace_buffer, &count);
+
+        ftrace_enabled = 1;
         trace->reset(tr);
         tracing_start();
 
@@ -1025,6 +1029,12 @@ trace_selftest_startup_nop(struct tracer *trace, struct trace_array *tr)
 #endif
 
 #ifdef CONFIG_SCHED_TRACER
+
+struct wakeup_test_data {
+        struct completion       is_ready;
+        int                     go;
+};
+
 static int trace_wakeup_test_thread(void *data)
 {
         /* Make this a -deadline thread */
@@ -1034,51 +1044,56 @@ static int trace_wakeup_test_thread(void *data)
                 .sched_deadline = 10000000ULL,
                 .sched_period = 10000000ULL
         };
-        struct completion *x = data;
+        struct wakeup_test_data *x = data;
 
         sched_setattr(current, &attr);
 
         /* Make it know we have a new prio */
-        complete(x);
+        complete(&x->is_ready);
 
         /* now go to sleep and let the test wake us up */
         set_current_state(TASK_INTERRUPTIBLE);
-        schedule();
+        while (!x->go) {
+                schedule();
+                set_current_state(TASK_INTERRUPTIBLE);
+        }
 
-        complete(x);
+        complete(&x->is_ready);
+
+        set_current_state(TASK_INTERRUPTIBLE);
 
         /* we are awake, now wait to disappear */
         while (!kthread_should_stop()) {
-                /*
-                 * This will likely be the system top priority
-                 * task, do short sleeps to let others run.
-                 */
-                msleep(100);
+                schedule();
+                set_current_state(TASK_INTERRUPTIBLE);
         }
 
+        __set_current_state(TASK_RUNNING);
+
         return 0;
 }
-
 int
 trace_selftest_startup_wakeup(struct tracer *trace, struct trace_array *tr)
 {
         unsigned long save_max = tr->max_latency;
         struct task_struct *p;
-        struct completion is_ready;
+        struct wakeup_test_data data;
         unsigned long count;
         int ret;
 
-        init_completion(&is_ready);
+        memset(&data, 0, sizeof(data));
+
+        init_completion(&data.is_ready);
 
         /* create a -deadline thread */
-        p = kthread_run(trace_wakeup_test_thread, &is_ready, "ftrace-test");
+        p = kthread_run(trace_wakeup_test_thread, &data, "ftrace-test");
         if (IS_ERR(p)) {
                 printk(KERN_CONT "Failed to create ftrace wakeup test thread ");
                 return -1;
         }
 
         /* make sure the thread is running at -deadline policy */
-        wait_for_completion(&is_ready);
+        wait_for_completion(&data.is_ready);
 
         /* start the tracing */
         ret = tracer_init(trace, tr);
@@ -1099,18 +1114,20 @@ trace_selftest_startup_wakeup(struct tracer *trace, struct trace_array *tr)
                 msleep(100);
         }
 
-        init_completion(&is_ready);
+        init_completion(&data.is_ready);
+
+        data.go = 1;
+        /* memory barrier is in the wake_up_process() */
 
         wake_up_process(p);
 
         /* Wait for the task to wake up */
-        wait_for_completion(&is_ready);
+        wait_for_completion(&data.is_ready);
 
         /* stop the tracing. */
         tracing_stop();
         /* check both trace buffers */
         ret = trace_test_buffer(&tr->trace_buffer, NULL);
-        printk("ret = %d\n", ret);
         if (!ret)
                 ret = trace_test_buffer(&tr->max_buffer, &count);
 
diff --git a/kernel/trace/trace_stack.c b/kernel/trace/trace_stack.c
index 8a4e5cb66a4c..16eddb308c33 100644
--- a/kernel/trace/trace_stack.c
+++ b/kernel/trace/trace_stack.c
@@ -13,7 +13,6 @@
 #include <linux/sysctl.h>
 #include <linux/init.h>
 #include <linux/fs.h>
-#include <linux/magic.h>
 
 #include <asm/setup.h>
 
@@ -171,8 +170,7 @@ check_stack(unsigned long ip, unsigned long *stack)
                         i++;
         }
 
-        if ((current != &init_task &&
-                *(end_of_stack(current)) != STACK_END_MAGIC)) {
+        if (task_stack_end_corrupted(current)) {
                 print_max_stack();
                 BUG();
         }
diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c
index 759d5e004517..29228c4d5696 100644
--- a/kernel/trace/trace_syscalls.c
+++ b/kernel/trace/trace_syscalls.c
@@ -313,7 +313,7 @@ static void ftrace_syscall_enter(void *data, struct pt_regs *regs, long id)
         int size;
 
         syscall_nr = trace_get_syscall_nr(current, regs);
-        if (syscall_nr < 0)
+        if (syscall_nr < 0 || syscall_nr >= NR_syscalls)
                 return;
 
         /* Here we're inside tp handler's rcu_read_lock_sched (__DO_TRACE) */
@@ -360,7 +360,7 @@ static void ftrace_syscall_exit(void *data, struct pt_regs *regs, long ret)
         int syscall_nr;
 
         syscall_nr = trace_get_syscall_nr(current, regs);
-        if (syscall_nr < 0)
+        if (syscall_nr < 0 || syscall_nr >= NR_syscalls)
                 return;
 
         /* Here we're inside tp handler's rcu_read_lock_sched (__DO_TRACE()) */
@@ -425,7 +425,7 @@ static void unreg_event_syscall_enter(struct ftrace_event_file *file,
                 return;
         mutex_lock(&syscall_trace_lock);
         tr->sys_refcount_enter--;
-        rcu_assign_pointer(tr->enter_syscall_files[num], NULL);
+        RCU_INIT_POINTER(tr->enter_syscall_files[num], NULL);
         if (!tr->sys_refcount_enter)
                 unregister_trace_sys_enter(ftrace_syscall_enter, tr);
         mutex_unlock(&syscall_trace_lock);
@@ -463,7 +463,7 @@ static void unreg_event_syscall_exit(struct ftrace_event_file *file,
                 return;
         mutex_lock(&syscall_trace_lock);
         tr->sys_refcount_exit--;
-        rcu_assign_pointer(tr->exit_syscall_files[num], NULL);
+        RCU_INIT_POINTER(tr->exit_syscall_files[num], NULL);
         if (!tr->sys_refcount_exit)
                 unregister_trace_sys_exit(ftrace_syscall_exit, tr);
         mutex_unlock(&syscall_trace_lock);
@@ -567,7 +567,7 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
         int size;
 
         syscall_nr = trace_get_syscall_nr(current, regs);
-        if (syscall_nr < 0)
+        if (syscall_nr < 0 || syscall_nr >= NR_syscalls)
                 return;
         if (!test_bit(syscall_nr, enabled_perf_enter_syscalls))
                 return;
@@ -641,7 +641,7 @@ static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
         int size;
 
         syscall_nr = trace_get_syscall_nr(current, regs);
-        if (syscall_nr < 0)
+        if (syscall_nr < 0 || syscall_nr >= NR_syscalls)
                 return;
         if (!test_bit(syscall_nr, enabled_perf_exit_syscalls))
                 return;