72 files changed, 3988 insertions, 2796 deletions
diff --git a/kernel/Makefile b/kernel/Makefile
index 470839d1a30e..35ef1185e359 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -2,7 +2,7 @@
 # Makefile for the linux kernel.
 #
-obj-y     = fork.o exec_domain.o panic.o printk.o \
+obj-y     = fork.o exec_domain.o panic.o \
            cpu.o exit.o itimer.o time.o softirq.o resource.o \
            sysctl.o sysctl_binary.o capability.o ptrace.o timer.o user.o \
            signal.o sys.o kmod.o workqueue.o pid.o task_work.o \
@@ -24,6 +24,7 @@ endif
 obj-y += sched/
 obj-y += power/
+obj-y += printk/
 obj-y += cpu/
 obj-$(CONFIG_CHECKPOINT_RESTORE) += kcmp.o
diff --git a/kernel/cgroup.c b/kernel/cgroup.c
index 0e0b20b8c5db..e0aeb32415ff 100644
--- a/kernel/cgroup.c
+++ b/kernel/cgroup.c
@@ -81,7 +81,7 @@
 */
 #ifdef CONFIG_PROVE_RCU
 DEFINE_MUTEX(cgroup_mutex);
-EXPORT_SYMBOL_GPL(cgroup_mutex);        /* only for task_subsys_state_check() */
+EXPORT_SYMBOL_GPL(cgroup_mutex);        /* only for lockdep */
 #else
 static DEFINE_MUTEX(cgroup_mutex);
 #endif
@@ -117,6 +117,7 @@ struct cfent {
        struct list_head                node;
        struct dentry                   *dentry;
        struct cftype                   *type;
+        struct cgroup_subsys_state      *css;
        /* file xattrs */
        struct simple_xattrs            xattrs;
@@ -159,9 +160,9 @@ struct css_id {
 */
 struct cgroup_event {
        /*
-         * Cgroup which the event belongs to.
+         * css which the event belongs to.
         */
-        struct cgroup *cgrp;
+        struct cgroup_subsys_state *css;
        /*
         * Control file which the event associated.
         */
@@ -215,10 +216,33 @@ static u64 cgroup_serial_nr_next = 1;
 */
 static int need_forkexit_callback __read_mostly;
-static void cgroup_offline_fn(struct work_struct *work);
+static struct cftype cgroup_base_files[];
+static void cgroup_destroy_css_killed(struct cgroup *cgrp);
 static int cgroup_destroy_locked(struct cgroup *cgrp);
-static int cgroup_addrm_files(struct cgroup *cgrp, struct cgroup_subsys *subsys,
+static int cgroup_addrm_files(struct cgroup *cgrp, struct cftype cfts[],
-                              struct cftype cfts[], bool is_add);
+                              bool is_add);
+/**
+ * cgroup_css - obtain a cgroup's css for the specified subsystem
+ * @cgrp: the cgroup of interest
+ * @ss: the subsystem of interest (%NULL returns the dummy_css)
+ *
+ * Return @cgrp's css (cgroup_subsys_state) associated with @ss.  This
+ * function must be called either under cgroup_mutex or rcu_read_lock() and
+ * the caller is responsible for pinning the returned css if it wants to
+ * keep accessing it outside the said locks.  This function may return
+ * %NULL if @cgrp doesn't have @subsys_id enabled.
+ */
+static struct cgroup_subsys_state *cgroup_css(struct cgroup *cgrp,
+                                              struct cgroup_subsys *ss)
+{
+        if (ss)
+                return rcu_dereference_check(cgrp->subsys[ss->subsys_id],
+                                             lockdep_is_held(&cgroup_mutex));
+        else
+                return &cgrp->dummy_css;
+}
 /* convenient tests for these bits */
 static inline bool cgroup_is_dead(const struct cgroup *cgrp)
@@ -365,9 +389,11 @@ static struct cgrp_cset_link init_cgrp_cset_link;
 static int cgroup_init_idr(struct cgroup_subsys *ss,
                           struct cgroup_subsys_state *css);
-/* css_set_lock protects the list of css_set objects, and the
+/*
- * chain of tasks off each css_set.  Nests outside task->alloc_lock
+ * css_set_lock protects the list of css_set objects, and the chain of
- * due to cgroup_iter_start() */
+ * tasks off each css_set.  Nests outside task->alloc_lock due to
+ * css_task_iter_start().
+ */
 static DEFINE_RWLOCK(css_set_lock);
 static int css_set_count;
@@ -392,10 +418,12 @@ static unsigned long css_set_hash(struct cgroup_subsys_state *css[])
        return key;
 }
-/* We don't maintain the lists running through each css_set to its
+/*
- * task until after the first call to cgroup_iter_start(). This
+ * We don't maintain the lists running through each css_set to its task
- * reduces the fork()/exit() overhead for people who have cgroups
+ * until after the first call to css_task_iter_start().  This reduces the
- * compiled into their kernel but not actually in use */
+ * fork()/exit() overhead for people who have cgroups compiled into their
+ * kernel but not actually in use.
+ */
 static int use_task_css_set_links __read_mostly;
 static void __put_css_set(struct css_set *cset, int taskexit)
@@ -464,7 +492,7 @@ static inline void put_css_set_taskexit(struct css_set *cset)
 * @new_cgrp: cgroup that's being entered by the task
 * @template: desired set of css pointers in css_set (pre-calculated)
 *
- * Returns true if "cg" matches "old_cg" except for the hierarchy
+ * Returns true if "cset" matches "old_cset" except for the hierarchy
 * which "new_cgrp" belongs to, for which it should match "new_cgrp".
 */
 static bool compare_css_sets(struct css_set *cset,
@@ -555,7 +583,7 @@ static struct css_set *find_existing_css_set(struct css_set *old_cset,
                        /* Subsystem is in this hierarchy. So we want
                         * the subsystem state from the new
                         * cgroup */
-                        template[i] = cgrp->subsys[i];
+                        template[i] = cgroup_css(cgrp, ss);
                } else {
                        /* Subsystem is not in this hierarchy, so we
                         * don't want to change the subsystem state */
@@ -803,8 +831,7 @@ static struct cgroup *task_cgroup_from_root(struct task_struct *task,
 static int cgroup_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode);
 static int cgroup_rmdir(struct inode *unused_dir, struct dentry *dentry);
-static int cgroup_populate_dir(struct cgroup *cgrp, bool base_files,
+static int cgroup_populate_dir(struct cgroup *cgrp, unsigned long subsys_mask);
-                               unsigned long subsys_mask);
 static const struct inode_operations cgroup_dir_inode_operations;
 static const struct file_operations proc_cgroupstats_operations;
@@ -813,8 +840,7 @@ static struct backing_dev_info cgroup_backing_dev_info = {
        .capabilities   = BDI_CAP_NO_ACCT_AND_WRITEBACK,
 };
-static int alloc_css_id(struct cgroup_subsys *ss,
+static int alloc_css_id(struct cgroup_subsys_state *child_css);
-                        struct cgroup *parent, struct cgroup *child);
 static struct inode *cgroup_new_inode(umode_t mode, struct super_block *sb)
 {
@@ -845,15 +871,8 @@ static struct cgroup_name *cgroup_alloc_name(struct dentry *dentry)
 static void cgroup_free_fn(struct work_struct *work)
 {
        struct cgroup *cgrp = container_of(work, struct cgroup, destroy_work);
-        struct cgroup_subsys *ss;
        mutex_lock(&cgroup_mutex);
-        /*
-         * Release the subsystem state objects.
-         */
-        for_each_root_subsys(cgrp->root, ss)
-                ss->css_free(cgrp);
        cgrp->root->number_of_cgroups--;
        mutex_unlock(&cgroup_mutex);
@@ -864,8 +883,6 @@ static void cgroup_free_fn(struct work_struct *work)
         */
        dput(cgrp->parent->dentry);
-        ida_simple_remove(&cgrp->root->cgroup_ida, cgrp->id);
        /*
         * Drop the active superblock reference that we took when we
         * created the cgroup. This will free cgrp->root, if we are
@@ -956,27 +973,22 @@ static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft)
 }
 /**
- * cgroup_clear_directory - selective removal of base and subsystem files
+ * cgroup_clear_dir - remove subsys files in a cgroup directory
- * @dir: directory containing the files
+ * @cgrp: target cgroup
- * @base_files: true if the base files should be removed
 * @subsys_mask: mask of the subsystem ids whose files should be removed
 */
-static void cgroup_clear_directory(struct dentry *dir, bool base_files,
+static void cgroup_clear_dir(struct cgroup *cgrp, unsigned long subsys_mask)
-                                   unsigned long subsys_mask)
 {
-        struct cgroup *cgrp = __d_cgrp(dir);
        struct cgroup_subsys *ss;
+        int i;
-        for_each_root_subsys(cgrp->root, ss) {
+        for_each_subsys(ss, i) {
                struct cftype_set *set;
-                if (!test_bit(ss->subsys_id, &subsys_mask))
+                if (!test_bit(i, &subsys_mask))
                        continue;
                list_for_each_entry(set, &ss->cftsets, node)
-                        cgroup_addrm_files(cgrp, NULL, set->cfts, false);
+                        cgroup_addrm_files(cgrp, set->cfts, false);
-        }
-        if (base_files) {
-                while (!list_empty(&cgrp->files))
-                        cgroup_rm_file(cgrp, NULL);
        }
 }
@@ -986,9 +998,6 @@ static void cgroup_clear_directory(struct dentry *dir, bool base_files,
 static void cgroup_d_remove_dir(struct dentry *dentry)
 {
        struct dentry *parent;
-        struct cgroupfs_root *root = dentry->d_sb->s_fs_info;
-        cgroup_clear_directory(dentry, true, root->subsys_mask);
        parent = dentry->d_parent;
        spin_lock(&parent->d_lock);
@@ -1009,79 +1018,84 @@ static int rebind_subsystems(struct cgroupfs_root *root,
 {
        struct cgroup *cgrp = &root->top_cgroup;
        struct cgroup_subsys *ss;
-        int i;
+        unsigned long pinned = 0;
+        int i, ret;
        BUG_ON(!mutex_is_locked(&cgroup_mutex));
        BUG_ON(!mutex_is_locked(&cgroup_root_mutex));
        /* Check that any added subsystems are currently free */
        for_each_subsys(ss, i) {
-                unsigned long bit = 1UL << i;
+                if (!(added_mask & (1 << i)))
-                if (!(bit & added_mask))
                        continue;
+                /* is the subsystem mounted elsewhere? */
                if (ss->root != &cgroup_dummy_root) {
-                        /* Subsystem isn't free */
+                        ret = -EBUSY;
-                        return -EBUSY;
+                        goto out_put;
                }
+                /* pin the module */
+                if (!try_module_get(ss->module)) {
+                        ret = -ENOENT;
+                        goto out_put;
+                }
+                pinned |= 1 << i;
        }
-        /* Currently we don't handle adding/removing subsystems when
+        /* subsys could be missing if unloaded between parsing and here */
-         * any child cgroups exist. This is theoretically supportable
+        if (added_mask != pinned) {
-         * but involves complex error handling, so it's being left until
+                ret = -ENOENT;
-         * later */
+                goto out_put;
-        if (root->number_of_cgroups > 1)
+        }
-                return -EBUSY;
+        ret = cgroup_populate_dir(cgrp, added_mask);
+        if (ret)
+                goto out_put;
+        /*
+         * Nothing can fail from this point on.  Remove files for the
+         * removed subsystems and rebind each subsystem.
+         */
+        cgroup_clear_dir(cgrp, removed_mask);
-        /* Process each subsystem */
        for_each_subsys(ss, i) {
                unsigned long bit = 1UL << i;
                if (bit & added_mask) {
                        /* We're binding this subsystem to this hierarchy */
-                        BUG_ON(cgrp->subsys[i]);
+                        BUG_ON(cgroup_css(cgrp, ss));
-                        BUG_ON(!cgroup_dummy_top->subsys[i]);
+                        BUG_ON(!cgroup_css(cgroup_dummy_top, ss));
-                        BUG_ON(cgroup_dummy_top->subsys[i]->cgroup != cgroup_dummy_top);
+                        BUG_ON(cgroup_css(cgroup_dummy_top, ss)->cgroup != cgroup_dummy_top);
+                        rcu_assign_pointer(cgrp->subsys[i],
+                                           cgroup_css(cgroup_dummy_top, ss));
+                        cgroup_css(cgrp, ss)->cgroup = cgrp;
-                        cgrp->subsys[i] = cgroup_dummy_top->subsys[i];
-                        cgrp->subsys[i]->cgroup = cgrp;
                        list_move(&ss->sibling, &root->subsys_list);
                        ss->root = root;
                        if (ss->bind)
-                                ss->bind(cgrp);
+                                ss->bind(cgroup_css(cgrp, ss));
                        /* refcount was already taken, and we're keeping it */
                        root->subsys_mask |= bit;
                } else if (bit & removed_mask) {
                        /* We're removing this subsystem */
-                        BUG_ON(cgrp->subsys[i] != cgroup_dummy_top->subsys[i]);
+                        BUG_ON(cgroup_css(cgrp, ss) != cgroup_css(cgroup_dummy_top, ss));
-                        BUG_ON(cgrp->subsys[i]->cgroup != cgrp);
+                        BUG_ON(cgroup_css(cgrp, ss)->cgroup != cgrp);
                        if (ss->bind)
-                                ss->bind(cgroup_dummy_top);
+                                ss->bind(cgroup_css(cgroup_dummy_top, ss));
-                        cgroup_dummy_top->subsys[i]->cgroup = cgroup_dummy_top;
-                        cgrp->subsys[i] = NULL;
+                        cgroup_css(cgroup_dummy_top, ss)->cgroup = cgroup_dummy_top;
+                        RCU_INIT_POINTER(cgrp->subsys[i], NULL);
                        cgroup_subsys[i]->root = &cgroup_dummy_root;
                        list_move(&ss->sibling, &cgroup_dummy_root.subsys_list);
                        /* subsystem is now free - drop reference on module */
                        module_put(ss->module);
                        root->subsys_mask &= ~bit;
-                } else if (bit & root->subsys_mask) {
-                        /* Subsystem state should already exist */
-                        BUG_ON(!cgrp->subsys[i]);
-                        /*
-                         * a refcount was taken, but we already had one, so
-                         * drop the extra reference.
-                         */
-                        module_put(ss->module);
-#ifdef CONFIG_MODULE_UNLOAD
-                        BUG_ON(ss->module && !module_refcount(ss->module));
-#endif
-                } else {
-                        /* Subsystem state shouldn't exist */
-                        BUG_ON(cgrp->subsys[i]);
                }
        }
@@ -1092,6 +1106,12 @@ static int rebind_subsystems(struct cgroupfs_root *root,
        root->flags |= CGRP_ROOT_SUBSYS_BOUND;
        return 0;
+out_put:
+        for_each_subsys(ss, i)
+                if (pinned & (1 << i))
+                        module_put(ss->module);
+        return ret;
 }
 static int cgroup_show_options(struct seq_file *seq, struct dentry *dentry)
@@ -1142,7 +1162,6 @@ static int parse_cgroupfs_options(char *data, struct cgroup_sb_opts *opts)
        char *token, *o = data;
        bool all_ss = false, one_ss = false;
        unsigned long mask = (unsigned long)-1;
-        bool module_pin_failed = false;
        struct cgroup_subsys *ss;
        int i;
@@ -1285,52 +1304,9 @@ static int parse_cgroupfs_options(char *data, struct cgroup_sb_opts *opts)
        if (!opts->subsys_mask && !opts->name)
                return -EINVAL;
-        /*
-         * Grab references on all the modules we'll need, so the subsystems
-         * don't dance around before rebind_subsystems attaches them. This may
-         * take duplicate reference counts on a subsystem that's already used,
-         * but rebind_subsystems handles this case.
-         */
-        for_each_subsys(ss, i) {
-                if (!(opts->subsys_mask & (1UL << i)))
-                        continue;
-                if (!try_module_get(cgroup_subsys[i]->module)) {
-                        module_pin_failed = true;
-                        break;
-                }
-        }
-        if (module_pin_failed) {
-                /*
-                 * oops, one of the modules was going away. this means that we
-                 * raced with a module_delete call, and to the user this is
-                 * essentially a "subsystem doesn't exist" case.
-                 */
-                for (i--; i >= 0; i--) {
-                        /* drop refcounts only on the ones we took */
-                        unsigned long bit = 1UL << i;
-                        if (!(bit & opts->subsys_mask))
-                                continue;
-                        module_put(cgroup_subsys[i]->module);
-                }
-                return -ENOENT;
-        }
        return 0;
 }
-static void drop_parsed_module_refcounts(unsigned long subsys_mask)
-{
-        struct cgroup_subsys *ss;
-        int i;
-        mutex_lock(&cgroup_mutex);
-        for_each_subsys(ss, i)
-                if (subsys_mask & (1UL << i))
-                        module_put(cgroup_subsys[i]->module);
-        mutex_unlock(&cgroup_mutex);
-}
 static int cgroup_remount(struct super_block *sb, int *flags, char *data)
 {
        int ret = 0;
@@ -1370,22 +1346,15 @@ static int cgroup_remount(struct super_block *sb, int *flags, char *data)
                goto out_unlock;
        }
-        /*
+        /* remounting is not allowed for populated hierarchies */
-         * Clear out the files of subsystems that should be removed, do
+        if (root->number_of_cgroups > 1) {
-         * this before rebind_subsystems, since rebind_subsystems may
+                ret = -EBUSY;
-         * change this hierarchy's subsys_list.
-         */
-        cgroup_clear_directory(cgrp->dentry, false, removed_mask);
-        ret = rebind_subsystems(root, added_mask, removed_mask);
-        if (ret) {
-                /* rebind_subsystems failed, re-populate the removed files */
-                cgroup_populate_dir(cgrp, false, removed_mask);
                goto out_unlock;
        }
-        /* re-populate subsystem files */
+        ret = rebind_subsystems(root, added_mask, removed_mask);
-        cgroup_populate_dir(cgrp, false, added_mask);
+        if (ret)
+                goto out_unlock;
        if (opts.release_agent)
                strcpy(root->release_agent_path, opts.release_agent);
@@ -1395,8 +1364,6 @@ static int cgroup_remount(struct super_block *sb, int *flags, char *data)
        mutex_unlock(&cgroup_root_mutex);
        mutex_unlock(&cgroup_mutex);
        mutex_unlock(&cgrp->dentry->d_inode->i_mutex);
-        if (ret)
-                drop_parsed_module_refcounts(opts.subsys_mask);
        return ret;
 }
@@ -1416,6 +1383,7 @@ static void init_cgroup_housekeeping(struct cgroup *cgrp)
        INIT_LIST_HEAD(&cgrp->release_list);
        INIT_LIST_HEAD(&cgrp->pidlists);
        mutex_init(&cgrp->pidlist_mutex);
+        cgrp->dummy_css.cgroup = cgrp;
        INIT_LIST_HEAD(&cgrp->event_list);
        spin_lock_init(&cgrp->event_list_lock);
        simple_xattrs_init(&cgrp->xattrs);
@@ -1431,6 +1399,7 @@ static void init_cgroup_root(struct cgroupfs_root *root)
        cgrp->root = root;
        RCU_INIT_POINTER(cgrp->name, &root_cgroup_name);
        init_cgroup_housekeeping(cgrp);
+        idr_init(&root->cgroup_idr);
 }
 static int cgroup_init_root_id(struct cgroupfs_root *root, int start, int end)
@@ -1503,7 +1472,6 @@ static struct cgroupfs_root *cgroup_root_from_opts(struct cgroup_sb_opts *opts)
         */
        root->subsys_mask = opts->subsys_mask;
        root->flags = opts->flags;
-        ida_init(&root->cgroup_ida);
        if (opts->release_agent)
                strcpy(root->release_agent_path, opts->release_agent);
        if (opts->name)
@@ -1519,7 +1487,7 @@ static void cgroup_free_root(struct cgroupfs_root *root)
                /* hierarhcy ID shoulid already have been released */
                WARN_ON_ONCE(root->hierarchy_id);
-                ida_destroy(&root->cgroup_ida);
+                idr_destroy(&root->cgroup_idr);
                kfree(root);
        }
 }
@@ -1584,7 +1552,9 @@ static struct dentry *cgroup_mount(struct file_system_type *fs_type,
        int ret = 0;
        struct super_block *sb;
        struct cgroupfs_root *new_root;
+        struct list_head tmp_links;
        struct inode *inode;
+        const struct cred *cred;
        /* First find the desired set of subsystems */
        mutex_lock(&cgroup_mutex);
@@ -1600,7 +1570,7 @@ static struct dentry *cgroup_mount(struct file_system_type *fs_type,
        new_root = cgroup_root_from_opts(&opts);
        if (IS_ERR(new_root)) {
                ret = PTR_ERR(new_root);
-                goto drop_modules;
+                goto out_err;
        }
        opts.new_root = new_root;
@@ -1609,17 +1579,15 @@ static struct dentry *cgroup_mount(struct file_system_type *fs_type,
        if (IS_ERR(sb)) {
                ret = PTR_ERR(sb);
                cgroup_free_root(opts.new_root);
-                goto drop_modules;
+                goto out_err;
        }
        root = sb->s_fs_info;
        BUG_ON(!root);
        if (root == opts.new_root) {
                /* We used the new root structure, so this is a new hierarchy */
-                struct list_head tmp_links;
                struct cgroup *root_cgrp = &root->top_cgroup;
                struct cgroupfs_root *existing_root;
-                const struct cred *cred;
                int i;
                struct css_set *cset;
@@ -1634,6 +1602,11 @@ static struct dentry *cgroup_mount(struct file_system_type *fs_type,
                mutex_lock(&cgroup_mutex);
                mutex_lock(&cgroup_root_mutex);
+                root_cgrp->id = idr_alloc(&root->cgroup_idr, root_cgrp,
+                                           0, 1, GFP_KERNEL);
+                if (root_cgrp->id < 0)
+                        goto unlock_drop;
                /* Check for name clashes with existing mounts */
                ret = -EBUSY;
                if (strlen(root->name))
@@ -1657,26 +1630,37 @@ static struct dentry *cgroup_mount(struct file_system_type *fs_type,
                if (ret)
                        goto unlock_drop;
+                sb->s_root->d_fsdata = root_cgrp;
+                root_cgrp->dentry = sb->s_root;
+                /*
+                 * We're inside get_sb() and will call lookup_one_len() to
+                 * create the root files, which doesn't work if SELinux is
+                 * in use.  The following cred dancing somehow works around
+                 * it.  See 2ce9738ba ("cgroupfs: use init_cred when
+                 * populating new cgroupfs mount") for more details.
+                 */
+                cred = override_creds(&init_cred);
+                ret = cgroup_addrm_files(root_cgrp, cgroup_base_files, true);
+                if (ret)
+                        goto rm_base_files;
                ret = rebind_subsystems(root, root->subsys_mask, 0);
-                if (ret == -EBUSY) {
+                if (ret)
-                        free_cgrp_cset_links(&tmp_links);
+                        goto rm_base_files;
-                        goto unlock_drop;
-                }
+                revert_creds(cred);
                /*
                 * There must be no failure case after here, since rebinding
                 * takes care of subsystems' refcounts, which are explicitly
                 * dropped in the failure exit path.
                 */
-                /* EBUSY should be the only error here */
-                BUG_ON(ret);
                list_add(&root->root_list, &cgroup_roots);
                cgroup_root_count++;
-                sb->s_root->d_fsdata = root_cgrp;
-                root->top_cgroup.dentry = sb->s_root;
                /* Link the top cgroup in this hierarchy into all
                 * the css_set objects */
                write_lock(&css_set_lock);
@@ -1689,9 +1673,6 @@ static struct dentry *cgroup_mount(struct file_system_type *fs_type,
                BUG_ON(!list_empty(&root_cgrp->children));
                BUG_ON(root->number_of_cgroups != 1);
-                cred = override_creds(&init_cred);
-                cgroup_populate_dir(root_cgrp, true, root->subsys_mask);
-                revert_creds(cred);
                mutex_unlock(&cgroup_root_mutex);
                mutex_unlock(&cgroup_mutex);
                mutex_unlock(&inode->i_mutex);
@@ -1711,15 +1692,16 @@ static struct dentry *cgroup_mount(struct file_system_type *fs_type,
                                pr_warning("cgroup: new mount options do not match the existing superblock, will be ignored\n");
                        }
                }
-                /* no subsys rebinding, so refcounts don't change */
-                drop_parsed_module_refcounts(opts.subsys_mask);
        }
        kfree(opts.release_agent);
        kfree(opts.name);
        return dget(sb->s_root);
+ rm_base_files:
+        free_cgrp_cset_links(&tmp_links);
+        cgroup_addrm_files(&root->top_cgroup, cgroup_base_files, false);
+        revert_creds(cred);
 unlock_drop:
        cgroup_exit_root_id(root);
        mutex_unlock(&cgroup_root_mutex);
@@ -1727,8 +1709,6 @@ static struct dentry *cgroup_mount(struct file_system_type *fs_type,
        mutex_unlock(&inode->i_mutex);
 drop_new_super:
        deactivate_locked_super(sb);
- drop_modules:
-        drop_parsed_module_refcounts(opts.subsys_mask);
 out_err:
        kfree(opts.release_agent);
        kfree(opts.name);
@@ -1746,6 +1726,7 @@ static void cgroup_kill_sb(struct super_block *sb) {
        BUG_ON(root->number_of_cgroups != 1);
        BUG_ON(!list_empty(&cgrp->children));
+        mutex_lock(&cgrp->dentry->d_inode->i_mutex);
        mutex_lock(&cgroup_mutex);
        mutex_lock(&cgroup_root_mutex);
@@ -1778,6 +1759,7 @@ static void cgroup_kill_sb(struct super_block *sb) {
        mutex_unlock(&cgroup_root_mutex);
        mutex_unlock(&cgroup_mutex);
+        mutex_unlock(&cgrp->dentry->d_inode->i_mutex);
        simple_xattrs_free(&cgrp->xattrs);
@@ -1845,36 +1827,43 @@ out:
 EXPORT_SYMBOL_GPL(cgroup_path);
 /**
- * task_cgroup_path_from_hierarchy - cgroup path of a task on a hierarchy
+ * task_cgroup_path - cgroup path of a task in the first cgroup hierarchy
 * @task: target task
- * @hierarchy_id: the hierarchy to look up @task's cgroup from
 * @buf: the buffer to write the path into
 * @buflen: the length of the buffer
 *
- * Determine @task's cgroup on the hierarchy specified by @hierarchy_id and
+ * Determine @task's cgroup on the first (the one with the lowest non-zero
- * copy its path into @buf.  This function grabs cgroup_mutex and shouldn't
+ * hierarchy_id) cgroup hierarchy and copy its path into @buf.  This
- * be used inside locks used by cgroup controller callbacks.
+ * function grabs cgroup_mutex and shouldn't be used inside locks used by
+ * cgroup controller callbacks.
+ *
+ * Returns 0 on success, fails with -%ENAMETOOLONG if @buflen is too short.
 */
-int task_cgroup_path_from_hierarchy(struct task_struct *task, int hierarchy_id,
+int task_cgroup_path(struct task_struct *task, char *buf, size_t buflen)
-                                    char *buf, size_t buflen)
 {
        struct cgroupfs_root *root;
-        struct cgroup *cgrp = NULL;
+        struct cgroup *cgrp;
-        int ret = -ENOENT;
+        int hierarchy_id = 1, ret = 0;
+        if (buflen < 2)
+                return -ENAMETOOLONG;
        mutex_lock(&cgroup_mutex);
-        root = idr_find(&cgroup_hierarchy_idr, hierarchy_id);
+        root = idr_get_next(&cgroup_hierarchy_idr, &hierarchy_id);
        if (root) {
                cgrp = task_cgroup_from_root(task, root);
                ret = cgroup_path(cgrp, buf, buflen);
+        } else {
+                /* if no hierarchy exists, everyone is in "/" */
+                memcpy(buf, "/", 2);
        }
        mutex_unlock(&cgroup_mutex);
        return ret;
 }
-EXPORT_SYMBOL_GPL(task_cgroup_path_from_hierarchy);
+EXPORT_SYMBOL_GPL(task_cgroup_path);
 /*
 * Control Group taskset
@@ -1882,7 +1871,7 @@ EXPORT_SYMBOL_GPL(task_cgroup_path_from_hierarchy);
 struct task_and_cgroup {
        struct task_struct      *task;
        struct cgroup           *cgrp;
-        struct css_set          *cg;
+        struct css_set          *cset;
 };
 struct cgroup_taskset {
@@ -1932,18 +1921,20 @@ struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset)
 EXPORT_SYMBOL_GPL(cgroup_taskset_next);
 /**
- * cgroup_taskset_cur_cgroup - return the matching cgroup for the current task
+ * cgroup_taskset_cur_css - return the matching css for the current task
 * @tset: taskset of interest
+ * @subsys_id: the ID of the target subsystem
 *
- * Return the cgroup for the current (last returned) task of @tset.  This
+ * Return the css for the current (last returned) task of @tset for
- * function must be preceded by either cgroup_taskset_first() or
+ * subsystem specified by @subsys_id.  This function must be preceded by
- * cgroup_taskset_next().
+ * either cgroup_taskset_first() or cgroup_taskset_next().
 */
-struct cgroup *cgroup_taskset_cur_cgroup(struct cgroup_taskset *tset)
+struct cgroup_subsys_state *cgroup_taskset_cur_css(struct cgroup_taskset *tset,
+                                                   int subsys_id)
 {
-        return tset->cur_cgrp;
+        return cgroup_css(tset->cur_cgrp, cgroup_subsys[subsys_id]);
 }
-EXPORT_SYMBOL_GPL(cgroup_taskset_cur_cgroup);
+EXPORT_SYMBOL_GPL(cgroup_taskset_cur_css);
 /**
 * cgroup_taskset_size - return the number of tasks in taskset
@@ -2082,8 +2073,10 @@ static int cgroup_attach_task(struct cgroup *cgrp, struct task_struct *tsk,
         * step 1: check that we can legitimately attach to the cgroup.
         */
        for_each_root_subsys(root, ss) {
+                struct cgroup_subsys_state *css = cgroup_css(cgrp, ss);
                if (ss->can_attach) {
-                        retval = ss->can_attach(cgrp, &tset);
+                        retval = ss->can_attach(css, &tset);
                        if (retval) {
                                failed_ss = ss;
                                goto out_cancel_attach;
@@ -2100,8 +2093,8 @@ static int cgroup_attach_task(struct cgroup *cgrp, struct task_struct *tsk,
                tc = flex_array_get(group, i);
                old_cset = task_css_set(tc->task);
-                tc->cg = find_css_set(old_cset, cgrp);
+                tc->cset = find_css_set(old_cset, cgrp);
-                if (!tc->cg) {
+                if (!tc->cset) {
                        retval = -ENOMEM;
                        goto out_put_css_set_refs;
                }
@@ -2114,7 +2107,7 @@ static int cgroup_attach_task(struct cgroup *cgrp, struct task_struct *tsk,
         */
        for (i = 0; i < group_size; i++) {
                tc = flex_array_get(group, i);
-                cgroup_task_migrate(tc->cgrp, tc->task, tc->cg);
+                cgroup_task_migrate(tc->cgrp, tc->task, tc->cset);
        }
        /* nothing is sensitive to fork() after this point. */
@@ -2122,8 +2115,10 @@ static int cgroup_attach_task(struct cgroup *cgrp, struct task_struct *tsk,
         * step 4: do subsystem attach callbacks.
         */
        for_each_root_subsys(root, ss) {
+                struct cgroup_subsys_state *css = cgroup_css(cgrp, ss);
                if (ss->attach)
-                        ss->attach(cgrp, &tset);
+                        ss->attach(css, &tset);
        }
        /*
@@ -2134,18 +2129,20 @@ out_put_css_set_refs:
        if (retval) {
                for (i = 0; i < group_size; i++) {
                        tc = flex_array_get(group, i);
-                        if (!tc->cg)
+                        if (!tc->cset)
                                break;
-                        put_css_set(tc->cg);
+                        put_css_set(tc->cset);
                }
        }
 out_cancel_attach:
        if (retval) {
                for_each_root_subsys(root, ss) {
+                        struct cgroup_subsys_state *css = cgroup_css(cgrp, ss);
                        if (ss == failed_ss)
                                break;
                        if (ss->cancel_attach)
-                                ss->cancel_attach(cgrp, &tset);
+                                ss->cancel_attach(css, &tset);
                }
        }
 out_free_group_list:
@@ -2246,9 +2243,9 @@ int cgroup_attach_task_all(struct task_struct *from, struct task_struct *tsk)
        mutex_lock(&cgroup_mutex);
        for_each_active_root(root) {
-                struct cgroup *from_cg = task_cgroup_from_root(from, root);
+                struct cgroup *from_cgrp = task_cgroup_from_root(from, root);
-                retval = cgroup_attach_task(from_cg, tsk, false);
+                retval = cgroup_attach_task(from_cgrp, tsk, false);
                if (retval)
                        break;
        }
@@ -2258,34 +2255,38 @@ int cgroup_attach_task_all(struct task_struct *from, struct task_struct *tsk)
 }
 EXPORT_SYMBOL_GPL(cgroup_attach_task_all);
-static int cgroup_tasks_write(struct cgroup *cgrp, struct cftype *cft, u64 pid)
+static int cgroup_tasks_write(struct cgroup_subsys_state *css,
+                              struct cftype *cft, u64 pid)
 {
-        return attach_task_by_pid(cgrp, pid, false);
+        return attach_task_by_pid(css->cgroup, pid, false);
 }
-static int cgroup_procs_write(struct cgroup *cgrp, struct cftype *cft, u64 tgid)
+static int cgroup_procs_write(struct cgroup_subsys_state *css,
+                              struct cftype *cft, u64 tgid)
 {
-        return attach_task_by_pid(cgrp, tgid, true);
+        return attach_task_by_pid(css->cgroup, tgid, true);
 }
-static int cgroup_release_agent_write(struct cgroup *cgrp, struct cftype *cft,
+static int cgroup_release_agent_write(struct cgroup_subsys_state *css,
-                                      const char *buffer)
+                                      struct cftype *cft, const char *buffer)
 {
-        BUILD_BUG_ON(sizeof(cgrp->root->release_agent_path) < PATH_MAX);
+        BUILD_BUG_ON(sizeof(css->cgroup->root->release_agent_path) < PATH_MAX);
        if (strlen(buffer) >= PATH_MAX)
                return -EINVAL;
-        if (!cgroup_lock_live_group(cgrp))
+        if (!cgroup_lock_live_group(css->cgroup))
                return -ENODEV;
        mutex_lock(&cgroup_root_mutex);
-        strcpy(cgrp->root->release_agent_path, buffer);
+        strcpy(css->cgroup->root->release_agent_path, buffer);
        mutex_unlock(&cgroup_root_mutex);
        mutex_unlock(&cgroup_mutex);
        return 0;
 }
-static int cgroup_release_agent_show(struct cgroup *cgrp, struct cftype *cft,
+static int cgroup_release_agent_show(struct cgroup_subsys_state *css,
-                                     struct seq_file *seq)
+                                     struct cftype *cft, struct seq_file *seq)
 {
+        struct cgroup *cgrp = css->cgroup;
        if (!cgroup_lock_live_group(cgrp))
                return -ENODEV;
        seq_puts(seq, cgrp->root->release_agent_path);
@@ -2294,20 +2295,20 @@ static int cgroup_release_agent_show(struct cgroup *cgrp, struct cftype *cft,
        return 0;
 }
-static int cgroup_sane_behavior_show(struct cgroup *cgrp, struct cftype *cft,
+static int cgroup_sane_behavior_show(struct cgroup_subsys_state *css,
-                                     struct seq_file *seq)
+                                     struct cftype *cft, struct seq_file *seq)
 {
-        seq_printf(seq, "%d\n", cgroup_sane_behavior(cgrp));
+        seq_printf(seq, "%d\n", cgroup_sane_behavior(css->cgroup));
        return 0;
 }
 /* A buffer size big enough for numbers or short strings */
 #define CGROUP_LOCAL_BUFFER_SIZE 64
-static ssize_t cgroup_write_X64(struct cgroup *cgrp, struct cftype *cft,
+static ssize_t cgroup_write_X64(struct cgroup_subsys_state *css,
-                                struct file *file,
+                                struct cftype *cft, struct file *file,
-                                const char __user *userbuf,
+                                const char __user *userbuf, size_t nbytes,
-                                size_t nbytes, loff_t *unused_ppos)
+                                loff_t *unused_ppos)
 {
        char buffer[CGROUP_LOCAL_BUFFER_SIZE];
        int retval = 0;
@@ -2325,22 +2326,22 @@ static ssize_t cgroup_write_X64(struct cgroup *cgrp, struct cftype *cft,
                u64 val = simple_strtoull(strstrip(buffer), &end, 0);
                if (*end)
                        return -EINVAL;
-                retval = cft->write_u64(cgrp, cft, val);
+                retval = cft->write_u64(css, cft, val);
        } else {
                s64 val = simple_strtoll(strstrip(buffer), &end, 0);
                if (*end)
                        return -EINVAL;
-                retval = cft->write_s64(cgrp, cft, val);
+                retval = cft->write_s64(css, cft, val);
        }
        if (!retval)
                retval = nbytes;
        return retval;
 }
-static ssize_t cgroup_write_string(struct cgroup *cgrp, struct cftype *cft,
+static ssize_t cgroup_write_string(struct cgroup_subsys_state *css,
-                                   struct file *file,
+                                   struct cftype *cft, struct file *file,
-                                   const char __user *userbuf,
+                                   const char __user *userbuf, size_t nbytes,
-                                   size_t nbytes, loff_t *unused_ppos)
+                                   loff_t *unused_ppos)
 {
        char local_buffer[CGROUP_LOCAL_BUFFER_SIZE];
        int retval = 0;
@@ -2363,7 +2364,7 @@ static ssize_t cgroup_write_string(struct cgroup *cgrp, struct cftype *cft,
        }
        buffer[nbytes] = 0;     /* nul-terminate */
-        retval = cft->write_string(cgrp, cft, strstrip(buffer));
+        retval = cft->write_string(css, cft, strstrip(buffer));
        if (!retval)
                retval = nbytes;
 out:
@@ -2373,65 +2374,60 @@ out:
 }
 static ssize_t cgroup_file_write(struct file *file, const char __user *buf,
-                                                size_t nbytes, loff_t *ppos)
+                                 size_t nbytes, loff_t *ppos)
 {
+        struct cfent *cfe = __d_cfe(file->f_dentry);
        struct cftype *cft = __d_cft(file->f_dentry);
-        struct cgroup *cgrp = __d_cgrp(file->f_dentry->d_parent);
+        struct cgroup_subsys_state *css = cfe->css;
-        if (cgroup_is_dead(cgrp))
-                return -ENODEV;
        if (cft->write)
-                return cft->write(cgrp, cft, file, buf, nbytes, ppos);
+                return cft->write(css, cft, file, buf, nbytes, ppos);
        if (cft->write_u64 || cft->write_s64)
-                return cgroup_write_X64(cgrp, cft, file, buf, nbytes, ppos);
+                return cgroup_write_X64(css, cft, file, buf, nbytes, ppos);
        if (cft->write_string)
-                return cgroup_write_string(cgrp, cft, file, buf, nbytes, ppos);
+                return cgroup_write_string(css, cft, file, buf, nbytes, ppos);
        if (cft->trigger) {
-                int ret = cft->trigger(cgrp, (unsigned int)cft->private);
+                int ret = cft->trigger(css, (unsigned int)cft->private);
                return ret ? ret : nbytes;
        }
        return -EINVAL;
 }
-static ssize_t cgroup_read_u64(struct cgroup *cgrp, struct cftype *cft,
+static ssize_t cgroup_read_u64(struct cgroup_subsys_state *css,
-                               struct file *file,
+                               struct cftype *cft, struct file *file,
-                               char __user *buf, size_t nbytes,
+                               char __user *buf, size_t nbytes, loff_t *ppos)
-                               loff_t *ppos)
 {
        char tmp[CGROUP_LOCAL_BUFFER_SIZE];
-        u64 val = cft->read_u64(cgrp, cft);
+        u64 val = cft->read_u64(css, cft);
        int len = sprintf(tmp, "%llu\n", (unsigned long long) val);
        return simple_read_from_buffer(buf, nbytes, ppos, tmp, len);
 }
-static ssize_t cgroup_read_s64(struct cgroup *cgrp, struct cftype *cft,
+static ssize_t cgroup_read_s64(struct cgroup_subsys_state *css,
-                               struct file *file,
+                               struct cftype *cft, struct file *file,
-                               char __user *buf, size_t nbytes,
+                               char __user *buf, size_t nbytes, loff_t *ppos)
-                               loff_t *ppos)
 {
        char tmp[CGROUP_LOCAL_BUFFER_SIZE];
-        s64 val = cft->read_s64(cgrp, cft);
+        s64 val = cft->read_s64(css, cft);
        int len = sprintf(tmp, "%lld\n", (long long) val);
        return simple_read_from_buffer(buf, nbytes, ppos, tmp, len);
 }
 static ssize_t cgroup_file_read(struct file *file, char __user *buf,
-                                   size_t nbytes, loff_t *ppos)
+                                size_t nbytes, loff_t *ppos)
 {
+        struct cfent *cfe = __d_cfe(file->f_dentry);
        struct cftype *cft = __d_cft(file->f_dentry);
-        struct cgroup *cgrp = __d_cgrp(file->f_dentry->d_parent);
+        struct cgroup_subsys_state *css = cfe->css;
-        if (cgroup_is_dead(cgrp))
-                return -ENODEV;
        if (cft->read)
-                return cft->read(cgrp, cft, file, buf, nbytes, ppos);
+                return cft->read(css, cft, file, buf, nbytes, ppos);
        if (cft->read_u64)
-                return cgroup_read_u64(cgrp, cft, file, buf, nbytes, ppos);
+                return cgroup_read_u64(css, cft, file, buf, nbytes, ppos);
        if (cft->read_s64)
-                return cgroup_read_s64(cgrp, cft, file, buf, nbytes, ppos);
+                return cgroup_read_s64(css, cft, file, buf, nbytes, ppos);
        return -EINVAL;
 }
@@ -2440,11 +2436,6 @@ static ssize_t cgroup_file_read(struct file *file, char __user *buf,
 * supports string->u64 maps, but can be extended in future.
 */
-struct cgroup_seqfile_state {
-        struct cftype *cft;
-        struct cgroup *cgroup;
-};
 static int cgroup_map_add(struct cgroup_map_cb *cb, const char *key, u64 value)
 {
        struct seq_file *sf = cb->state;
@@ -2453,69 +2444,86 @@ static int cgroup_map_add(struct cgroup_map_cb *cb, const char *key, u64 value)
 static int cgroup_seqfile_show(struct seq_file *m, void *arg)
 {
-        struct cgroup_seqfile_state *state = m->private;
+        struct cfent *cfe = m->private;
-        struct cftype *cft = state->cft;
+        struct cftype *cft = cfe->type;
+        struct cgroup_subsys_state *css = cfe->css;
        if (cft->read_map) {
                struct cgroup_map_cb cb = {
                        .fill = cgroup_map_add,
                        .state = m,
                };
-                return cft->read_map(state->cgroup, cft, &cb);
+                return cft->read_map(css, cft, &cb);
        }
-        return cft->read_seq_string(state->cgroup, cft, m);
+        return cft->read_seq_string(css, cft, m);
-}
-static int cgroup_seqfile_release(struct inode *inode, struct file *file)
-{
-        struct seq_file *seq = file->private_data;
-        kfree(seq->private);
-        return single_release(inode, file);
 }
 static const struct file_operations cgroup_seqfile_operations = {
        .read = seq_read,
        .write = cgroup_file_write,
        .llseek = seq_lseek,
-        .release = cgroup_seqfile_release,
+        .release = single_release,
 };
 static int cgroup_file_open(struct inode *inode, struct file *file)
 {
+        struct cfent *cfe = __d_cfe(file->f_dentry);
+        struct cftype *cft = __d_cft(file->f_dentry);
+        struct cgroup *cgrp = __d_cgrp(cfe->dentry->d_parent);
+        struct cgroup_subsys_state *css;
        int err;
-        struct cftype *cft;
        err = generic_file_open(inode, file);
        if (err)
                return err;
-        cft = __d_cft(file->f_dentry);
-        if (cft->read_map || cft->read_seq_string) {
+        /*
-                struct cgroup_seqfile_state *state;
+         * If the file belongs to a subsystem, pin the css.  Will be
+         * unpinned either on open failure or release.  This ensures that
+         * @css stays alive for all file operations.
+         */
+        rcu_read_lock();
+        css = cgroup_css(cgrp, cft->ss);
+        if (cft->ss && !css_tryget(css))
+                css = NULL;
+        rcu_read_unlock();
-                state = kzalloc(sizeof(*state), GFP_USER);
+        if (!css)
-                if (!state)
+                return -ENODEV;
-                        return -ENOMEM;
+        /*
+         * @cfe->css is used by read/write/close to determine the
+         * associated css.  @file->private_data would be a better place but
+         * that's already used by seqfile.  Multiple accessors may use it
+         * simultaneously which is okay as the association never changes.
+         */
+        WARN_ON_ONCE(cfe->css && cfe->css != css);
+        cfe->css = css;
-                state->cft = cft;
+        if (cft->read_map || cft->read_seq_string) {
-                state->cgroup = __d_cgrp(file->f_dentry->d_parent);
                file->f_op = &cgroup_seqfile_operations;
-                err = single_open(file, cgroup_seqfile_show, state);
+                err = single_open(file, cgroup_seqfile_show, cfe);
-                if (err < 0)
+        } else if (cft->open) {
-                        kfree(state);
-        } else if (cft->open)
                err = cft->open(inode, file);
-        else
+        }
-                err = 0;
+        if (css->ss && err)
+                css_put(css);
        return err;
 }
 static int cgroup_file_release(struct inode *inode, struct file *file)
 {
+        struct cfent *cfe = __d_cfe(file->f_dentry);
        struct cftype *cft = __d_cft(file->f_dentry);
+        struct cgroup_subsys_state *css = cfe->css;
+        int ret = 0;
        if (cft->release)
-                return cft->release(inode, file);
+                ret = cft->release(inode, file);
-        return 0;
+        if (css->ss)
+                css_put(css);
+        return ret;
 }
 /*
@@ -2729,8 +2737,7 @@ static umode_t cgroup_file_mode(const struct cftype *cft)
        return mode;
 }
-static int cgroup_add_file(struct cgroup *cgrp, struct cgroup_subsys *subsys,
+static int cgroup_add_file(struct cgroup *cgrp, struct cftype *cft)
-                           struct cftype *cft)
 {
        struct dentry *dir = cgrp->dentry;
        struct cgroup *parent = __d_cgrp(dir);
@@ -2740,8 +2747,9 @@ static int cgroup_add_file(struct cgroup *cgrp, struct cgroup_subsys *subsys,
        umode_t mode;
        char name[MAX_CGROUP_TYPE_NAMELEN + MAX_CFTYPE_NAME + 2] = { 0 };
-        if (subsys && !(cgrp->root->flags & CGRP_ROOT_NOPREFIX)) {
+        if (cft->ss && !(cft->flags & CFTYPE_NO_PREFIX) &&
-                strcpy(name, subsys->name);
+            !(cgrp->root->flags & CGRP_ROOT_NOPREFIX)) {
+                strcpy(name, cft->ss->name);
                strcat(name, ".");
        }
        strcat(name, cft->name);
@@ -2775,11 +2783,25 @@ out:
        return error;
 }
-static int cgroup_addrm_files(struct cgroup *cgrp, struct cgroup_subsys *subsys,
+/**
-                              struct cftype cfts[], bool is_add)
+ * cgroup_addrm_files - add or remove files to a cgroup directory
+ * @cgrp: the target cgroup
+ * @cfts: array of cftypes to be added
+ * @is_add: whether to add or remove
+ *
+ * Depending on @is_add, add or remove files defined by @cfts on @cgrp.
+ * For removals, this function never fails.  If addition fails, this
+ * function doesn't remove files already added.  The caller is responsible
+ * for cleaning up.
+ */
+static int cgroup_addrm_files(struct cgroup *cgrp, struct cftype cfts[],
+                              bool is_add)
 {
        struct cftype *cft;
-        int err, ret = 0;
+        int ret;
+        lockdep_assert_held(&cgrp->dentry->d_inode->i_mutex);
+        lockdep_assert_held(&cgroup_mutex);
        for (cft = cfts; cft->name[0] != '\0'; cft++) {
                /* does cft->flags tell us to skip this file on @cgrp? */
@@ -2791,16 +2813,17 @@ static int cgroup_addrm_files(struct cgroup *cgrp, struct cgroup_subsys *subsys,
                        continue;
                if (is_add) {
-                        err = cgroup_add_file(cgrp, subsys, cft);
+                        ret = cgroup_add_file(cgrp, cft);
-                        if (err)
+                        if (ret) {
                                pr_warn("cgroup_addrm_files: failed to add %s, err=%d\n",
-                                        cft->name, err);
+                                        cft->name, ret);
-                        ret = err;
+                                return ret;
+                        }
                } else {
                        cgroup_rm_file(cgrp, cft);
                }
        }
-        return ret;
+        return 0;
 }
 static void cgroup_cfts_prepare(void)
@@ -2809,28 +2832,30 @@ static void cgroup_cfts_prepare(void)
        /*
         * Thanks to the entanglement with vfs inode locking, we can't walk
         * the existing cgroups under cgroup_mutex and create files.
-         * Instead, we use cgroup_for_each_descendant_pre() and drop RCU
+         * Instead, we use css_for_each_descendant_pre() and drop RCU read
-         * read lock before calling cgroup_addrm_files().
+         * lock before calling cgroup_addrm_files().
         */
        mutex_lock(&cgroup_mutex);
 }
-static void cgroup_cfts_commit(struct cgroup_subsys *ss,
+static int cgroup_cfts_commit(struct cftype *cfts, bool is_add)
-                               struct cftype *cfts, bool is_add)
        __releases(&cgroup_mutex)
 {
        LIST_HEAD(pending);
-        struct cgroup *cgrp, *root = &ss->root->top_cgroup;
+        struct cgroup_subsys *ss = cfts[0].ss;
+        struct cgroup *root = &ss->root->top_cgroup;
        struct super_block *sb = ss->root->sb;
        struct dentry *prev = NULL;
        struct inode *inode;
+        struct cgroup_subsys_state *css;
        u64 update_before;
+        int ret = 0;
        /* %NULL @cfts indicates abort and don't bother if @ss isn't attached */
        if (!cfts || ss->root == &cgroup_dummy_root ||
            !atomic_inc_not_zero(&sb->s_active)) {
                mutex_unlock(&cgroup_mutex);
-                return;
+                return 0;
        }
        /*
@@ -2842,17 +2867,11 @@ static void cgroup_cfts_commit(struct cgroup_subsys *ss,
        mutex_unlock(&cgroup_mutex);
-        /* @root always needs to be updated */
-        inode = root->dentry->d_inode;
-        mutex_lock(&inode->i_mutex);
-        mutex_lock(&cgroup_mutex);
-        cgroup_addrm_files(root, ss, cfts, is_add);
-        mutex_unlock(&cgroup_mutex);
-        mutex_unlock(&inode->i_mutex);
        /* add/rm files for all cgroups created before */
        rcu_read_lock();
-        cgroup_for_each_descendant_pre(cgrp, root) {
+        css_for_each_descendant_pre(css, cgroup_css(root, ss)) {
+                struct cgroup *cgrp = css->cgroup;
                if (cgroup_is_dead(cgrp))
                        continue;
@@ -2866,15 +2885,18 @@ static void cgroup_cfts_commit(struct cgroup_subsys *ss,
                mutex_lock(&inode->i_mutex);
                mutex_lock(&cgroup_mutex);
                if (cgrp->serial_nr < update_before && !cgroup_is_dead(cgrp))
-                        cgroup_addrm_files(cgrp, ss, cfts, is_add);
+                        ret = cgroup_addrm_files(cgrp, cfts, is_add);
                mutex_unlock(&cgroup_mutex);
                mutex_unlock(&inode->i_mutex);
                rcu_read_lock();
+                if (ret)
+                        break;
        }
        rcu_read_unlock();
        dput(prev);
        deactivate_super(sb);
+        return ret;
 }
 /**
@@ -2894,49 +2916,56 @@ static void cgroup_cfts_commit(struct cgroup_subsys *ss,
 int cgroup_add_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
 {
        struct cftype_set *set;
+        struct cftype *cft;
+        int ret;
        set = kzalloc(sizeof(*set), GFP_KERNEL);
        if (!set)
                return -ENOMEM;
+        for (cft = cfts; cft->name[0] != '\0'; cft++)
+                cft->ss = ss;
        cgroup_cfts_prepare();
        set->cfts = cfts;
        list_add_tail(&set->node, &ss->cftsets);
-        cgroup_cfts_commit(ss, cfts, true);
+        ret = cgroup_cfts_commit(cfts, true);
+        if (ret)
-        return 0;
+                cgroup_rm_cftypes(cfts);
+        return ret;
 }
 EXPORT_SYMBOL_GPL(cgroup_add_cftypes);
 /**
 * cgroup_rm_cftypes - remove an array of cftypes from a subsystem
- * @ss: target cgroup subsystem
 * @cfts: zero-length name terminated array of cftypes
 *
- * Unregister @cfts from @ss.  Files described by @cfts are removed from
+ * Unregister @cfts.  Files described by @cfts are removed from all
- * all existing cgroups to which @ss is attached and all future cgroups
+ * existing cgroups and all future cgroups won't have them either.  This
- * won't have them either.  This function can be called anytime whether @ss
+ * function can be called anytime whether @cfts' subsys is attached or not.
- * is attached or not.
 *
 * Returns 0 on successful unregistration, -ENOENT if @cfts is not
- * registered with @ss.
+ * registered.
 */
-int cgroup_rm_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
+int cgroup_rm_cftypes(struct cftype *cfts)
 {
        struct cftype_set *set;
+        if (!cfts || !cfts[0].ss)
+                return -ENOENT;
        cgroup_cfts_prepare();
-        list_for_each_entry(set, &ss->cftsets, node) {
+        list_for_each_entry(set, &cfts[0].ss->cftsets, node) {
                if (set->cfts == cfts) {
                        list_del(&set->node);
                        kfree(set);
-                        cgroup_cfts_commit(ss, cfts, false);
+                        cgroup_cfts_commit(cfts, false);
                        return 0;
                }
        }
-        cgroup_cfts_commit(ss, NULL, false);
+        cgroup_cfts_commit(NULL, false);
        return -ENOENT;
 }
@@ -2959,34 +2988,10 @@ int cgroup_task_count(const struct cgroup *cgrp)
 }
 /*
- * Advance a list_head iterator.  The iterator should be positioned at
+ * To reduce the fork() overhead for systems that are not actually using
- * the start of a css_set
+ * their cgroups capability, we don't maintain the lists running through
- */
+ * each css_set to its tasks until we see the list actually used - in other
-static void cgroup_advance_iter(struct cgroup *cgrp, struct cgroup_iter *it)
+ * words after the first call to css_task_iter_start().
-{
-        struct list_head *l = it->cset_link;
-        struct cgrp_cset_link *link;
-        struct css_set *cset;
-        /* Advance to the next non-empty css_set */
-        do {
-                l = l->next;
-                if (l == &cgrp->cset_links) {
-                        it->cset_link = NULL;
-                        return;
-                }
-                link = list_entry(l, struct cgrp_cset_link, cset_link);
-                cset = link->cset;
-        } while (list_empty(&cset->tasks));
-        it->cset_link = l;
-        it->task = cset->tasks.next;
-}
-/*
- * To reduce the fork() overhead for systems that are not actually
- * using their cgroups capability, we don't maintain the lists running
- * through each css_set to its tasks until we see the list actually
- * used - in other words after the first call to cgroup_iter_start().
 */
 static void cgroup_enable_task_cg_lists(void)
 {
@@ -3017,16 +3022,21 @@ static void cgroup_enable_task_cg_lists(void)
 }
 /**
- * cgroup_next_sibling - find the next sibling of a given cgroup
+ * css_next_child - find the next child of a given css
- * @pos: the current cgroup
+ * @pos_css: the current position (%NULL to initiate traversal)
+ * @parent_css: css whose children to walk
 *
- * This function returns the next sibling of @pos and should be called
+ * This function returns the next child of @parent_css and should be called
- * under RCU read lock.  The only requirement is that @pos is accessible.
+ * under RCU read lock.  The only requirement is that @parent_css and
- * The next sibling is guaranteed to be returned regardless of @pos's
+ * @pos_css are accessible.  The next sibling is guaranteed to be returned
- * state.
+ * regardless of their states.
 */
-struct cgroup *cgroup_next_sibling(struct cgroup *pos)
+struct cgroup_subsys_state *
+css_next_child(struct cgroup_subsys_state *pos_css,
+               struct cgroup_subsys_state *parent_css)
 {
+        struct cgroup *pos = pos_css ? pos_css->cgroup : NULL;
+        struct cgroup *cgrp = parent_css->cgroup;
        struct cgroup *next;
        WARN_ON_ONCE(!rcu_read_lock_held());
@@ -3041,78 +3051,81 @@ struct cgroup *cgroup_next_sibling(struct cgroup *pos)
         * safe to dereference from this RCU critical section.  If
         * ->sibling.next is inaccessible, cgroup_is_dead() is guaranteed
         * to be visible as %true here.
+         *
+         * If @pos is dead, its next pointer can't be dereferenced;
+         * however, as each cgroup is given a monotonically increasing
+         * unique serial number and always appended to the sibling list,
+         * the next one can be found by walking the parent's children until
+         * we see a cgroup with higher serial number than @pos's.  While
+         * this path can be slower, it's taken only when either the current
+         * cgroup is removed or iteration and removal race.
         */
-        if (likely(!cgroup_is_dead(pos))) {
+        if (!pos) {
+                next = list_entry_rcu(cgrp->children.next, struct cgroup, sibling);
+        } else if (likely(!cgroup_is_dead(pos))) {
                next = list_entry_rcu(pos->sibling.next, struct cgroup, sibling);
-                if (&next->sibling != &pos->parent->children)
+        } else {
-                        return next;
+                list_for_each_entry_rcu(next, &cgrp->children, sibling)
-                return NULL;
+                        if (next->serial_nr > pos->serial_nr)
+                                break;
        }
-        /*
+        if (&next->sibling == &cgrp->children)
-         * Can't dereference the next pointer.  Each cgroup is given a
+                return NULL;
-         * monotonically increasing unique serial number and always
-         * appended to the sibling list, so the next one can be found by
+        return cgroup_css(next, parent_css->ss);
-         * walking the parent's children until we see a cgroup with higher
-         * serial number than @pos's.
-         *
-         * While this path can be slow, it's taken only when either the
-         * current cgroup is removed or iteration and removal race.
-         */
-        list_for_each_entry_rcu(next, &pos->parent->children, sibling)
-                if (next->serial_nr > pos->serial_nr)
-                        return next;
-        return NULL;
 }
-EXPORT_SYMBOL_GPL(cgroup_next_sibling);
+EXPORT_SYMBOL_GPL(css_next_child);
 /**
- * cgroup_next_descendant_pre - find the next descendant for pre-order walk
+ * css_next_descendant_pre - find the next descendant for pre-order walk
 * @pos: the current position (%NULL to initiate traversal)
- * @cgroup: cgroup whose descendants to walk
+ * @root: css whose descendants to walk
 *
- * To be used by cgroup_for_each_descendant_pre().  Find the next
+ * To be used by css_for_each_descendant_pre().  Find the next descendant
- * descendant to visit for pre-order traversal of @cgroup's descendants.
+ * to visit for pre-order traversal of @root's descendants.  @root is
+ * included in the iteration and the first node to be visited.
 *
 * While this function requires RCU read locking, it doesn't require the
 * whole traversal to be contained in a single RCU critical section.  This
 * function will return the correct next descendant as long as both @pos
- * and @cgroup are accessible and @pos is a descendant of @cgroup.
+ * and @root are accessible and @pos is a descendant of @root.
 */
-struct cgroup *cgroup_next_descendant_pre(struct cgroup *pos,
+struct cgroup_subsys_state *
-                                          struct cgroup *cgroup)
+css_next_descendant_pre(struct cgroup_subsys_state *pos,
+                        struct cgroup_subsys_state *root)
 {
-        struct cgroup *next;
+        struct cgroup_subsys_state *next;
        WARN_ON_ONCE(!rcu_read_lock_held());
-        /* if first iteration, pretend we just visited @cgroup */
+        /* if first iteration, visit @root */
        if (!pos)
-                pos = cgroup;
+                return root;
        /* visit the first child if exists */
-        next = list_first_or_null_rcu(&pos->children, struct cgroup, sibling);
+        next = css_next_child(NULL, pos);
        if (next)
                return next;
        /* no child, visit my or the closest ancestor's next sibling */
-        while (pos != cgroup) {
+        while (pos != root) {
-                next = cgroup_next_sibling(pos);
+                next = css_next_child(pos, css_parent(pos));
                if (next)
                        return next;
-                pos = pos->parent;
+                pos = css_parent(pos);
        }
        return NULL;
 }
-EXPORT_SYMBOL_GPL(cgroup_next_descendant_pre);
+EXPORT_SYMBOL_GPL(css_next_descendant_pre);
 /**
- * cgroup_rightmost_descendant - return the rightmost descendant of a cgroup
+ * css_rightmost_descendant - return the rightmost descendant of a css
- * @pos: cgroup of interest
+ * @pos: css of interest
 *
- * Return the rightmost descendant of @pos.  If there's no descendant,
+ * Return the rightmost descendant of @pos.  If there's no descendant, @pos
- * @pos is returned.  This can be used during pre-order traversal to skip
+ * is returned.  This can be used during pre-order traversal to skip
 * subtree of @pos.
 *
 * While this function requires RCU read locking, it doesn't require the
@@ -3120,9 +3133,10 @@ EXPORT_SYMBOL_GPL(cgroup_next_descendant_pre);
 * function will return the correct rightmost descendant as long as @pos is
 * accessible.
 */
-struct cgroup *cgroup_rightmost_descendant(struct cgroup *pos)
+struct cgroup_subsys_state *
+css_rightmost_descendant(struct cgroup_subsys_state *pos)
 {
-        struct cgroup *last, *tmp;
+        struct cgroup_subsys_state *last, *tmp;
        WARN_ON_ONCE(!rcu_read_lock_held());
@@ -3130,82 +3144,138 @@ struct cgroup *cgroup_rightmost_descendant(struct cgroup *pos)
                last = pos;
                /* ->prev isn't RCU safe, walk ->next till the end */
                pos = NULL;
-                list_for_each_entry_rcu(tmp, &last->children, sibling)
+                css_for_each_child(tmp, last)
                        pos = tmp;
        } while (pos);
        return last;
 }
-EXPORT_SYMBOL_GPL(cgroup_rightmost_descendant);
+EXPORT_SYMBOL_GPL(css_rightmost_descendant);
-static struct cgroup *cgroup_leftmost_descendant(struct cgroup *pos)
+static struct cgroup_subsys_state *
+css_leftmost_descendant(struct cgroup_subsys_state *pos)
 {
-        struct cgroup *last;
+        struct cgroup_subsys_state *last;
        do {
                last = pos;
-                pos = list_first_or_null_rcu(&pos->children, struct cgroup,
+                pos = css_next_child(NULL, pos);
-                                             sibling);
        } while (pos);
        return last;
 }
 /**
- * cgroup_next_descendant_post - find the next descendant for post-order walk
+ * css_next_descendant_post - find the next descendant for post-order walk
 * @pos: the current position (%NULL to initiate traversal)
- * @cgroup: cgroup whose descendants to walk
+ * @root: css whose descendants to walk
 *
- * To be used by cgroup_for_each_descendant_post().  Find the next
+ * To be used by css_for_each_descendant_post().  Find the next descendant
- * descendant to visit for post-order traversal of @cgroup's descendants.
+ * to visit for post-order traversal of @root's descendants.  @root is
+ * included in the iteration and the last node to be visited.
 *
 * While this function requires RCU read locking, it doesn't require the
 * whole traversal to be contained in a single RCU critical section.  This
 * function will return the correct next descendant as long as both @pos
 * and @cgroup are accessible and @pos is a descendant of @cgroup.
 */
-struct cgroup *cgroup_next_descendant_post(struct cgroup *pos,
+struct cgroup_subsys_state *
-                                           struct cgroup *cgroup)
+css_next_descendant_post(struct cgroup_subsys_state *pos,
+                         struct cgroup_subsys_state *root)
 {
-        struct cgroup *next;
+        struct cgroup_subsys_state *next;
        WARN_ON_ONCE(!rcu_read_lock_held());
        /* if first iteration, visit the leftmost descendant */
        if (!pos) {
-                next = cgroup_leftmost_descendant(cgroup);
+                next = css_leftmost_descendant(root);
-                return next != cgroup ? next : NULL;
+                return next != root ? next : NULL;
        }
+        /* if we visited @root, we're done */
+        if (pos == root)
+                return NULL;
        /* if there's an unvisited sibling, visit its leftmost descendant */
-        next = cgroup_next_sibling(pos);
+        next = css_next_child(pos, css_parent(pos));
        if (next)
-                return cgroup_leftmost_descendant(next);
+                return css_leftmost_descendant(next);
        /* no sibling left, visit parent */
-        next = pos->parent;
+        return css_parent(pos);
-        return next != cgroup ? next : NULL;
+}
+EXPORT_SYMBOL_GPL(css_next_descendant_post);
+/**
+ * css_advance_task_iter - advance a task itererator to the next css_set
+ * @it: the iterator to advance
+ *
+ * Advance @it to the next css_set to walk.
+ */
+static void css_advance_task_iter(struct css_task_iter *it)
+{
+        struct list_head *l = it->cset_link;
+        struct cgrp_cset_link *link;
+        struct css_set *cset;
+        /* Advance to the next non-empty css_set */
+        do {
+                l = l->next;
+                if (l == &it->origin_css->cgroup->cset_links) {
+                        it->cset_link = NULL;
+                        return;
+                }
+                link = list_entry(l, struct cgrp_cset_link, cset_link);
+                cset = link->cset;
+        } while (list_empty(&cset->tasks));
+        it->cset_link = l;
+        it->task = cset->tasks.next;
 }
-EXPORT_SYMBOL_GPL(cgroup_next_descendant_post);
-void cgroup_iter_start(struct cgroup *cgrp, struct cgroup_iter *it)
+/**
+ * css_task_iter_start - initiate task iteration
+ * @css: the css to walk tasks of
+ * @it: the task iterator to use
+ *
+ * Initiate iteration through the tasks of @css.  The caller can call
+ * css_task_iter_next() to walk through the tasks until the function
+ * returns NULL.  On completion of iteration, css_task_iter_end() must be
+ * called.
+ *
+ * Note that this function acquires a lock which is released when the
+ * iteration finishes.  The caller can't sleep while iteration is in
+ * progress.
+ */
+void css_task_iter_start(struct cgroup_subsys_state *css,
+                         struct css_task_iter *it)
        __acquires(css_set_lock)
 {
        /*
-         * The first time anyone tries to iterate across a cgroup,
+         * The first time anyone tries to iterate across a css, we need to
-         * we need to enable the list linking each css_set to its
+         * enable the list linking each css_set to its tasks, and fix up
-         * tasks, and fix up all existing tasks.
+         * all existing tasks.
         */
        if (!use_task_css_set_links)
                cgroup_enable_task_cg_lists();
        read_lock(&css_set_lock);
-        it->cset_link = &cgrp->cset_links;
-        cgroup_advance_iter(cgrp, it);
+        it->origin_css = css;
+        it->cset_link = &css->cgroup->cset_links;
+        css_advance_task_iter(it);
 }
-struct task_struct *cgroup_iter_next(struct cgroup *cgrp,
+/**
-                                        struct cgroup_iter *it)
+ * css_task_iter_next - return the next task for the iterator
+ * @it: the task iterator being iterated
+ *
+ * The "next" function for task iteration.  @it should have been
+ * initialized via css_task_iter_start().  Returns NULL when the iteration
+ * reaches the end.
+ */
+struct task_struct *css_task_iter_next(struct css_task_iter *it)
 {
        struct task_struct *res;
        struct list_head *l = it->task;
@@ -3219,16 +3289,24 @@ struct task_struct *cgroup_iter_next(struct cgroup *cgrp,
        l = l->next;
        link = list_entry(it->cset_link, struct cgrp_cset_link, cset_link);
        if (l == &link->cset->tasks) {
-                /* We reached the end of this task list - move on to
+                /*
-                 * the next cg_cgroup_link */
+                 * We reached the end of this task list - move on to the
-                cgroup_advance_iter(cgrp, it);
+                 * next cgrp_cset_link.
+                 */
+                css_advance_task_iter(it);
        } else {
                it->task = l;
        }
        return res;
 }
-void cgroup_iter_end(struct cgroup *cgrp, struct cgroup_iter *it)
+/**
+ * css_task_iter_end - finish task iteration
+ * @it: the task iterator to finish
+ *
+ * Finish task iteration started by css_task_iter_start().
+ */
+void css_task_iter_end(struct css_task_iter *it)
        __releases(css_set_lock)
 {
        read_unlock(&css_set_lock);
@@ -3269,46 +3347,49 @@ static inline int started_after(void *p1, void *p2)
 }
 /**
- * cgroup_scan_tasks - iterate though all the tasks in a cgroup
+ * css_scan_tasks - iterate though all the tasks in a css
- * @scan: struct cgroup_scanner containing arguments for the scan
+ * @css: the css to iterate tasks of
+ * @test: optional test callback
+ * @process: process callback
+ * @data: data passed to @test and @process
+ * @heap: optional pre-allocated heap used for task iteration
+ *
+ * Iterate through all the tasks in @css, calling @test for each, and if it
+ * returns %true, call @process for it also.
 *
- * Arguments include pointers to callback functions test_task() and
+ * @test may be NULL, meaning always true (select all tasks), which
- * process_task().
+ * effectively duplicates css_task_iter_{start,next,end}() but does not
- * Iterate through all the tasks in a cgroup, calling test_task() for each,
+ * lock css_set_lock for the call to @process.
- * and if it returns true, call process_task() for it also.
- * The test_task pointer may be NULL, meaning always true (select all tasks).
- * Effectively duplicates cgroup_iter_{start,next,end}()
- * but does not lock css_set_lock for the call to process_task().
- * The struct cgroup_scanner may be embedded in any structure of the caller's
- * creation.
- * It is guaranteed that process_task() will act on every task that
- * is a member of the cgroup for the duration of this call. This
- * function may or may not call process_task() for tasks that exit
- * or move to a different cgroup during the call, or are forked or
- * move into the cgroup during the call.
 *
- * Note that test_task() may be called with locks held, and may in some
+ * It is guaranteed that @process will act on every task that is a member
- * situations be called multiple times for the same task, so it should
+ * of @css for the duration of this call.  This function may or may not
- * be cheap.
+ * call @process for tasks that exit or move to a different css during the
- * If the heap pointer in the struct cgroup_scanner is non-NULL, a heap has been
+ * call, or are forked or move into the css during the call.
- * pre-allocated and will be used for heap operations (and its "gt" member will
+ *
- * be overwritten), else a temporary heap will be used (allocation of which
+ * Note that @test may be called with locks held, and may in some
- * may cause this function to fail).
+ * situations be called multiple times for the same task, so it should be
+ * cheap.
+ *
+ * If @heap is non-NULL, a heap has been pre-allocated and will be used for
+ * heap operations (and its "gt" member will be overwritten), else a
+ * temporary heap will be used (allocation of which may cause this function
+ * to fail).
 */
-int cgroup_scan_tasks(struct cgroup_scanner *scan)
+int css_scan_tasks(struct cgroup_subsys_state *css,
+                   bool (*test)(struct task_struct *, void *),
+                   void (*process)(struct task_struct *, void *),
+                   void *data, struct ptr_heap *heap)
 {
        int retval, i;
-        struct cgroup_iter it;
+        struct css_task_iter it;
        struct task_struct *p, *dropped;
        /* Never dereference latest_task, since it's not refcounted */
        struct task_struct *latest_task = NULL;
        struct ptr_heap tmp_heap;
-        struct ptr_heap *heap;
        struct timespec latest_time = { 0, 0 };
-        if (scan->heap) {
+        if (heap) {
                /* The caller supplied our heap and pre-allocated its memory */
-                heap = scan->heap;
                heap->gt = &started_after;
        } else {
                /* We need to allocate our own heap memory */
@@ -3321,25 +3402,24 @@ int cgroup_scan_tasks(struct cgroup_scanner *scan)
 again:
        /*
-         * Scan tasks in the cgroup, using the scanner's "test_task" callback
+         * Scan tasks in the css, using the @test callback to determine
-         * to determine which are of interest, and using the scanner's
+         * which are of interest, and invoking @process callback on the
-         * "process_task" callback to process any of them that need an update.
+         * ones which need an update.  Since we don't want to hold any
-         * Since we don't want to hold any locks during the task updates,
+         * locks during the task updates, gather tasks to be processed in a
-         * gather tasks to be processed in a heap structure.
+         * heap structure.  The heap is sorted by descending task start
-         * The heap is sorted by descending task start time.
+         * time.  If the statically-sized heap fills up, we overflow tasks
-         * If the statically-sized heap fills up, we overflow tasks that
+         * that started later, and in future iterations only consider tasks
-         * started later, and in future iterations only consider tasks that
+         * that started after the latest task in the previous pass. This
-         * started after the latest task in the previous pass. This
         * guarantees forward progress and that we don't miss any tasks.
         */
        heap->size = 0;
-        cgroup_iter_start(scan->cg, &it);
+        css_task_iter_start(css, &it);
-        while ((p = cgroup_iter_next(scan->cg, &it))) {
+        while ((p = css_task_iter_next(&it))) {
                /*
                 * Only affect tasks that qualify per the caller's callback,
                 * if he provided one
                 */
-                if (scan->test_task && !scan->test_task(p, scan))
+                if (test && !test(p, data))
                        continue;
                /*
                 * Only process tasks that started after the last task
@@ -3367,7 +3447,7 @@ int cgroup_scan_tasks(struct cgroup_scanner *scan)
                 * the heap and wasn't inserted
                 */
        }
-        cgroup_iter_end(scan->cg, &it);
+        css_task_iter_end(&it);
        if (heap->size) {
                for (i = 0; i < heap->size; i++) {
@@ -3377,7 +3457,7 @@ int cgroup_scan_tasks(struct cgroup_scanner *scan)
                                latest_task = q;
                        }
                        /* Process the task per the caller's callback */
-                        scan->process_task(q, scan);
+                        process(q, data);
                        put_task_struct(q);
                }
                /*
@@ -3394,10 +3474,9 @@ int cgroup_scan_tasks(struct cgroup_scanner *scan)
        return 0;
 }
-static void cgroup_transfer_one_task(struct task_struct *task,
+static void cgroup_transfer_one_task(struct task_struct *task, void *data)
-                                     struct cgroup_scanner *scan)
 {
-        struct cgroup *new_cgroup = scan->data;
+        struct cgroup *new_cgroup = data;
        mutex_lock(&cgroup_mutex);
        cgroup_attach_task(new_cgroup, task, false);
@@ -3411,15 +3490,8 @@ static void cgroup_transfer_one_task(struct task_struct *task,
 */
 int cgroup_transfer_tasks(struct cgroup *to, struct cgroup *from)
 {
-        struct cgroup_scanner scan;
+        return css_scan_tasks(&from->dummy_css, NULL, cgroup_transfer_one_task,
+                              to, NULL);
-        scan.cg = from;
-        scan.test_task = NULL; /* select all tasks in cgroup */
-        scan.process_task = cgroup_transfer_one_task;
-        scan.heap = NULL;
-        scan.data = to;
-        return cgroup_scan_tasks(&scan);
 }
 /*
@@ -3461,7 +3533,7 @@ struct cgroup_pidlist {
        /* pointer to the cgroup we belong to, for list removal purposes */
        struct cgroup *owner;
        /* protects the other fields */
-        struct rw_semaphore mutex;
+        struct rw_semaphore rwsem;
 };
 /*
@@ -3534,7 +3606,7 @@ static struct cgroup_pidlist *cgroup_pidlist_find(struct cgroup *cgrp,
        struct pid_namespace *ns = task_active_pid_ns(current);
        /*
-         * We can't drop the pidlist_mutex before taking the l->mutex in case
+         * We can't drop the pidlist_mutex before taking the l->rwsem in case
         * the last ref-holder is trying to remove l from the list at the same
         * time. Holding the pidlist_mutex precludes somebody taking whichever
         * list we find out from under us - compare release_pid_array().
@@ -3543,7 +3615,7 @@ static struct cgroup_pidlist *cgroup_pidlist_find(struct cgroup *cgrp,
        list_for_each_entry(l, &cgrp->pidlists, links) {
                if (l->key.type == type && l->key.ns == ns) {
                        /* make sure l doesn't vanish out from under us */
-                        down_write(&l->mutex);
+                        down_write(&l->rwsem);
                        mutex_unlock(&cgrp->pidlist_mutex);
                        return l;
                }
@@ -3554,8 +3626,8 @@ static struct cgroup_pidlist *cgroup_pidlist_find(struct cgroup *cgrp,
                mutex_unlock(&cgrp->pidlist_mutex);
                return l;
        }
-        init_rwsem(&l->mutex);
+        init_rwsem(&l->rwsem);
-        down_write(&l->mutex);
+        down_write(&l->rwsem);
        l->key.type = type;
        l->key.ns = get_pid_ns(ns);
        l->owner = cgrp;
@@ -3573,7 +3645,7 @@ static int pidlist_array_load(struct cgroup *cgrp, enum cgroup_filetype type,
        pid_t *array;
        int length;
        int pid, n = 0; /* used for populating the array */
-        struct cgroup_iter it;
+        struct css_task_iter it;
        struct task_struct *tsk;
        struct cgroup_pidlist *l;
@@ -3588,8 +3660,8 @@ static int pidlist_array_load(struct cgroup *cgrp, enum cgroup_filetype type,
        if (!array)
                return -ENOMEM;
        /* now, populate the array */
-        cgroup_iter_start(cgrp, &it);
+        css_task_iter_start(&cgrp->dummy_css, &it);
-        while ((tsk = cgroup_iter_next(cgrp, &it))) {
+        while ((tsk = css_task_iter_next(&it))) {
                if (unlikely(n == length))
                        break;
                /* get tgid or pid for procs or tasks file respectively */
@@ -3600,7 +3672,7 @@ static int pidlist_array_load(struct cgroup *cgrp, enum cgroup_filetype type,
                if (pid > 0) /* make sure to only use valid results */
                        array[n++] = pid;
        }
-        cgroup_iter_end(cgrp, &it);
+        css_task_iter_end(&it);
        length = n;
        /* now sort & (if procs) strip out duplicates */
        sort(array, length, sizeof(pid_t), cmppid, NULL);
@@ -3616,7 +3688,7 @@ static int pidlist_array_load(struct cgroup *cgrp, enum cgroup_filetype type,
        l->list = array;
        l->length = length;
        l->use_count++;
-        up_write(&l->mutex);
+        up_write(&l->rwsem);
        *lp = l;
        return 0;
 }
@@ -3634,7 +3706,7 @@ int cgroupstats_build(struct cgroupstats *stats, struct dentry *dentry)
 {
        int ret = -EINVAL;
        struct cgroup *cgrp;
-        struct cgroup_iter it;
+        struct css_task_iter it;
        struct task_struct *tsk;
        /*
@@ -3648,8 +3720,8 @@ int cgroupstats_build(struct cgroupstats *stats, struct dentry *dentry)
        ret = 0;
        cgrp = dentry->d_fsdata;
-        cgroup_iter_start(cgrp, &it);
+        css_task_iter_start(&cgrp->dummy_css, &it);
-        while ((tsk = cgroup_iter_next(cgrp, &it))) {
+        while ((tsk = css_task_iter_next(&it))) {
                switch (tsk->state) {
                case TASK_RUNNING:
                        stats->nr_running++;
@@ -3669,7 +3741,7 @@ int cgroupstats_build(struct cgroupstats *stats, struct dentry *dentry)
                        break;
                }
        }
-        cgroup_iter_end(cgrp, &it);
+        css_task_iter_end(&it);
 err:
        return ret;
@@ -3694,7 +3766,7 @@ static void *cgroup_pidlist_start(struct seq_file *s, loff_t *pos)
        int index = 0, pid = *pos;
        int *iter;
-        down_read(&l->mutex);
+        down_read(&l->rwsem);
        if (pid) {
                int end = l->length;
@@ -3721,7 +3793,7 @@ static void *cgroup_pidlist_start(struct seq_file *s, loff_t *pos)
 static void cgroup_pidlist_stop(struct seq_file *s, void *v)
 {
        struct cgroup_pidlist *l = s->private;
-        up_read(&l->mutex);
+        up_read(&l->rwsem);
 }
 static void *cgroup_pidlist_next(struct seq_file *s, void *v, loff_t *pos)
@@ -3767,7 +3839,7 @@ static void cgroup_release_pid_array(struct cgroup_pidlist *l)
         * pidlist_mutex, we have to take pidlist_mutex first.
         */
        mutex_lock(&l->owner->pidlist_mutex);
-        down_write(&l->mutex);
+        down_write(&l->rwsem);
        BUG_ON(!l->use_count);
        if (!--l->use_count) {
                /* we're the last user if refcount is 0; remove and free */
@@ -3775,12 +3847,12 @@ static void cgroup_release_pid_array(struct cgroup_pidlist *l)
                mutex_unlock(&l->owner->pidlist_mutex);
                pidlist_free(l->list);
                put_pid_ns(l->key.ns);
-                up_write(&l->mutex);
+                up_write(&l->rwsem);
                kfree(l);
                return;
        }
        mutex_unlock(&l->owner->pidlist_mutex);
-        up_write(&l->mutex);
+        up_write(&l->rwsem);
 }
 static int cgroup_pidlist_release(struct inode *inode, struct file *file)
@@ -3844,21 +3916,20 @@ static int cgroup_procs_open(struct inode *unused, struct file *file)
        return cgroup_pidlist_open(file, CGROUP_FILE_PROCS);
 }
-static u64 cgroup_read_notify_on_release(struct cgroup *cgrp,
+static u64 cgroup_read_notify_on_release(struct cgroup_subsys_state *css,
-                                            struct cftype *cft)
+                                         struct cftype *cft)
 {
-        return notify_on_release(cgrp);
+        return notify_on_release(css->cgroup);
 }
-static int cgroup_write_notify_on_release(struct cgroup *cgrp,
+static int cgroup_write_notify_on_release(struct cgroup_subsys_state *css,
-                                          struct cftype *cft,
+                                          struct cftype *cft, u64 val)
-                                          u64 val)
 {
-        clear_bit(CGRP_RELEASABLE, &cgrp->flags);
+        clear_bit(CGRP_RELEASABLE, &css->cgroup->flags);
        if (val)
-                set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
+                set_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags);
        else
-                clear_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
+                clear_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags);
        return 0;
 }
@@ -3888,18 +3959,18 @@ static void cgroup_event_remove(struct work_struct *work)
 {
        struct cgroup_event *event = container_of(work, struct cgroup_event,
                        remove);
-        struct cgroup *cgrp = event->cgrp;
+        struct cgroup_subsys_state *css = event->css;
        remove_wait_queue(event->wqh, &event->wait);
-        event->cft->unregister_event(cgrp, event->cft, event->eventfd);
+        event->cft->unregister_event(css, event->cft, event->eventfd);
        /* Notify userspace the event is going away. */
        eventfd_signal(event->eventfd, 1);
        eventfd_ctx_put(event->eventfd);
        kfree(event);
-        cgroup_dput(cgrp);
+        css_put(css);
 }
 /*
@@ -3912,7 +3983,7 @@ static int cgroup_event_wake(wait_queue_t *wait, unsigned mode,
 {
        struct cgroup_event *event = container_of(wait,
                        struct cgroup_event, wait);
-        struct cgroup *cgrp = event->cgrp;
+        struct cgroup *cgrp = event->css->cgroup;
        unsigned long flags = (unsigned long)key;
        if (flags & POLLHUP) {
@@ -3956,14 +4027,15 @@ static void cgroup_event_ptable_queue_proc(struct file *file,
 * Input must be in format '<event_fd> <control_fd> <args>'.
 * Interpretation of args is defined by control file implementation.
 */
-static int cgroup_write_event_control(struct cgroup *cgrp, struct cftype *cft,
+static int cgroup_write_event_control(struct cgroup_subsys_state *dummy_css,
-                                      const char *buffer)
+                                      struct cftype *cft, const char *buffer)
 {
-        struct cgroup_event *event = NULL;
+        struct cgroup *cgrp = dummy_css->cgroup;
-        struct cgroup *cgrp_cfile;
+        struct cgroup_event *event;
+        struct cgroup_subsys_state *cfile_css;
        unsigned int efd, cfd;
-        struct file *efile = NULL;
+        struct file *efile;
-        struct file *cfile = NULL;
+        struct file *cfile;
        char *endp;
        int ret;
@@ -3980,7 +4052,7 @@ static int cgroup_write_event_control(struct cgroup *cgrp, struct cftype *cft,
        event = kzalloc(sizeof(*event), GFP_KERNEL);
        if (!event)
                return -ENOMEM;
-        event->cgrp = cgrp;
        INIT_LIST_HEAD(&event->list);
        init_poll_funcptr(&event->pt, cgroup_event_ptable_queue_proc);
        init_waitqueue_func_entry(&event->wait, cgroup_event_wake);
@@ -3989,62 +4061,68 @@ static int cgroup_write_event_control(struct cgroup *cgrp, struct cftype *cft,
        efile = eventfd_fget(efd);
        if (IS_ERR(efile)) {
                ret = PTR_ERR(efile);
-                goto fail;
+                goto out_kfree;
        }
        event->eventfd = eventfd_ctx_fileget(efile);
        if (IS_ERR(event->eventfd)) {
                ret = PTR_ERR(event->eventfd);
-                goto fail;
+                goto out_put_efile;
        }
        cfile = fget(cfd);
        if (!cfile) {
                ret = -EBADF;
-                goto fail;
+                goto out_put_eventfd;
        }
        /* the process need read permission on control file */
        /* AV: shouldn't we check that it's been opened for read instead? */
        ret = inode_permission(file_inode(cfile), MAY_READ);
        if (ret < 0)
-                goto fail;
+                goto out_put_cfile;
        event->cft = __file_cft(cfile);
        if (IS_ERR(event->cft)) {
                ret = PTR_ERR(event->cft);
-                goto fail;
+                goto out_put_cfile;
+        }
+        if (!event->cft->ss) {
+                ret = -EBADF;
+                goto out_put_cfile;
        }
        /*
-         * The file to be monitored must be in the same cgroup as
+         * Determine the css of @cfile, verify it belongs to the same
-         * cgroup.event_control is.
+         * cgroup as cgroup.event_control, and associate @event with it.
+         * Remaining events are automatically removed on cgroup destruction
+         * but the removal is asynchronous, so take an extra ref.
         */
-        cgrp_cfile = __d_cgrp(cfile->f_dentry->d_parent);
+        rcu_read_lock();
-        if (cgrp_cfile != cgrp) {
-                ret = -EINVAL;
+        ret = -EINVAL;
-                goto fail;
+        event->css = cgroup_css(cgrp, event->cft->ss);
-        }
+        cfile_css = css_from_dir(cfile->f_dentry->d_parent, event->cft->ss);
+        if (event->css && event->css == cfile_css && css_tryget(event->css))
+                ret = 0;
+        rcu_read_unlock();
+        if (ret)
+                goto out_put_cfile;
        if (!event->cft->register_event || !event->cft->unregister_event) {
                ret = -EINVAL;
-                goto fail;
+                goto out_put_css;
        }
-        ret = event->cft->register_event(cgrp, event->cft,
+        ret = event->cft->register_event(event->css, event->cft,
                        event->eventfd, buffer);
        if (ret)
-                goto fail;
+                goto out_put_css;
        efile->f_op->poll(efile, &event->pt);
-        /*
-         * Events should be removed after rmdir of cgroup directory, but before
-         * destroying subsystem state objects. Let's take reference to cgroup
-         * directory dentry to do that.
-         */
-        dget(cgrp->dentry);
        spin_lock(&cgrp->event_list_lock);
        list_add(&event->list, &cgrp->event_list);
        spin_unlock(&cgrp->event_list_lock);
@@ -4054,35 +4132,33 @@ static int cgroup_write_event_control(struct cgroup *cgrp, struct cftype *cft,
        return 0;
-fail:
+out_put_css:
-        if (cfile)
+        css_put(event->css);
-                fput(cfile);
+out_put_cfile:
+        fput(cfile);
-        if (event && event->eventfd && !IS_ERR(event->eventfd))
+out_put_eventfd:
-                eventfd_ctx_put(event->eventfd);
+        eventfd_ctx_put(event->eventfd);
+out_put_efile:
-        if (!IS_ERR_OR_NULL(efile))
+        fput(efile);
-                fput(efile);
+out_kfree:
        kfree(event);
        return ret;
 }
-static u64 cgroup_clone_children_read(struct cgroup *cgrp,
+static u64 cgroup_clone_children_read(struct cgroup_subsys_state *css,
-                                    struct cftype *cft)
+                                      struct cftype *cft)
 {
-        return test_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags);
+        return test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
 }
-static int cgroup_clone_children_write(struct cgroup *cgrp,
+static int cgroup_clone_children_write(struct cgroup_subsys_state *css,
-                                     struct cftype *cft,
+                                       struct cftype *cft, u64 val)
-                                     u64 val)
 {
        if (val)
-                set_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags);
+                set_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
        else
-                clear_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags);
+                clear_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
        return 0;
 }
@@ -4141,36 +4217,34 @@ static struct cftype cgroup_base_files[] = {
 };
 /**
- * cgroup_populate_dir - selectively creation of files in a directory
+ * cgroup_populate_dir - create subsys files in a cgroup directory
 * @cgrp: target cgroup
- * @base_files: true if the base files should be added
 * @subsys_mask: mask of the subsystem ids whose files should be added
+ *
+ * On failure, no file is added.
 */
-static int cgroup_populate_dir(struct cgroup *cgrp, bool base_files,
+static int cgroup_populate_dir(struct cgroup *cgrp, unsigned long subsys_mask)
-                               unsigned long subsys_mask)
 {
-        int err;
        struct cgroup_subsys *ss;
+        int i, ret = 0;
-        if (base_files) {
-                err = cgroup_addrm_files(cgrp, NULL, cgroup_base_files, true);
-                if (err < 0)
-                        return err;
-        }
        /* process cftsets of each subsystem */
-        for_each_root_subsys(cgrp->root, ss) {
+        for_each_subsys(ss, i) {
                struct cftype_set *set;
-                if (!test_bit(ss->subsys_id, &subsys_mask))
+                if (!test_bit(i, &subsys_mask))
                        continue;
-                list_for_each_entry(set, &ss->cftsets, node)
+                list_for_each_entry(set, &ss->cftsets, node) {
-                        cgroup_addrm_files(cgrp, ss, set->cfts, true);
+                        ret = cgroup_addrm_files(cgrp, set->cfts, true);
+                        if (ret < 0)
+                                goto err;
+                }
        }
        /* This cgroup is ready now */
        for_each_root_subsys(cgrp->root, ss) {
-                struct cgroup_subsys_state *css = cgrp->subsys[ss->subsys_id];
+                struct cgroup_subsys_state *css = cgroup_css(cgrp, ss);
                struct css_id *id = rcu_dereference_protected(css->id, true);
                /*
@@ -4183,14 +4257,57 @@ static int cgroup_populate_dir(struct cgroup *cgrp, bool base_files,
        }
        return 0;
+err:
+        cgroup_clear_dir(cgrp, subsys_mask);
+        return ret;
 }
-static void css_dput_fn(struct work_struct *work)
+/*
+ * css destruction is four-stage process.
+ *
+ * 1. Destruction starts.  Killing of the percpu_ref is initiated.
+ *    Implemented in kill_css().
+ *
+ * 2. When the percpu_ref is confirmed to be visible as killed on all CPUs
+ *    and thus css_tryget() is guaranteed to fail, the css can be offlined
+ *    by invoking offline_css().  After offlining, the base ref is put.
+ *    Implemented in css_killed_work_fn().
+ *
+ * 3. When the percpu_ref reaches zero, the only possible remaining
+ *    accessors are inside RCU read sections.  css_release() schedules the
+ *    RCU callback.
+ *
+ * 4. After the grace period, the css can be freed.  Implemented in
+ *    css_free_work_fn().
+ *
+ * It is actually hairier because both step 2 and 4 require process context
+ * and thus involve punting to css->destroy_work adding two additional
+ * steps to the already complex sequence.
+ */
+static void css_free_work_fn(struct work_struct *work)
 {
        struct cgroup_subsys_state *css =
-                container_of(work, struct cgroup_subsys_state, dput_work);
+                container_of(work, struct cgroup_subsys_state, destroy_work);
+        struct cgroup *cgrp = css->cgroup;
-        cgroup_dput(css->cgroup);
+        if (css->parent)
+                css_put(css->parent);
+        css->ss->css_free(css);
+        cgroup_dput(cgrp);
+}
+static void css_free_rcu_fn(struct rcu_head *rcu_head)
+{
+        struct cgroup_subsys_state *css =
+                container_of(rcu_head, struct cgroup_subsys_state, rcu_head);
+        /*
+         * css holds an extra ref to @cgrp->dentry which is put on the last
+         * css_put().  dput() requires process context which we don't have.
+         */
+        INIT_WORK(&css->destroy_work, css_free_work_fn);
+        schedule_work(&css->destroy_work);
 }
 static void css_release(struct percpu_ref *ref)
@@ -4198,49 +4315,47 @@ static void css_release(struct percpu_ref *ref)
        struct cgroup_subsys_state *css =
                container_of(ref, struct cgroup_subsys_state, refcnt);
-        schedule_work(&css->dput_work);
+        call_rcu(&css->rcu_head, css_free_rcu_fn);
 }
-static void init_cgroup_css(struct cgroup_subsys_state *css,
+static void init_css(struct cgroup_subsys_state *css, struct cgroup_subsys *ss,
-                               struct cgroup_subsys *ss,
+                     struct cgroup *cgrp)
-                               struct cgroup *cgrp)
 {
        css->cgroup = cgrp;
+        css->ss = ss;
        css->flags = 0;
        css->id = NULL;
-        if (cgrp == cgroup_dummy_top)
+        if (cgrp->parent)
+                css->parent = cgroup_css(cgrp->parent, ss);
+        else
                css->flags |= CSS_ROOT;
-        BUG_ON(cgrp->subsys[ss->subsys_id]);
-        cgrp->subsys[ss->subsys_id] = css;
-        /*
+        BUG_ON(cgroup_css(cgrp, ss));
-         * css holds an extra ref to @cgrp->dentry which is put on the last
-         * css_put().  dput() requires process context, which css_put() may
-         * be called without.  @css->dput_work will be used to invoke
-         * dput() asynchronously from css_put().
-         */
-        INIT_WORK(&css->dput_work, css_dput_fn);
 }
-/* invoke ->post_create() on a new CSS and mark it online if successful */
+/* invoke ->css_online() on a new CSS and mark it online if successful */
-static int online_css(struct cgroup_subsys *ss, struct cgroup *cgrp)
+static int online_css(struct cgroup_subsys_state *css)
 {
+        struct cgroup_subsys *ss = css->ss;
        int ret = 0;
        lockdep_assert_held(&cgroup_mutex);
        if (ss->css_online)
-                ret = ss->css_online(cgrp);
+                ret = ss->css_online(css);
-        if (!ret)
+        if (!ret) {
-                cgrp->subsys[ss->subsys_id]->flags |= CSS_ONLINE;
+                css->flags |= CSS_ONLINE;
+                css->cgroup->nr_css++;
+                rcu_assign_pointer(css->cgroup->subsys[ss->subsys_id], css);
+        }
        return ret;
 }
-/* if the CSS is online, invoke ->pre_destory() on it and mark it offline */
+/* if the CSS is online, invoke ->css_offline() on it and mark it offline */
-static void offline_css(struct cgroup_subsys *ss, struct cgroup *cgrp)
+static void offline_css(struct cgroup_subsys_state *css)
-        __releases(&cgroup_mutex) __acquires(&cgroup_mutex)
 {
-        struct cgroup_subsys_state *css = cgrp->subsys[ss->subsys_id];
+        struct cgroup_subsys *ss = css->ss;
        lockdep_assert_held(&cgroup_mutex);
@@ -4248,9 +4363,11 @@ static void offline_css(struct cgroup_subsys *ss, struct cgroup *cgrp)
                return;
        if (ss->css_offline)
-                ss->css_offline(cgrp);
+                ss->css_offline(css);
-        cgrp->subsys[ss->subsys_id]->flags &= ~CSS_ONLINE;
+        css->flags &= ~CSS_ONLINE;
+        css->cgroup->nr_css--;
+        RCU_INIT_POINTER(css->cgroup->subsys[ss->subsys_id], css);
 }
 /*
@@ -4264,6 +4381,7 @@ static void offline_css(struct cgroup_subsys *ss, struct cgroup *cgrp)
 static long cgroup_create(struct cgroup *parent, struct dentry *dentry,
                             umode_t mode)
 {
+        struct cgroup_subsys_state *css_ar[CGROUP_SUBSYS_COUNT] = { };
        struct cgroup *cgrp;
        struct cgroup_name *name;
        struct cgroupfs_root *root = parent->root;
@@ -4281,7 +4399,11 @@ static long cgroup_create(struct cgroup *parent, struct dentry *dentry,
                goto err_free_cgrp;
        rcu_assign_pointer(cgrp->name, name);
-        cgrp->id = ida_simple_get(&root->cgroup_ida, 1, 0, GFP_KERNEL);
+        /*
+         * Temporarily set the pointer to NULL, so idr_find() won't return
+         * a half-baked cgroup.
+         */
+        cgrp->id = idr_alloc(&root->cgroup_idr, NULL, 1, 0, GFP_KERNEL);
        if (cgrp->id < 0)
                goto err_free_name;
@@ -4310,6 +4432,7 @@ static long cgroup_create(struct cgroup *parent, struct dentry *dentry,
        cgrp->dentry = dentry;
        cgrp->parent = parent;
+        cgrp->dummy_css.parent = &parent->dummy_css;
        cgrp->root = parent->root;
        if (notify_on_release(parent))
@@ -4321,20 +4444,21 @@ static long cgroup_create(struct cgroup *parent, struct dentry *dentry,
        for_each_root_subsys(root, ss) {
                struct cgroup_subsys_state *css;
-                css = ss->css_alloc(cgrp);
+                css = ss->css_alloc(cgroup_css(parent, ss));
                if (IS_ERR(css)) {
                        err = PTR_ERR(css);
                        goto err_free_all;
                }
+                css_ar[ss->subsys_id] = css;
                err = percpu_ref_init(&css->refcnt, css_release);
                if (err)
                        goto err_free_all;
-                init_cgroup_css(css, ss, cgrp);
+                init_css(css, ss, cgrp);
                if (ss->use_id) {
-                        err = alloc_css_id(ss, parent, cgrp);
+                        err = alloc_css_id(css);
                        if (err)
                                goto err_free_all;
                }
@@ -4356,16 +4480,22 @@ static long cgroup_create(struct cgroup *parent, struct dentry *dentry,
        list_add_tail_rcu(&cgrp->sibling, &cgrp->parent->children);
        root->number_of_cgroups++;
-        /* each css holds a ref to the cgroup's dentry */
+        /* each css holds a ref to the cgroup's dentry and the parent css */
-        for_each_root_subsys(root, ss)
+        for_each_root_subsys(root, ss) {
+                struct cgroup_subsys_state *css = css_ar[ss->subsys_id];
                dget(dentry);
+                css_get(css->parent);
+        }
        /* hold a ref to the parent's dentry */
        dget(parent->dentry);
        /* creation succeeded, notify subsystems */
        for_each_root_subsys(root, ss) {
-                err = online_css(ss, cgrp);
+                struct cgroup_subsys_state *css = css_ar[ss->subsys_id];
+                err = online_css(css);
                if (err)
                        goto err_destroy;
@@ -4379,7 +4509,13 @@ static long cgroup_create(struct cgroup *parent, struct dentry *dentry,
                }
        }
-        err = cgroup_populate_dir(cgrp, true, root->subsys_mask);
+        idr_replace(&root->cgroup_idr, cgrp, cgrp->id);
+        err = cgroup_addrm_files(cgrp, cgroup_base_files, true);
+        if (err)
+                goto err_destroy;
+        err = cgroup_populate_dir(cgrp, root->subsys_mask);
        if (err)
                goto err_destroy;
@@ -4390,18 +4526,18 @@ static long cgroup_create(struct cgroup *parent, struct dentry *dentry,
 err_free_all:
        for_each_root_subsys(root, ss) {
-                struct cgroup_subsys_state *css = cgrp->subsys[ss->subsys_id];
+                struct cgroup_subsys_state *css = css_ar[ss->subsys_id];
                if (css) {
                        percpu_ref_cancel_init(&css->refcnt);
-                        ss->css_free(cgrp);
+                        ss->css_free(css);
                }
        }
        mutex_unlock(&cgroup_mutex);
        /* Release the reference count that we took on the superblock */
        deactivate_super(sb);
 err_free_id:
-        ida_simple_remove(&root->cgroup_ida, cgrp->id);
+        idr_remove(&root->cgroup_idr, cgrp->id);
 err_free_name:
        kfree(rcu_dereference_raw(cgrp->name));
 err_free_cgrp:
@@ -4423,22 +4559,84 @@ static int cgroup_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
        return cgroup_create(c_parent, dentry, mode | S_IFDIR);
 }
-static void cgroup_css_killed(struct cgroup *cgrp)
+/*
+ * This is called when the refcnt of a css is confirmed to be killed.
+ * css_tryget() is now guaranteed to fail.
+ */
+static void css_killed_work_fn(struct work_struct *work)
 {
-        if (!atomic_dec_and_test(&cgrp->css_kill_cnt))
+        struct cgroup_subsys_state *css =
-                return;
+                container_of(work, struct cgroup_subsys_state, destroy_work);
+        struct cgroup *cgrp = css->cgroup;
-        /* percpu ref's of all css's are killed, kick off the next step */
+        mutex_lock(&cgroup_mutex);
-        INIT_WORK(&cgrp->destroy_work, cgroup_offline_fn);
-        schedule_work(&cgrp->destroy_work);
+        /*
+         * css_tryget() is guaranteed to fail now.  Tell subsystems to
+         * initate destruction.
+         */
+        offline_css(css);
+        /*
+         * If @cgrp is marked dead, it's waiting for refs of all css's to
+         * be disabled before proceeding to the second phase of cgroup
+         * destruction.  If we are the last one, kick it off.
+         */
+        if (!cgrp->nr_css && cgroup_is_dead(cgrp))
+                cgroup_destroy_css_killed(cgrp);
+        mutex_unlock(&cgroup_mutex);
+        /*
+         * Put the css refs from kill_css().  Each css holds an extra
+         * reference to the cgroup's dentry and cgroup removal proceeds
+         * regardless of css refs.  On the last put of each css, whenever
+         * that may be, the extra dentry ref is put so that dentry
+         * destruction happens only after all css's are released.
+         */
+        css_put(css);
 }
-static void css_ref_killed_fn(struct percpu_ref *ref)
+/* css kill confirmation processing requires process context, bounce */
+static void css_killed_ref_fn(struct percpu_ref *ref)
 {
        struct cgroup_subsys_state *css =
                container_of(ref, struct cgroup_subsys_state, refcnt);
-        cgroup_css_killed(css->cgroup);
+        INIT_WORK(&css->destroy_work, css_killed_work_fn);
+        schedule_work(&css->destroy_work);
+}
+/**
+ * kill_css - destroy a css
+ * @css: css to destroy
+ *
+ * This function initiates destruction of @css by removing cgroup interface
+ * files and putting its base reference.  ->css_offline() will be invoked
+ * asynchronously once css_tryget() is guaranteed to fail and when the
+ * reference count reaches zero, @css will be released.
+ */
+static void kill_css(struct cgroup_subsys_state *css)
+{
+        cgroup_clear_dir(css->cgroup, 1 << css->ss->subsys_id);
+        /*
+         * Killing would put the base ref, but we need to keep it alive
+         * until after ->css_offline().
+         */
+        css_get(css);
+        /*
+         * cgroup core guarantees that, by the time ->css_offline() is
+         * invoked, no new css reference will be given out via
+         * css_tryget().  We can't simply call percpu_ref_kill() and
+         * proceed to offlining css's because percpu_ref_kill() doesn't
+         * guarantee that the ref is seen as killed on all CPUs on return.
+         *
+         * Use percpu_ref_kill_and_confirm() to get notifications as each
+         * css is confirmed to be seen as killed on all CPUs.
+         */
+        percpu_ref_kill_and_confirm(&css->refcnt, css_killed_ref_fn);
 }
 /**
@@ -4471,6 +4669,7 @@ static int cgroup_destroy_locked(struct cgroup *cgrp)
        struct dentry *d = cgrp->dentry;
        struct cgroup_event *event, *tmp;
        struct cgroup_subsys *ss;
+        struct cgroup *child;
        bool empty;
        lockdep_assert_held(&d->d_inode->i_mutex);
@@ -4481,47 +4680,41 @@ static int cgroup_destroy_locked(struct cgroup *cgrp)
         * @cgrp from being removed while __put_css_set() is in progress.
         */
        read_lock(&css_set_lock);
-        empty = list_empty(&cgrp->cset_links) && list_empty(&cgrp->children);
+        empty = list_empty(&cgrp->cset_links);
        read_unlock(&css_set_lock);
        if (!empty)
                return -EBUSY;
        /*
-         * Block new css_tryget() by killing css refcnts.  cgroup core
+         * Make sure there's no live children.  We can't test ->children
-         * guarantees that, by the time ->css_offline() is invoked, no new
+         * emptiness as dead children linger on it while being destroyed;
-         * css reference will be given out via css_tryget().  We can't
+         * otherwise, "rmdir parent/child parent" may fail with -EBUSY.
-         * simply call percpu_ref_kill() and proceed to offlining css's
-         * because percpu_ref_kill() doesn't guarantee that the ref is seen
-         * as killed on all CPUs on return.
-         *
-         * Use percpu_ref_kill_and_confirm() to get notifications as each
-         * css is confirmed to be seen as killed on all CPUs.  The
-         * notification callback keeps track of the number of css's to be
-         * killed and schedules cgroup_offline_fn() to perform the rest of
-         * destruction once the percpu refs of all css's are confirmed to
-         * be killed.
         */
-        atomic_set(&cgrp->css_kill_cnt, 1);
+        empty = true;
-        for_each_root_subsys(cgrp->root, ss) {
+        rcu_read_lock();
-                struct cgroup_subsys_state *css = cgrp->subsys[ss->subsys_id];
+        list_for_each_entry_rcu(child, &cgrp->children, sibling) {
+                empty = cgroup_is_dead(child);
-                /*
+                if (!empty)
-                 * Killing would put the base ref, but we need to keep it
+                        break;
-                 * alive until after ->css_offline.
-                 */
-                percpu_ref_get(&css->refcnt);
-                atomic_inc(&cgrp->css_kill_cnt);
-                percpu_ref_kill_and_confirm(&css->refcnt, css_ref_killed_fn);
        }
-        cgroup_css_killed(cgrp);
+        rcu_read_unlock();
+        if (!empty)
+                return -EBUSY;
+        /*
+         * Initiate massacre of all css's.  cgroup_destroy_css_killed()
+         * will be invoked to perform the rest of destruction once the
+         * percpu refs of all css's are confirmed to be killed.
+         */
+        for_each_root_subsys(cgrp->root, ss)
+                kill_css(cgroup_css(cgrp, ss));
        /*
         * Mark @cgrp dead.  This prevents further task migration and child
         * creation by disabling cgroup_lock_live_group().  Note that
-         * CGRP_DEAD assertion is depended upon by cgroup_next_sibling() to
+         * CGRP_DEAD assertion is depended upon by css_next_child() to
         * resume iteration after dropping RCU read lock.  See
-         * cgroup_next_sibling() for details.
+         * css_next_child() for details.
         */
        set_bit(CGRP_DEAD, &cgrp->flags);
@@ -4532,9 +4725,20 @@ static int cgroup_destroy_locked(struct cgroup *cgrp)
        raw_spin_unlock(&release_list_lock);
        /*
-         * Remove @cgrp directory.  The removal puts the base ref but we
+         * If @cgrp has css's attached, the second stage of cgroup
-         * aren't quite done with @cgrp yet, so hold onto it.
+         * destruction is kicked off from css_killed_work_fn() after the
+         * refs of all attached css's are killed.  If @cgrp doesn't have
+         * any css, we kick it off here.
         */
+        if (!cgrp->nr_css)
+                cgroup_destroy_css_killed(cgrp);
+        /*
+         * Clear the base files and remove @cgrp directory.  The removal
+         * puts the base ref but we aren't quite done with @cgrp yet, so
+         * hold onto it.
+         */
+        cgroup_addrm_files(cgrp, cgroup_base_files, false);
        dget(d);
        cgroup_d_remove_dir(d);
@@ -4554,50 +4758,36 @@ static int cgroup_destroy_locked(struct cgroup *cgrp)
 };
 /**
- * cgroup_offline_fn - the second step of cgroup destruction
+ * cgroup_destroy_css_killed - the second step of cgroup destruction
 * @work: cgroup->destroy_free_work
 *
 * This function is invoked from a work item for a cgroup which is being
- * destroyed after the percpu refcnts of all css's are guaranteed to be
+ * destroyed after all css's are offlined and performs the rest of
- * seen as killed on all CPUs, and performs the rest of destruction.  This
+ * destruction.  This is the second step of destruction described in the
- * is the second step of destruction described in the comment above
+ * comment above cgroup_destroy_locked().
- * cgroup_destroy_locked().
 */
-static void cgroup_offline_fn(struct work_struct *work)
+static void cgroup_destroy_css_killed(struct cgroup *cgrp)
 {
-        struct cgroup *cgrp = container_of(work, struct cgroup, destroy_work);
        struct cgroup *parent = cgrp->parent;
        struct dentry *d = cgrp->dentry;
-        struct cgroup_subsys *ss;
-        mutex_lock(&cgroup_mutex);
+        lockdep_assert_held(&cgroup_mutex);
-        /*
+        /* delete this cgroup from parent->children */
-         * css_tryget() is guaranteed to fail now.  Tell subsystems to
+        list_del_rcu(&cgrp->sibling);
-         * initate destruction.
-         */
-        for_each_root_subsys(cgrp->root, ss)
-                offline_css(ss, cgrp);
        /*
-         * Put the css refs from cgroup_destroy_locked().  Each css holds
+         * We should remove the cgroup object from idr before its grace
-         * an extra reference to the cgroup's dentry and cgroup removal
+         * period starts, so we won't be looking up a cgroup while the
-         * proceeds regardless of css refs.  On the last put of each css,
+         * cgroup is being freed.
-         * whenever that may be, the extra dentry ref is put so that dentry
-         * destruction happens only after all css's are released.
         */
-        for_each_root_subsys(cgrp->root, ss)
+        idr_remove(&cgrp->root->cgroup_idr, cgrp->id);
-                css_put(cgrp->subsys[ss->subsys_id]);
+        cgrp->id = -1;
-        /* delete this cgroup from parent->children */
-        list_del_rcu(&cgrp->sibling);
        dput(d);
        set_bit(CGRP_RELEASABLE, &parent->flags);
        check_for_release(parent);
-        mutex_unlock(&cgroup_mutex);
 }
 static int cgroup_rmdir(struct inode *unused_dir, struct dentry *dentry)
@@ -4620,6 +4810,11 @@ static void __init_or_module cgroup_init_cftsets(struct cgroup_subsys *ss)
         * deregistration.
         */
        if (ss->base_cftypes) {
+                struct cftype *cft;
+                for (cft = ss->base_cftypes; cft->name[0] != '\0'; cft++)
+                        cft->ss = ss;
                ss->base_cftset.cfts = ss->base_cftypes;
                list_add_tail(&ss->base_cftset.node, &ss->cftsets);
        }
@@ -4639,10 +4834,10 @@ static void __init cgroup_init_subsys(struct cgroup_subsys *ss)
        /* Create the top cgroup state for this subsystem */
        list_add(&ss->sibling, &cgroup_dummy_root.subsys_list);
        ss->root = &cgroup_dummy_root;
-        css = ss->css_alloc(cgroup_dummy_top);
+        css = ss->css_alloc(cgroup_css(cgroup_dummy_top, ss));
        /* We don't handle early failures gracefully */
        BUG_ON(IS_ERR(css));
-        init_cgroup_css(css, ss, cgroup_dummy_top);
+        init_css(css, ss, cgroup_dummy_top);
        /* Update the init_css_set to contain a subsys
         * pointer to this state - since the subsystem is
@@ -4657,7 +4852,7 @@ static void __init cgroup_init_subsys(struct cgroup_subsys *ss)
         * need to invoke fork callbacks here. */
        BUG_ON(!list_empty(&init_task.tasks));
-        BUG_ON(online_css(ss, cgroup_dummy_top));
+        BUG_ON(online_css(css));
        mutex_unlock(&cgroup_mutex);
@@ -4718,7 +4913,7 @@ int __init_or_module cgroup_load_subsys(struct cgroup_subsys *ss)
         * struct, so this can happen first (i.e. before the dummy root
         * attachment).
         */
-        css = ss->css_alloc(cgroup_dummy_top);
+        css = ss->css_alloc(cgroup_css(cgroup_dummy_top, ss));
        if (IS_ERR(css)) {
                /* failure case - need to deassign the cgroup_subsys[] slot. */
                cgroup_subsys[ss->subsys_id] = NULL;
@@ -4730,8 +4925,8 @@ int __init_or_module cgroup_load_subsys(struct cgroup_subsys *ss)
        ss->root = &cgroup_dummy_root;
        /* our new subsystem will be attached to the dummy hierarchy. */
-        init_cgroup_css(css, ss, cgroup_dummy_top);
+        init_css(css, ss, cgroup_dummy_top);
-        /* init_idr must be after init_cgroup_css because it sets css->id. */
+        /* init_idr must be after init_css() because it sets css->id. */
        if (ss->use_id) {
                ret = cgroup_init_idr(ss, css);
                if (ret)
@@ -4761,7 +4956,7 @@ int __init_or_module cgroup_load_subsys(struct cgroup_subsys *ss)
        }
        write_unlock(&css_set_lock);
-        ret = online_css(ss, cgroup_dummy_top);
+        ret = online_css(css);
        if (ret)
                goto err_unload;
@@ -4793,14 +4988,14 @@ void cgroup_unload_subsys(struct cgroup_subsys *ss)
        /*
         * we shouldn't be called if the subsystem is in use, and the use of
-         * try_module_get in parse_cgroupfs_options should ensure that it
+         * try_module_get() in rebind_subsystems() should ensure that it
         * doesn't start being used while we're killing it off.
         */
        BUG_ON(ss->root != &cgroup_dummy_root);
        mutex_lock(&cgroup_mutex);
-        offline_css(ss, cgroup_dummy_top);
+        offline_css(cgroup_css(cgroup_dummy_top, ss));
        if (ss->use_id)
                idr_destroy(&ss->idr);
@@ -4834,8 +5029,8 @@ void cgroup_unload_subsys(struct cgroup_subsys *ss)
         * the cgrp->subsys pointer to find their state. note that this
         * also takes care of freeing the css_id.
         */
-        ss->css_free(cgroup_dummy_top);
+        ss->css_free(cgroup_css(cgroup_dummy_top, ss));
-        cgroup_dummy_top->subsys[ss->subsys_id] = NULL;
+        RCU_INIT_POINTER(cgroup_dummy_top->subsys[ss->subsys_id], NULL);
        mutex_unlock(&cgroup_mutex);
 }
@@ -4917,6 +5112,10 @@ int __init cgroup_init(void)
        BUG_ON(cgroup_init_root_id(&cgroup_dummy_root, 0, 1));
+        err = idr_alloc(&cgroup_dummy_root.cgroup_idr, cgroup_dummy_top,
+                        0, 1, GFP_KERNEL);
+        BUG_ON(err < 0);
        mutex_unlock(&cgroup_root_mutex);
        mutex_unlock(&cgroup_mutex);
@@ -5073,7 +5272,7 @@ void cgroup_fork(struct task_struct *child)
 * Adds the task to the list running through its css_set if necessary and
 * call the subsystem fork() callbacks.  Has to be after the task is
 * visible on the task list in case we race with the first call to
- * cgroup_iter_start() - to guarantee that the new task ends up on its
+ * cgroup_task_iter_start() - to guarantee that the new task ends up on its
 * list.
 */
 void cgroup_post_fork(struct task_struct *child)
@@ -5186,10 +5385,10 @@ void cgroup_exit(struct task_struct *tsk, int run_callbacks)
                 */
                for_each_builtin_subsys(ss, i) {
                        if (ss->exit) {
-                                struct cgroup *old_cgrp = cset->subsys[i]->cgroup;
+                                struct cgroup_subsys_state *old_css = cset->subsys[i];
-                                struct cgroup *cgrp = task_cgroup(tsk, i);
+                                struct cgroup_subsys_state *css = task_css(tsk, i);
-                                ss->exit(cgrp, old_cgrp, tsk);
+                                ss->exit(css, old_css, tsk);
                        }
                }
        }
@@ -5448,20 +5647,16 @@ static int __init_or_module cgroup_init_idr(struct cgroup_subsys *ss,
        return 0;
 }
-static int alloc_css_id(struct cgroup_subsys *ss, struct cgroup *parent,
+static int alloc_css_id(struct cgroup_subsys_state *child_css)
-                        struct cgroup *child)
 {
-        int subsys_id, i, depth = 0;
+        struct cgroup_subsys_state *parent_css = css_parent(child_css);
-        struct cgroup_subsys_state *parent_css, *child_css;
        struct css_id *child_id, *parent_id;
+        int i, depth;
-        subsys_id = ss->subsys_id;
-        parent_css = parent->subsys[subsys_id];
-        child_css = child->subsys[subsys_id];
        parent_id = rcu_dereference_protected(parent_css->id, true);
        depth = parent_id->depth + 1;
-        child_id = get_new_cssid(ss, depth);
+        child_id = get_new_cssid(child_css->ss, depth);
        if (IS_ERR(child_id))
                return PTR_ERR(child_id);
@@ -5499,31 +5694,56 @@ struct cgroup_subsys_state *css_lookup(struct cgroup_subsys *ss, int id)
 }
 EXPORT_SYMBOL_GPL(css_lookup);
-/*
+/**
- * get corresponding css from file open on cgroupfs directory
+ * css_from_dir - get corresponding css from the dentry of a cgroup dir
+ * @dentry: directory dentry of interest
+ * @ss: subsystem of interest
+ *
+ * Must be called under RCU read lock.  The caller is responsible for
+ * pinning the returned css if it needs to be accessed outside the RCU
+ * critical section.
 */
-struct cgroup_subsys_state *cgroup_css_from_dir(struct file *f, int id)
+struct cgroup_subsys_state *css_from_dir(struct dentry *dentry,
+                                         struct cgroup_subsys *ss)
 {
        struct cgroup *cgrp;
-        struct inode *inode;
-        struct cgroup_subsys_state *css;
-        inode = file_inode(f);
+        WARN_ON_ONCE(!rcu_read_lock_held());
-        /* check in cgroup filesystem dir */
-        if (inode->i_op != &cgroup_dir_inode_operations)
+        /* is @dentry a cgroup dir? */
+        if (!dentry->d_inode ||
+            dentry->d_inode->i_op != &cgroup_dir_inode_operations)
                return ERR_PTR(-EBADF);
-        if (id < 0 || id >= CGROUP_SUBSYS_COUNT)
+        cgrp = __d_cgrp(dentry);
-                return ERR_PTR(-EINVAL);
+        return cgroup_css(cgrp, ss) ?: ERR_PTR(-ENOENT);
+}
-        /* get cgroup */
+/**
-        cgrp = __d_cgrp(f->f_dentry);
+ * css_from_id - lookup css by id
-        css = cgrp->subsys[id];
+ * @id: the cgroup id
-        return css ? css : ERR_PTR(-ENOENT);
+ * @ss: cgroup subsys to be looked into
+ *
+ * Returns the css if there's valid one with @id, otherwise returns NULL.
+ * Should be called under rcu_read_lock().
+ */
+struct cgroup_subsys_state *css_from_id(int id, struct cgroup_subsys *ss)
+{
+        struct cgroup *cgrp;
+        rcu_lockdep_assert(rcu_read_lock_held() ||
+                           lockdep_is_held(&cgroup_mutex),
+                           "css_from_id() needs proper protection");
+        cgrp = idr_find(&ss->root->cgroup_idr, id);
+        if (cgrp)
+                return cgroup_css(cgrp, ss);
+        return NULL;
 }
 #ifdef CONFIG_CGROUP_DEBUG
-static struct cgroup_subsys_state *debug_css_alloc(struct cgroup *cgrp)
+static struct cgroup_subsys_state *
+debug_css_alloc(struct cgroup_subsys_state *parent_css)
 {
        struct cgroup_subsys_state *css = kzalloc(sizeof(*css), GFP_KERNEL);
@@ -5533,22 +5753,24 @@ static struct cgroup_subsys_state *debug_css_alloc(struct cgroup *cgrp)
        return css;
 }
-static void debug_css_free(struct cgroup *cgrp)
+static void debug_css_free(struct cgroup_subsys_state *css)
 {
-        kfree(cgrp->subsys[debug_subsys_id]);
+        kfree(css);
 }
-static u64 debug_taskcount_read(struct cgroup *cgrp, struct cftype *cft)
+static u64 debug_taskcount_read(struct cgroup_subsys_state *css,
+                                struct cftype *cft)
 {
-        return cgroup_task_count(cgrp);
+        return cgroup_task_count(css->cgroup);
 }
-static u64 current_css_set_read(struct cgroup *cgrp, struct cftype *cft)
+static u64 current_css_set_read(struct cgroup_subsys_state *css,
+                                struct cftype *cft)
 {
        return (u64)(unsigned long)current->cgroups;
 }
-static u64 current_css_set_refcount_read(struct cgroup *cgrp,
+static u64 current_css_set_refcount_read(struct cgroup_subsys_state *css,
                                         struct cftype *cft)
 {
        u64 count;
@@ -5559,7 +5781,7 @@ static u64 current_css_set_refcount_read(struct cgroup *cgrp,
        return count;
 }
-static int current_css_set_cg_links_read(struct cgroup *cgrp,
+static int current_css_set_cg_links_read(struct cgroup_subsys_state *css,
                                         struct cftype *cft,
                                         struct seq_file *seq)
 {
@@ -5586,14 +5808,13 @@ static int current_css_set_cg_links_read(struct cgroup *cgrp,
 }
 #define MAX_TASKS_SHOWN_PER_CSS 25
-static int cgroup_css_links_read(struct cgroup *cgrp,
+static int cgroup_css_links_read(struct cgroup_subsys_state *css,
-                                 struct cftype *cft,
+                                 struct cftype *cft, struct seq_file *seq)
-                                 struct seq_file *seq)
 {
        struct cgrp_cset_link *link;
        read_lock(&css_set_lock);
-        list_for_each_entry(link, &cgrp->cset_links, cset_link) {
+        list_for_each_entry(link, &css->cgroup->cset_links, cset_link) {
                struct css_set *cset = link->cset;
                struct task_struct *task;
                int count = 0;
@@ -5612,9 +5833,9 @@ static int cgroup_css_links_read(struct cgroup *cgrp,
        return 0;
 }
-static u64 releasable_read(struct cgroup *cgrp, struct cftype *cft)
+static u64 releasable_read(struct cgroup_subsys_state *css, struct cftype *cft)
 {
-        return test_bit(CGRP_RELEASABLE, &cgrp->flags);
+        return test_bit(CGRP_RELEASABLE, &css->cgroup->flags);
 }
 static struct cftype debug_files[] =  {
diff --git a/kernel/cgroup_freezer.c b/kernel/cgroup_freezer.c
index 75dda1ea5026..f0ff64d0ebaa 100644
--- a/kernel/cgroup_freezer.c
+++ b/kernel/cgroup_freezer.c
@@ -45,25 +45,19 @@ struct freezer {
        spinlock_t                      lock;
 };
-static inline struct freezer *cgroup_freezer(struct cgroup *cgroup)
+static inline struct freezer *css_freezer(struct cgroup_subsys_state *css)
 {
-        return container_of(cgroup_subsys_state(cgroup, freezer_subsys_id),
+        return css ? container_of(css, struct freezer, css) : NULL;
-                            struct freezer, css);
 }
 static inline struct freezer *task_freezer(struct task_struct *task)
 {
-        return container_of(task_subsys_state(task, freezer_subsys_id),
+        return css_freezer(task_css(task, freezer_subsys_id));
-                            struct freezer, css);
 }
 static struct freezer *parent_freezer(struct freezer *freezer)
 {
-        struct cgroup *pcg = freezer->css.cgroup->parent;
+        return css_freezer(css_parent(&freezer->css));
-        if (pcg)
-                return cgroup_freezer(pcg);
-        return NULL;
 }
 bool cgroup_freezing(struct task_struct *task)
@@ -92,7 +86,8 @@ static const char *freezer_state_strs(unsigned int state)
 struct cgroup_subsys freezer_subsys;
-static struct cgroup_subsys_state *freezer_css_alloc(struct cgroup *cgroup)
+static struct cgroup_subsys_state *
+freezer_css_alloc(struct cgroup_subsys_state *parent_css)
 {
        struct freezer *freezer;
@@ -105,22 +100,22 @@ static struct cgroup_subsys_state *freezer_css_alloc(struct cgroup *cgroup)
 }
 /**
- * freezer_css_online - commit creation of a freezer cgroup
+ * freezer_css_online - commit creation of a freezer css
- * @cgroup: cgroup being created
+ * @css: css being created
 *
- * We're committing to creation of @cgroup.  Mark it online and inherit
+ * We're committing to creation of @css.  Mark it online and inherit
 * parent's freezing state while holding both parent's and our
 * freezer->lock.
 */
-static int freezer_css_online(struct cgroup *cgroup)
+static int freezer_css_online(struct cgroup_subsys_state *css)
 {
-        struct freezer *freezer = cgroup_freezer(cgroup);
+        struct freezer *freezer = css_freezer(css);
        struct freezer *parent = parent_freezer(freezer);
        /*
         * The following double locking and freezing state inheritance
         * guarantee that @cgroup can never escape ancestors' freezing
-         * states.  See cgroup_for_each_descendant_pre() for details.
+         * states.  See css_for_each_descendant_pre() for details.
         */
        if (parent)
                spin_lock_irq(&parent->lock);
@@ -141,15 +136,15 @@ static int freezer_css_online(struct cgroup *cgroup)
 }
 /**
- * freezer_css_offline - initiate destruction of @cgroup
+ * freezer_css_offline - initiate destruction of a freezer css
- * @cgroup: cgroup being destroyed
+ * @css: css being destroyed
 *
- * @cgroup is going away.  Mark it dead and decrement system_freezing_count
+ * @css is going away.  Mark it dead and decrement system_freezing_count if
- * if it was holding one.
+ * it was holding one.
 */
-static void freezer_css_offline(struct cgroup *cgroup)
+static void freezer_css_offline(struct cgroup_subsys_state *css)
 {
-        struct freezer *freezer = cgroup_freezer(cgroup);
+        struct freezer *freezer = css_freezer(css);
        spin_lock_irq(&freezer->lock);
@@ -161,9 +156,9 @@ static void freezer_css_offline(struct cgroup *cgroup)
        spin_unlock_irq(&freezer->lock);
 }
-static void freezer_css_free(struct cgroup *cgroup)
+static void freezer_css_free(struct cgroup_subsys_state *css)
 {
-        kfree(cgroup_freezer(cgroup));
+        kfree(css_freezer(css));
 }
 /*
@@ -175,25 +170,26 @@ static void freezer_css_free(struct cgroup *cgroup)
 * @freezer->lock.  freezer_attach() makes the new tasks conform to the
 * current state and all following state changes can see the new tasks.
 */
-static void freezer_attach(struct cgroup *new_cgrp, struct cgroup_taskset *tset)
+static void freezer_attach(struct cgroup_subsys_state *new_css,
+                           struct cgroup_taskset *tset)
 {
-        struct freezer *freezer = cgroup_freezer(new_cgrp);
+        struct freezer *freezer = css_freezer(new_css);
        struct task_struct *task;
        bool clear_frozen = false;
        spin_lock_irq(&freezer->lock);
        /*
-         * Make the new tasks conform to the current state of @new_cgrp.
+         * Make the new tasks conform to the current state of @new_css.
         * For simplicity, when migrating any task to a FROZEN cgroup, we
         * revert it to FREEZING and let update_if_frozen() determine the
         * correct state later.
         *
-         * Tasks in @tset are on @new_cgrp but may not conform to its
+         * Tasks in @tset are on @new_css but may not conform to its
         * current state before executing the following - !frozen tasks may
         * be visible in a FROZEN cgroup and frozen tasks in a THAWED one.
         */
-        cgroup_taskset_for_each(task, new_cgrp, tset) {
+        cgroup_taskset_for_each(task, new_css, tset) {
                if (!(freezer->state & CGROUP_FREEZING)) {
                        __thaw_task(task);
                } else {
@@ -231,7 +227,7 @@ static void freezer_fork(struct task_struct *task)
         * The root cgroup is non-freezable, so we can skip the
         * following check.
         */
-        if (!freezer->css.cgroup->parent)
+        if (!parent_freezer(freezer))
                goto out;
        spin_lock_irq(&freezer->lock);
@@ -244,7 +240,7 @@ out:
 /**
 * update_if_frozen - update whether a cgroup finished freezing
- * @cgroup: cgroup of interest
+ * @css: css of interest
 *
 * Once FREEZING is initiated, transition to FROZEN is lazily updated by
 * calling this function.  If the current state is FREEZING but not FROZEN,
@@ -255,14 +251,14 @@ out:
 * update_if_frozen() on all descendants prior to invoking this function.
 *
 * Task states and freezer state might disagree while tasks are being
- * migrated into or out of @cgroup, so we can't verify task states against
+ * migrated into or out of @css, so we can't verify task states against
 * @freezer state here.  See freezer_attach() for details.
 */
-static void update_if_frozen(struct cgroup *cgroup)
+static void update_if_frozen(struct cgroup_subsys_state *css)
 {
-        struct freezer *freezer = cgroup_freezer(cgroup);
+        struct freezer *freezer = css_freezer(css);
-        struct cgroup *pos;
+        struct cgroup_subsys_state *pos;
-        struct cgroup_iter it;
+        struct css_task_iter it;
        struct task_struct *task;
        WARN_ON_ONCE(!rcu_read_lock_held());
@@ -274,8 +270,8 @@ static void update_if_frozen(struct cgroup *cgroup)
                goto out_unlock;
        /* are all (live) children frozen? */
-        cgroup_for_each_child(pos, cgroup) {
+        css_for_each_child(pos, css) {
-                struct freezer *child = cgroup_freezer(pos);
+                struct freezer *child = css_freezer(pos);
                if ((child->state & CGROUP_FREEZER_ONLINE) &&
                    !(child->state & CGROUP_FROZEN))
@@ -283,9 +279,9 @@ static void update_if_frozen(struct cgroup *cgroup)
        }
        /* are all tasks frozen? */
-        cgroup_iter_start(cgroup, &it);
+        css_task_iter_start(css, &it);
-        while ((task = cgroup_iter_next(cgroup, &it))) {
+        while ((task = css_task_iter_next(&it))) {
                if (freezing(task)) {
                        /*
                         * freezer_should_skip() indicates that the task
@@ -300,52 +296,49 @@ static void update_if_frozen(struct cgroup *cgroup)
        freezer->state |= CGROUP_FROZEN;
 out_iter_end:
-        cgroup_iter_end(cgroup, &it);
+        css_task_iter_end(&it);
 out_unlock:
        spin_unlock_irq(&freezer->lock);
 }
-static int freezer_read(struct cgroup *cgroup, struct cftype *cft,
+static int freezer_read(struct cgroup_subsys_state *css, struct cftype *cft,
                        struct seq_file *m)
 {
-        struct cgroup *pos;
+        struct cgroup_subsys_state *pos;
        rcu_read_lock();
        /* update states bottom-up */
-        cgroup_for_each_descendant_post(pos, cgroup)
+        css_for_each_descendant_post(pos, css)
                update_if_frozen(pos);
-        update_if_frozen(cgroup);
        rcu_read_unlock();
-        seq_puts(m, freezer_state_strs(cgroup_freezer(cgroup)->state));
+        seq_puts(m, freezer_state_strs(css_freezer(css)->state));
        seq_putc(m, '\n');
        return 0;
 }
 static void freeze_cgroup(struct freezer *freezer)
 {
-        struct cgroup *cgroup = freezer->css.cgroup;
+        struct css_task_iter it;
-        struct cgroup_iter it;
        struct task_struct *task;
-        cgroup_iter_start(cgroup, &it);
+        css_task_iter_start(&freezer->css, &it);
-        while ((task = cgroup_iter_next(cgroup, &it)))
+        while ((task = css_task_iter_next(&it)))
                freeze_task(task);
-        cgroup_iter_end(cgroup, &it);
+        css_task_iter_end(&it);
 }
 static void unfreeze_cgroup(struct freezer *freezer)
 {
-        struct cgroup *cgroup = freezer->css.cgroup;
+        struct css_task_iter it;
-        struct cgroup_iter it;
        struct task_struct *task;
-        cgroup_iter_start(cgroup, &it);
+        css_task_iter_start(&freezer->css, &it);
-        while ((task = cgroup_iter_next(cgroup, &it)))
+        while ((task = css_task_iter_next(&it)))
                __thaw_task(task);
-        cgroup_iter_end(cgroup, &it);
+        css_task_iter_end(&it);
 }
 /**
@@ -395,12 +388,7 @@ static void freezer_apply_state(struct freezer *freezer, bool freeze,
 */
 static void freezer_change_state(struct freezer *freezer, bool freeze)
 {
-        struct cgroup *pos;
+        struct cgroup_subsys_state *pos;
-        /* update @freezer */
-        spin_lock_irq(&freezer->lock);
-        freezer_apply_state(freezer, freeze, CGROUP_FREEZING_SELF);
-        spin_unlock_irq(&freezer->lock);
        /*
         * Update all its descendants in pre-order traversal.  Each
@@ -408,24 +396,33 @@ static void freezer_change_state(struct freezer *freezer, bool freeze)
         * CGROUP_FREEZING_PARENT.
         */
        rcu_read_lock();
-        cgroup_for_each_descendant_pre(pos, freezer->css.cgroup) {
+        css_for_each_descendant_pre(pos, &freezer->css) {
-                struct freezer *pos_f = cgroup_freezer(pos);
+                struct freezer *pos_f = css_freezer(pos);
                struct freezer *parent = parent_freezer(pos_f);
-                /*
-                 * Our update to @parent->state is already visible which is
-                 * all we need.  No need to lock @parent.  For more info on
-                 * synchronization, see freezer_post_create().
-                 */
                spin_lock_irq(&pos_f->lock);
-                freezer_apply_state(pos_f, parent->state & CGROUP_FREEZING,
-                                    CGROUP_FREEZING_PARENT);
+                if (pos_f == freezer) {
+                        freezer_apply_state(pos_f, freeze,
+                                            CGROUP_FREEZING_SELF);
+                } else {
+                        /*
+                         * Our update to @parent->state is already visible
+                         * which is all we need.  No need to lock @parent.
+                         * For more info on synchronization, see
+                         * freezer_post_create().
+                         */
+                        freezer_apply_state(pos_f,
+                                            parent->state & CGROUP_FREEZING,
+                                            CGROUP_FREEZING_PARENT);
+                }
                spin_unlock_irq(&pos_f->lock);
        }
        rcu_read_unlock();
 }
-static int freezer_write(struct cgroup *cgroup, struct cftype *cft,
+static int freezer_write(struct cgroup_subsys_state *css, struct cftype *cft,
                         const char *buffer)
 {
        bool freeze;
@@ -437,20 +434,22 @@ static int freezer_write(struct cgroup *cgroup, struct cftype *cft,
        else
                return -EINVAL;
-        freezer_change_state(cgroup_freezer(cgroup), freeze);
+        freezer_change_state(css_freezer(css), freeze);
        return 0;
 }
-static u64 freezer_self_freezing_read(struct cgroup *cgroup, struct cftype *cft)
+static u64 freezer_self_freezing_read(struct cgroup_subsys_state *css,
+                                      struct cftype *cft)
 {
-        struct freezer *freezer = cgroup_freezer(cgroup);
+        struct freezer *freezer = css_freezer(css);
        return (bool)(freezer->state & CGROUP_FREEZING_SELF);
 }
-static u64 freezer_parent_freezing_read(struct cgroup *cgroup, struct cftype *cft)
+static u64 freezer_parent_freezing_read(struct cgroup_subsys_state *css,
+                                        struct cftype *cft)
 {
-        struct freezer *freezer = cgroup_freezer(cgroup);
+        struct freezer *freezer = css_freezer(css);
        return (bool)(freezer->state & CGROUP_FREEZING_PARENT);
 }
diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c
index 383f8231e436..247091bf0587 100644
--- a/kernel/context_tracking.c
+++ b/kernel/context_tracking.c
@@ -20,22 +20,33 @@
 #include <linux/hardirq.h>
 #include <linux/export.h>
-DEFINE_PER_CPU(struct context_tracking, context_tracking) = {
+#define CREATE_TRACE_POINTS
-#ifdef CONFIG_CONTEXT_TRACKING_FORCE
+#include <trace/events/context_tracking.h>
-        .active = true,
-#endif
+struct static_key context_tracking_enabled = STATIC_KEY_INIT_FALSE;
-};
+EXPORT_SYMBOL_GPL(context_tracking_enabled);
+DEFINE_PER_CPU(struct context_tracking, context_tracking);
+EXPORT_SYMBOL_GPL(context_tracking);
+void context_tracking_cpu_set(int cpu)
+{
+        if (!per_cpu(context_tracking.active, cpu)) {
+                per_cpu(context_tracking.active, cpu) = true;
+                static_key_slow_inc(&context_tracking_enabled);
+        }
+}
 /**
- * user_enter - Inform the context tracking that the CPU is going to
+ * context_tracking_user_enter - Inform the context tracking that the CPU is going to
- *              enter userspace mode.
+ *                               enter userspace mode.
 *
 * This function must be called right before we switch from the kernel
 * to userspace, when it's guaranteed the remaining kernel instructions
 * to execute won't use any RCU read side critical section because this
 * function sets RCU in extended quiescent state.
 */
-void user_enter(void)
+void context_tracking_user_enter(void)
 {
        unsigned long flags;
@@ -54,17 +65,32 @@ void user_enter(void)
        WARN_ON_ONCE(!current->mm);
        local_irq_save(flags);
-        if (__this_cpu_read(context_tracking.active) &&
+        if ( __this_cpu_read(context_tracking.state) != IN_USER) {
-            __this_cpu_read(context_tracking.state) != IN_USER) {
+                if (__this_cpu_read(context_tracking.active)) {
+                        trace_user_enter(0);
+                        /*
+                         * At this stage, only low level arch entry code remains and
+                         * then we'll run in userspace. We can assume there won't be
+                         * any RCU read-side critical section until the next call to
+                         * user_exit() or rcu_irq_enter(). Let's remove RCU's dependency
+                         * on the tick.
+                         */
+                        vtime_user_enter(current);
+                        rcu_user_enter();
+                }
                /*
-                 * At this stage, only low level arch entry code remains and
+                 * Even if context tracking is disabled on this CPU, because it's outside
-                 * then we'll run in userspace. We can assume there won't be
+                 * the full dynticks mask for example, we still have to keep track of the
-                 * any RCU read-side critical section until the next call to
+                 * context transitions and states to prevent inconsistency on those of
-                 * user_exit() or rcu_irq_enter(). Let's remove RCU's dependency
+                 * other CPUs.
-                 * on the tick.
+                 * If a task triggers an exception in userspace, sleep on the exception
+                 * handler and then migrate to another CPU, that new CPU must know where
+                 * the exception returns by the time we call exception_exit().
+                 * This information can only be provided by the previous CPU when it called
+                 * exception_enter().
+                 * OTOH we can spare the calls to vtime and RCU when context_tracking.active
+                 * is false because we know that CPU is not tickless.
                 */
-                vtime_user_enter(current);
-                rcu_user_enter();
                __this_cpu_write(context_tracking.state, IN_USER);
        }
        local_irq_restore(flags);
@@ -87,10 +113,9 @@ void user_enter(void)
 */
 void __sched notrace preempt_schedule_context(void)
 {
-        struct thread_info *ti = current_thread_info();
        enum ctx_state prev_ctx;
-        if (likely(ti->preempt_count || irqs_disabled()))
+        if (likely(!preemptible()))
                return;
        /*
@@ -112,8 +137,8 @@ EXPORT_SYMBOL_GPL(preempt_schedule_context);
 #endif /* CONFIG_PREEMPT */
 /**
- * user_exit - Inform the context tracking that the CPU is
+ * context_tracking_user_exit - Inform the context tracking that the CPU is
- *             exiting userspace mode and entering the kernel.
+ *                              exiting userspace mode and entering the kernel.
 *
 * This function must be called after we entered the kernel from userspace
 * before any use of RCU read side critical section. This potentially include
@@ -122,7 +147,7 @@ EXPORT_SYMBOL_GPL(preempt_schedule_context);
 * This call supports re-entrancy. This way it can be called from any exception
 * handler without needing to know if we came from userspace or not.
 */
-void user_exit(void)
+void context_tracking_user_exit(void)
 {
        unsigned long flags;
@@ -131,38 +156,22 @@ void user_exit(void)
        local_irq_save(flags);
        if (__this_cpu_read(context_tracking.state) == IN_USER) {
-                /*
+                if (__this_cpu_read(context_tracking.active)) {
-                 * We are going to run code that may use RCU. Inform
+                        /*
-                 * RCU core about that (ie: we may need the tick again).
+                         * We are going to run code that may use RCU. Inform
-                 */
+                         * RCU core about that (ie: we may need the tick again).
-                rcu_user_exit();
+                         */
-                vtime_user_exit(current);
+                        rcu_user_exit();
+                        vtime_user_exit(current);
+                        trace_user_exit(0);
+                }
                __this_cpu_write(context_tracking.state, IN_KERNEL);
        }
        local_irq_restore(flags);
 }
-void guest_enter(void)
-{
-        if (vtime_accounting_enabled())
-                vtime_guest_enter(current);
-        else
-                __guest_enter();
-}
-EXPORT_SYMBOL_GPL(guest_enter);
-void guest_exit(void)
-{
-        if (vtime_accounting_enabled())
-                vtime_guest_exit(current);
-        else
-                __guest_exit();
-}
-EXPORT_SYMBOL_GPL(guest_exit);
 /**
- * context_tracking_task_switch - context switch the syscall callbacks
+ * __context_tracking_task_switch - context switch the syscall callbacks
 * @prev: the task that is being switched out
 * @next: the task that is being switched in
 *
@@ -174,11 +183,19 @@ EXPORT_SYMBOL_GPL(guest_exit);
 * migrate to some CPU that doesn't do the context tracking. As such the TIF
 * flag may not be desired there.
 */
-void context_tracking_task_switch(struct task_struct *prev,
+void __context_tracking_task_switch(struct task_struct *prev,
-                             struct task_struct *next)
+                                    struct task_struct *next)
 {
-        if (__this_cpu_read(context_tracking.active)) {
+        clear_tsk_thread_flag(prev, TIF_NOHZ);
-                clear_tsk_thread_flag(prev, TIF_NOHZ);
+        set_tsk_thread_flag(next, TIF_NOHZ);
-                set_tsk_thread_flag(next, TIF_NOHZ);
-        }
 }
+#ifdef CONFIG_CONTEXT_TRACKING_FORCE
+void __init context_tracking_init(void)
+{
+        int cpu;
+        for_each_possible_cpu(cpu)
+                context_tracking_cpu_set(cpu);
+}
+#endif
diff --git a/kernel/cpu.c b/kernel/cpu.c
index 198a38883e64..d7f07a2da5a6 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -113,7 +113,7 @@ EXPORT_SYMBOL_GPL(put_online_cpus);
 * get_online_cpus() not an api which is called all that often.
 *
 */
-static void cpu_hotplug_begin(void)
+void cpu_hotplug_begin(void)
 {
        cpu_hotplug.active_writer = current;
@@ -127,7 +127,7 @@ static void cpu_hotplug_begin(void)
        }
 }
-static void cpu_hotplug_done(void)
+void cpu_hotplug_done(void)
 {
        cpu_hotplug.active_writer = NULL;
        mutex_unlock(&cpu_hotplug.lock);
@@ -154,10 +154,7 @@ void cpu_hotplug_enable(void)
        cpu_maps_update_done();
 }
-#else /* #if CONFIG_HOTPLUG_CPU */
+#endif  /* CONFIG_HOTPLUG_CPU */
-static void cpu_hotplug_begin(void) {}
-static void cpu_hotplug_done(void) {}
-#endif  /* #else #if CONFIG_HOTPLUG_CPU */
 /* Need to know about CPUs going up/down? */
 int __ref register_cpu_notifier(struct notifier_block *nb)
@@ -366,7 +363,7 @@ EXPORT_SYMBOL(cpu_down);
 #endif /*CONFIG_HOTPLUG_CPU*/
 /* Requires cpu_add_remove_lock to be held */
-static int __cpuinit _cpu_up(unsigned int cpu, int tasks_frozen)
+static int _cpu_up(unsigned int cpu, int tasks_frozen)
 {
        int ret, nr_calls = 0;
        void *hcpu = (void *)(long)cpu;
@@ -419,7 +416,7 @@ out:
        return ret;
 }
-int __cpuinit cpu_up(unsigned int cpu)
+int cpu_up(unsigned int cpu)
 {
        int err = 0;
@@ -618,7 +615,7 @@ core_initcall(cpu_hotplug_pm_sync_init);
 * It must be called by the arch code on the new cpu, before the new cpu
 * enables interrupts and before the "boot" cpu returns from __cpu_up().
 */
-void __cpuinit notify_cpu_starting(unsigned int cpu)
+void notify_cpu_starting(unsigned int cpu)
 {
        unsigned long val = CPU_STARTING;
diff --git a/kernel/cpuset.c b/kernel/cpuset.c
index e5657788fedd..6bf981e13c43 100644
--- a/kernel/cpuset.c
+++ b/kernel/cpuset.c
@@ -68,10 +68,6 @@
 */
 int number_of_cpusets __read_mostly;
-/* Forward declare cgroup structures */
-struct cgroup_subsys cpuset_subsys;
-struct cpuset;
 /* See "Frequency meter" comments, below. */
 struct fmeter {
@@ -115,27 +111,20 @@ struct cpuset {
        int relax_domain_level;
 };
-/* Retrieve the cpuset for a cgroup */
+static inline struct cpuset *css_cs(struct cgroup_subsys_state *css)
-static inline struct cpuset *cgroup_cs(struct cgroup *cgrp)
 {
-        return container_of(cgroup_subsys_state(cgrp, cpuset_subsys_id),
+        return css ? container_of(css, struct cpuset, css) : NULL;
-                            struct cpuset, css);
 }
 /* Retrieve the cpuset for a task */
 static inline struct cpuset *task_cs(struct task_struct *task)
 {
-        return container_of(task_subsys_state(task, cpuset_subsys_id),
+        return css_cs(task_css(task, cpuset_subsys_id));
-                            struct cpuset, css);
 }
-static inline struct cpuset *parent_cs(const struct cpuset *cs)
+static inline struct cpuset *parent_cs(struct cpuset *cs)
 {
-        struct cgroup *pcgrp = cs->css.cgroup->parent;
+        return css_cs(css_parent(&cs->css));
-        if (pcgrp)
-                return cgroup_cs(pcgrp);
-        return NULL;
 }
 #ifdef CONFIG_NUMA
@@ -212,29 +201,30 @@ static struct cpuset top_cpuset = {
 /**
 * cpuset_for_each_child - traverse online children of a cpuset
 * @child_cs: loop cursor pointing to the current child
- * @pos_cgrp: used for iteration
+ * @pos_css: used for iteration
 * @parent_cs: target cpuset to walk children of
 *
 * Walk @child_cs through the online children of @parent_cs.  Must be used
 * with RCU read locked.
 */
-#define cpuset_for_each_child(child_cs, pos_cgrp, parent_cs)            \
+#define cpuset_for_each_child(child_cs, pos_css, parent_cs)             \
-        cgroup_for_each_child((pos_cgrp), (parent_cs)->css.cgroup)      \
+        css_for_each_child((pos_css), &(parent_cs)->css)                \
-                if (is_cpuset_online(((child_cs) = cgroup_cs((pos_cgrp)))))
+                if (is_cpuset_online(((child_cs) = css_cs((pos_css)))))
 /**
 * cpuset_for_each_descendant_pre - pre-order walk of a cpuset's descendants
 * @des_cs: loop cursor pointing to the current descendant
- * @pos_cgrp: used for iteration
+ * @pos_css: used for iteration
 * @root_cs: target cpuset to walk ancestor of
 *
 * Walk @des_cs through the online descendants of @root_cs.  Must be used
- * with RCU read locked.  The caller may modify @pos_cgrp by calling
+ * with RCU read locked.  The caller may modify @pos_css by calling
- * cgroup_rightmost_descendant() to skip subtree.
+ * css_rightmost_descendant() to skip subtree.  @root_cs is included in the
+ * iteration and the first node to be visited.
 */
-#define cpuset_for_each_descendant_pre(des_cs, pos_cgrp, root_cs)       \
+#define cpuset_for_each_descendant_pre(des_cs, pos_css, root_cs)        \
-        cgroup_for_each_descendant_pre((pos_cgrp), (root_cs)->css.cgroup) \
+        css_for_each_descendant_pre((pos_css), &(root_cs)->css)         \
-                if (is_cpuset_online(((des_cs) = cgroup_cs((pos_cgrp)))))
+                if (is_cpuset_online(((des_cs) = css_cs((pos_css)))))
 /*
 * There are two global mutexes guarding cpuset structures - cpuset_mutex
@@ -320,8 +310,7 @@ static struct file_system_type cpuset_fs_type = {
 *
 * Call with callback_mutex held.
 */
-static void guarantee_online_cpus(const struct cpuset *cs,
+static void guarantee_online_cpus(struct cpuset *cs, struct cpumask *pmask)
-                                  struct cpumask *pmask)
 {
        while (!cpumask_intersects(cs->cpus_allowed, cpu_online_mask))
                cs = parent_cs(cs);
@@ -339,7 +328,7 @@ static void guarantee_online_cpus(const struct cpuset *cs,
 *
 * Call with callback_mutex held.
 */
-static void guarantee_online_mems(const struct cpuset *cs, nodemask_t *pmask)
+static void guarantee_online_mems(struct cpuset *cs, nodemask_t *pmask)
 {
        while (!nodes_intersects(cs->mems_allowed, node_states[N_MEMORY]))
                cs = parent_cs(cs);
@@ -384,7 +373,7 @@ static int is_cpuset_subset(const struct cpuset *p, const struct cpuset *q)
 * alloc_trial_cpuset - allocate a trial cpuset
 * @cs: the cpuset that the trial cpuset duplicates
 */
-static struct cpuset *alloc_trial_cpuset(const struct cpuset *cs)
+static struct cpuset *alloc_trial_cpuset(struct cpuset *cs)
 {
        struct cpuset *trial;
@@ -431,9 +420,9 @@ static void free_trial_cpuset(struct cpuset *trial)
 * Return 0 if valid, -errno if not.
 */
-static int validate_change(const struct cpuset *cur, const struct cpuset *trial)
+static int validate_change(struct cpuset *cur, struct cpuset *trial)
 {
-        struct cgroup *cgrp;
+        struct cgroup_subsys_state *css;
        struct cpuset *c, *par;
        int ret;
@@ -441,7 +430,7 @@ static int validate_change(const struct cpuset *cur, const struct cpuset *trial)
        /* Each of our child cpusets must be a subset of us */
        ret = -EBUSY;
-        cpuset_for_each_child(c, cgrp, cur)
+        cpuset_for_each_child(c, css, cur)
                if (!is_cpuset_subset(c, trial))
                        goto out;
@@ -462,7 +451,7 @@ static int validate_change(const struct cpuset *cur, const struct cpuset *trial)
         * overlap
         */
        ret = -EINVAL;
-        cpuset_for_each_child(c, cgrp, par) {
+        cpuset_for_each_child(c, css, par) {
                if ((is_cpu_exclusive(trial) || is_cpu_exclusive(c)) &&
                    c != cur &&
                    cpumask_intersects(trial->cpus_allowed, c->cpus_allowed))
@@ -475,13 +464,17 @@ static int validate_change(const struct cpuset *cur, const struct cpuset *trial)
        /*
         * Cpusets with tasks - existing or newly being attached - can't
-         * have empty cpus_allowed or mems_allowed.
+         * be changed to have empty cpus_allowed or mems_allowed.
         */
        ret = -ENOSPC;
-        if ((cgroup_task_count(cur->css.cgroup) || cur->attach_in_progress) &&
+        if ((cgroup_task_count(cur->css.cgroup) || cur->attach_in_progress)) {
-            (cpumask_empty(trial->cpus_allowed) &&
+                if (!cpumask_empty(cur->cpus_allowed) &&
-             nodes_empty(trial->mems_allowed)))
+                    cpumask_empty(trial->cpus_allowed))
-                goto out;
+                        goto out;
+                if (!nodes_empty(cur->mems_allowed) &&
+                    nodes_empty(trial->mems_allowed))
+                        goto out;
+        }
        ret = 0;
 out:
@@ -511,13 +504,16 @@ static void update_domain_attr_tree(struct sched_domain_attr *dattr,
                                    struct cpuset *root_cs)
 {
        struct cpuset *cp;
-        struct cgroup *pos_cgrp;
+        struct cgroup_subsys_state *pos_css;
        rcu_read_lock();
-        cpuset_for_each_descendant_pre(cp, pos_cgrp, root_cs) {
+        cpuset_for_each_descendant_pre(cp, pos_css, root_cs) {
+                if (cp == root_cs)
+                        continue;
                /* skip the whole subtree if @cp doesn't have any CPU */
                if (cpumask_empty(cp->cpus_allowed)) {
-                        pos_cgrp = cgroup_rightmost_descendant(pos_cgrp);
+                        pos_css = css_rightmost_descendant(pos_css);
                        continue;
                }
@@ -592,7 +588,7 @@ static int generate_sched_domains(cpumask_var_t **domains,
        struct sched_domain_attr *dattr;  /* attributes for custom domains */
        int ndoms = 0;          /* number of sched domains in result */
        int nslot;              /* next empty doms[] struct cpumask slot */
-        struct cgroup *pos_cgrp;
+        struct cgroup_subsys_state *pos_css;
        doms = NULL;
        dattr = NULL;
@@ -621,7 +617,9 @@ static int generate_sched_domains(cpumask_var_t **domains,
        csn = 0;
        rcu_read_lock();
-        cpuset_for_each_descendant_pre(cp, pos_cgrp, &top_cpuset) {
+        cpuset_for_each_descendant_pre(cp, pos_css, &top_cpuset) {
+                if (cp == &top_cpuset)
+                        continue;
                /*
                 * Continue traversing beyond @cp iff @cp has some CPUs and
                 * isn't load balancing.  The former is obvious.  The
@@ -638,7 +636,7 @@ static int generate_sched_domains(cpumask_var_t **domains,
                        csa[csn++] = cp;
                /* skip @cp's subtree */
-                pos_cgrp = cgroup_rightmost_descendant(pos_cgrp);
+                pos_css = css_rightmost_descendant(pos_css);
        }
        rcu_read_unlock();
@@ -833,52 +831,45 @@ static struct cpuset *effective_nodemask_cpuset(struct cpuset *cs)
 /**
 * cpuset_change_cpumask - make a task's cpus_allowed the same as its cpuset's
 * @tsk: task to test
- * @scan: struct cgroup_scanner containing the cgroup of the task
+ * @data: cpuset to @tsk belongs to
 *
- * Called by cgroup_scan_tasks() for each task in a cgroup whose
+ * Called by css_scan_tasks() for each task in a cgroup whose cpus_allowed
- * cpus_allowed mask needs to be changed.
+ * mask needs to be changed.
 *
 * We don't need to re-check for the cgroup/cpuset membership, since we're
 * holding cpuset_mutex at this point.
 */
-static void cpuset_change_cpumask(struct task_struct *tsk,
+static void cpuset_change_cpumask(struct task_struct *tsk, void *data)
-                                  struct cgroup_scanner *scan)
 {
-        struct cpuset *cpus_cs;
+        struct cpuset *cs = data;
+        struct cpuset *cpus_cs = effective_cpumask_cpuset(cs);
-        cpus_cs = effective_cpumask_cpuset(cgroup_cs(scan->cg));
        set_cpus_allowed_ptr(tsk, cpus_cs->cpus_allowed);
 }
 /**
 * update_tasks_cpumask - Update the cpumasks of tasks in the cpuset.
 * @cs: the cpuset in which each task's cpus_allowed mask needs to be changed
- * @heap: if NULL, defer allocating heap memory to cgroup_scan_tasks()
+ * @heap: if NULL, defer allocating heap memory to css_scan_tasks()
 *
 * Called with cpuset_mutex held
 *
- * The cgroup_scan_tasks() function will scan all the tasks in a cgroup,
+ * The css_scan_tasks() function will scan all the tasks in a cgroup,
 * calling callback functions for each.
 *
- * No return value. It's guaranteed that cgroup_scan_tasks() always returns 0
+ * No return value. It's guaranteed that css_scan_tasks() always returns 0
 * if @heap != NULL.
 */
 static void update_tasks_cpumask(struct cpuset *cs, struct ptr_heap *heap)
 {
-        struct cgroup_scanner scan;
+        css_scan_tasks(&cs->css, NULL, cpuset_change_cpumask, cs, heap);
-        scan.cg = cs->css.cgroup;
-        scan.test_task = NULL;
-        scan.process_task = cpuset_change_cpumask;
-        scan.heap = heap;
-        cgroup_scan_tasks(&scan);
 }
 /*
 * update_tasks_cpumask_hier - Update the cpumasks of tasks in the hierarchy.
 * @root_cs: the root cpuset of the hierarchy
 * @update_root: update root cpuset or not?
- * @heap: the heap used by cgroup_scan_tasks()
+ * @heap: the heap used by css_scan_tasks()
 *
 * This will update cpumasks of tasks in @root_cs and all other empty cpusets
 * which take on cpumask of @root_cs.
@@ -889,17 +880,19 @@ static void update_tasks_cpumask_hier(struct cpuset *root_cs,
                                      bool update_root, struct ptr_heap *heap)
 {
        struct cpuset *cp;
-        struct cgroup *pos_cgrp;
+        struct cgroup_subsys_state *pos_css;
-        if (update_root)
-                update_tasks_cpumask(root_cs, heap);
        rcu_read_lock();
-        cpuset_for_each_descendant_pre(cp, pos_cgrp, root_cs) {
+        cpuset_for_each_descendant_pre(cp, pos_css, root_cs) {
-                /* skip the whole subtree if @cp have some CPU */
+                if (cp == root_cs) {
-                if (!cpumask_empty(cp->cpus_allowed)) {
+                        if (!update_root)
-                        pos_cgrp = cgroup_rightmost_descendant(pos_cgrp);
+                                continue;
-                        continue;
+                } else {
+                        /* skip the whole subtree if @cp have some CPU */
+                        if (!cpumask_empty(cp->cpus_allowed)) {
+                                pos_css = css_rightmost_descendant(pos_css);
+                                continue;
+                        }
                }
                if (!css_tryget(&cp->css))
                        continue;
@@ -1055,20 +1048,24 @@ static void cpuset_change_task_nodemask(struct task_struct *tsk,
        task_unlock(tsk);
 }
+struct cpuset_change_nodemask_arg {
+        struct cpuset           *cs;
+        nodemask_t              *newmems;
+};
 /*
 * Update task's mems_allowed and rebind its mempolicy and vmas' mempolicy
 * of it to cpuset's new mems_allowed, and migrate pages to new nodes if
 * memory_migrate flag is set. Called with cpuset_mutex held.
 */
-static void cpuset_change_nodemask(struct task_struct *p,
+static void cpuset_change_nodemask(struct task_struct *p, void *data)
-                                   struct cgroup_scanner *scan)
 {
-        struct cpuset *cs = cgroup_cs(scan->cg);
+        struct cpuset_change_nodemask_arg *arg = data;
+        struct cpuset *cs = arg->cs;
        struct mm_struct *mm;
        int migrate;
-        nodemask_t *newmems = scan->data;
-        cpuset_change_task_nodemask(p, newmems);
+        cpuset_change_task_nodemask(p, arg->newmems);
        mm = get_task_mm(p);
        if (!mm)
@@ -1078,7 +1075,7 @@ static void cpuset_change_nodemask(struct task_struct *p,
        mpol_rebind_mm(mm, &cs->mems_allowed);
        if (migrate)
-                cpuset_migrate_mm(mm, &cs->old_mems_allowed, newmems);
+                cpuset_migrate_mm(mm, &cs->old_mems_allowed, arg->newmems);
        mmput(mm);
 }
@@ -1087,28 +1084,22 @@ static void *cpuset_being_rebound;
 /**
 * update_tasks_nodemask - Update the nodemasks of tasks in the cpuset.
 * @cs: the cpuset in which each task's mems_allowed mask needs to be changed
- * @heap: if NULL, defer allocating heap memory to cgroup_scan_tasks()
+ * @heap: if NULL, defer allocating heap memory to css_scan_tasks()
 *
- * Called with cpuset_mutex held
+ * Called with cpuset_mutex held.  No return value. It's guaranteed that
- * No return value. It's guaranteed that cgroup_scan_tasks() always returns 0
+ * css_scan_tasks() always returns 0 if @heap != NULL.
- * if @heap != NULL.
 */
 static void update_tasks_nodemask(struct cpuset *cs, struct ptr_heap *heap)
 {
        static nodemask_t newmems;      /* protected by cpuset_mutex */
-        struct cgroup_scanner scan;
        struct cpuset *mems_cs = effective_nodemask_cpuset(cs);
+        struct cpuset_change_nodemask_arg arg = { .cs = cs,
+                                                  .newmems = &newmems };
        cpuset_being_rebound = cs;              /* causes mpol_dup() rebind */
        guarantee_online_mems(mems_cs, &newmems);
-        scan.cg = cs->css.cgroup;
-        scan.test_task = NULL;
-        scan.process_task = cpuset_change_nodemask;
-        scan.heap = heap;
-        scan.data = &newmems;
        /*
         * The mpol_rebind_mm() call takes mmap_sem, which we couldn't
         * take while holding tasklist_lock.  Forks can happen - the
@@ -1119,7 +1110,7 @@ static void update_tasks_nodemask(struct cpuset *cs, struct ptr_heap *heap)
         * It's ok if we rebind the same mm twice; mpol_rebind_mm()
         * is idempotent.  Also migrate pages in each mm to new nodes.
         */
-        cgroup_scan_tasks(&scan);
+        css_scan_tasks(&cs->css, NULL, cpuset_change_nodemask, &arg, heap);
        /*
         * All the tasks' nodemasks have been updated, update
@@ -1135,7 +1126,7 @@ static void update_tasks_nodemask(struct cpuset *cs, struct ptr_heap *heap)
 * update_tasks_nodemask_hier - Update the nodemasks of tasks in the hierarchy.
 * @cs: the root cpuset of the hierarchy
 * @update_root: update the root cpuset or not?
- * @heap: the heap used by cgroup_scan_tasks()
+ * @heap: the heap used by css_scan_tasks()
 *
 * This will update nodemasks of tasks in @root_cs and all other empty cpusets
 * which take on nodemask of @root_cs.
@@ -1146,17 +1137,19 @@ static void update_tasks_nodemask_hier(struct cpuset *root_cs,
                                       bool update_root, struct ptr_heap *heap)
 {
        struct cpuset *cp;
-        struct cgroup *pos_cgrp;
+        struct cgroup_subsys_state *pos_css;
-        if (update_root)
-                update_tasks_nodemask(root_cs, heap);
        rcu_read_lock();
-        cpuset_for_each_descendant_pre(cp, pos_cgrp, root_cs) {
+        cpuset_for_each_descendant_pre(cp, pos_css, root_cs) {
-                /* skip the whole subtree if @cp have some CPU */
+                if (cp == root_cs) {
-                if (!nodes_empty(cp->mems_allowed)) {
+                        if (!update_root)
-                        pos_cgrp = cgroup_rightmost_descendant(pos_cgrp);
+                                continue;
-                        continue;
+                } else {
+                        /* skip the whole subtree if @cp have some CPU */
+                        if (!nodes_empty(cp->mems_allowed)) {
+                                pos_css = css_rightmost_descendant(pos_css);
+                                continue;
+                        }
                }
                if (!css_tryget(&cp->css))
                        continue;
@@ -1263,44 +1256,39 @@ static int update_relax_domain_level(struct cpuset *cs, s64 val)
        return 0;
 }
-/*
+/**
 * cpuset_change_flag - make a task's spread flags the same as its cpuset's
 * @tsk: task to be updated
- * @scan: struct cgroup_scanner containing the cgroup of the task
+ * @data: cpuset to @tsk belongs to
 *
- * Called by cgroup_scan_tasks() for each task in a cgroup.
+ * Called by css_scan_tasks() for each task in a cgroup.
 *
 * We don't need to re-check for the cgroup/cpuset membership, since we're
 * holding cpuset_mutex at this point.
 */
-static void cpuset_change_flag(struct task_struct *tsk,
+static void cpuset_change_flag(struct task_struct *tsk, void *data)
-                                struct cgroup_scanner *scan)
 {
-        cpuset_update_task_spread_flag(cgroup_cs(scan->cg), tsk);
+        struct cpuset *cs = data;
+        cpuset_update_task_spread_flag(cs, tsk);
 }
-/*
+/**
 * update_tasks_flags - update the spread flags of tasks in the cpuset.
 * @cs: the cpuset in which each task's spread flags needs to be changed
- * @heap: if NULL, defer allocating heap memory to cgroup_scan_tasks()
+ * @heap: if NULL, defer allocating heap memory to css_scan_tasks()
 *
 * Called with cpuset_mutex held
 *
- * The cgroup_scan_tasks() function will scan all the tasks in a cgroup,
+ * The css_scan_tasks() function will scan all the tasks in a cgroup,
 * calling callback functions for each.
 *
- * No return value. It's guaranteed that cgroup_scan_tasks() always returns 0
+ * No return value. It's guaranteed that css_scan_tasks() always returns 0
 * if @heap != NULL.
 */
 static void update_tasks_flags(struct cpuset *cs, struct ptr_heap *heap)
 {
-        struct cgroup_scanner scan;
+        css_scan_tasks(&cs->css, NULL, cpuset_change_flag, cs, heap);
-        scan.cg = cs->css.cgroup;
-        scan.test_task = NULL;
-        scan.process_task = cpuset_change_flag;
-        scan.heap = heap;
-        cgroup_scan_tasks(&scan);
 }
 /*
@@ -1458,9 +1446,10 @@ static int fmeter_getrate(struct fmeter *fmp)
 }
 /* Called by cgroups to determine if a cpuset is usable; cpuset_mutex held */
-static int cpuset_can_attach(struct cgroup *cgrp, struct cgroup_taskset *tset)
+static int cpuset_can_attach(struct cgroup_subsys_state *css,
+                             struct cgroup_taskset *tset)
 {
-        struct cpuset *cs = cgroup_cs(cgrp);
+        struct cpuset *cs = css_cs(css);
        struct task_struct *task;
        int ret;
@@ -1471,11 +1460,11 @@ static int cpuset_can_attach(struct cgroup *cgrp, struct cgroup_taskset *tset)
         * flag is set.
         */
        ret = -ENOSPC;
-        if (!cgroup_sane_behavior(cgrp) &&
+        if (!cgroup_sane_behavior(css->cgroup) &&
            (cpumask_empty(cs->cpus_allowed) || nodes_empty(cs->mems_allowed)))
                goto out_unlock;
-        cgroup_taskset_for_each(task, cgrp, tset) {
+        cgroup_taskset_for_each(task, css, tset) {
                /*
                 * Kthreads which disallow setaffinity shouldn't be moved
                 * to a new cpuset; we don't want to change their cpu
@@ -1504,11 +1493,11 @@ out_unlock:
        return ret;
 }
-static void cpuset_cancel_attach(struct cgroup *cgrp,
+static void cpuset_cancel_attach(struct cgroup_subsys_state *css,
                                 struct cgroup_taskset *tset)
 {
        mutex_lock(&cpuset_mutex);
-        cgroup_cs(cgrp)->attach_in_progress--;
+        css_cs(css)->attach_in_progress--;
        mutex_unlock(&cpuset_mutex);
 }
@@ -1519,16 +1508,18 @@ static void cpuset_cancel_attach(struct cgroup *cgrp,
 */
 static cpumask_var_t cpus_attach;
-static void cpuset_attach(struct cgroup *cgrp, struct cgroup_taskset *tset)
+static void cpuset_attach(struct cgroup_subsys_state *css,
+                          struct cgroup_taskset *tset)
 {
        /* static buf protected by cpuset_mutex */
        static nodemask_t cpuset_attach_nodemask_to;
        struct mm_struct *mm;
        struct task_struct *task;
        struct task_struct *leader = cgroup_taskset_first(tset);
-        struct cgroup *oldcgrp = cgroup_taskset_cur_cgroup(tset);
+        struct cgroup_subsys_state *oldcss = cgroup_taskset_cur_css(tset,
-        struct cpuset *cs = cgroup_cs(cgrp);
+                                                        cpuset_subsys_id);
-        struct cpuset *oldcs = cgroup_cs(oldcgrp);
+        struct cpuset *cs = css_cs(css);
+        struct cpuset *oldcs = css_cs(oldcss);
        struct cpuset *cpus_cs = effective_cpumask_cpuset(cs);
        struct cpuset *mems_cs = effective_nodemask_cpuset(cs);
@@ -1542,7 +1533,7 @@ static void cpuset_attach(struct cgroup *cgrp, struct cgroup_taskset *tset)
        guarantee_online_mems(mems_cs, &cpuset_attach_nodemask_to);
-        cgroup_taskset_for_each(task, cgrp, tset) {
+        cgroup_taskset_for_each(task, css, tset) {
                /*
                 * can_attach beforehand should guarantee that this doesn't
                 * fail.  TODO: have a better way to handle failure here
@@ -1604,15 +1595,18 @@ typedef enum {
        FILE_SPREAD_SLAB,
 } cpuset_filetype_t;
-static int cpuset_write_u64(struct cgroup *cgrp, struct cftype *cft, u64 val)
+static int cpuset_write_u64(struct cgroup_subsys_state *css, struct cftype *cft,
+                            u64 val)
 {
-        struct cpuset *cs = cgroup_cs(cgrp);
+        struct cpuset *cs = css_cs(css);
        cpuset_filetype_t type = cft->private;
-        int retval = -ENODEV;
+        int retval = 0;
        mutex_lock(&cpuset_mutex);
-        if (!is_cpuset_online(cs))
+        if (!is_cpuset_online(cs)) {
+                retval = -ENODEV;
                goto out_unlock;
+        }
        switch (type) {
        case FILE_CPU_EXCLUSIVE:
@@ -1651,9 +1645,10 @@ out_unlock:
        return retval;
 }
-static int cpuset_write_s64(struct cgroup *cgrp, struct cftype *cft, s64 val)
+static int cpuset_write_s64(struct cgroup_subsys_state *css, struct cftype *cft,
+                            s64 val)
 {
-        struct cpuset *cs = cgroup_cs(cgrp);
+        struct cpuset *cs = css_cs(css);
        cpuset_filetype_t type = cft->private;
        int retval = -ENODEV;
@@ -1677,10 +1672,10 @@ out_unlock:
 /*
 * Common handling for a write to a "cpus" or "mems" file.
 */
-static int cpuset_write_resmask(struct cgroup *cgrp, struct cftype *cft,
+static int cpuset_write_resmask(struct cgroup_subsys_state *css,
-                                const char *buf)
+                                struct cftype *cft, const char *buf)
 {
-        struct cpuset *cs = cgroup_cs(cgrp);
+        struct cpuset *cs = css_cs(css);
        struct cpuset *trialcs;
        int retval = -ENODEV;
@@ -1759,13 +1754,12 @@ static size_t cpuset_sprintf_memlist(char *page, struct cpuset *cs)
        return count;
 }
-static ssize_t cpuset_common_file_read(struct cgroup *cgrp,
+static ssize_t cpuset_common_file_read(struct cgroup_subsys_state *css,
-                                       struct cftype *cft,
+                                       struct cftype *cft, struct file *file,
-                                       struct file *file,
+                                       char __user *buf, size_t nbytes,
-                                       char __user *buf,
+                                       loff_t *ppos)
-                                       size_t nbytes, loff_t *ppos)
 {
-        struct cpuset *cs = cgroup_cs(cgrp);
+        struct cpuset *cs = css_cs(css);
        cpuset_filetype_t type = cft->private;
        char *page;
        ssize_t retval = 0;
@@ -1795,9 +1789,9 @@ out:
        return retval;
 }
-static u64 cpuset_read_u64(struct cgroup *cgrp, struct cftype *cft)
+static u64 cpuset_read_u64(struct cgroup_subsys_state *css, struct cftype *cft)
 {
-        struct cpuset *cs = cgroup_cs(cgrp);
+        struct cpuset *cs = css_cs(css);
        cpuset_filetype_t type = cft->private;
        switch (type) {
        case FILE_CPU_EXCLUSIVE:
@@ -1826,9 +1820,9 @@ static u64 cpuset_read_u64(struct cgroup *cgrp, struct cftype *cft)
        return 0;
 }
-static s64 cpuset_read_s64(struct cgroup *cgrp, struct cftype *cft)
+static s64 cpuset_read_s64(struct cgroup_subsys_state *css, struct cftype *cft)
 {
-        struct cpuset *cs = cgroup_cs(cgrp);
+        struct cpuset *cs = css_cs(css);
        cpuset_filetype_t type = cft->private;
        switch (type) {
        case FILE_SCHED_RELAX_DOMAIN_LEVEL:
@@ -1943,11 +1937,12 @@ static struct cftype files[] = {
 *      cgrp:   control group that the new cpuset will be part of
 */
-static struct cgroup_subsys_state *cpuset_css_alloc(struct cgroup *cgrp)
+static struct cgroup_subsys_state *
+cpuset_css_alloc(struct cgroup_subsys_state *parent_css)
 {
        struct cpuset *cs;
-        if (!cgrp->parent)
+        if (!parent_css)
                return &top_cpuset.css;
        cs = kzalloc(sizeof(*cs), GFP_KERNEL);
@@ -1967,12 +1962,12 @@ static struct cgroup_subsys_state *cpuset_css_alloc(struct cgroup *cgrp)
        return &cs->css;
 }
-static int cpuset_css_online(struct cgroup *cgrp)
+static int cpuset_css_online(struct cgroup_subsys_state *css)
 {
-        struct cpuset *cs = cgroup_cs(cgrp);
+        struct cpuset *cs = css_cs(css);
        struct cpuset *parent = parent_cs(cs);
        struct cpuset *tmp_cs;
-        struct cgroup *pos_cg;
+        struct cgroup_subsys_state *pos_css;
        if (!parent)
                return 0;
@@ -1987,7 +1982,7 @@ static int cpuset_css_online(struct cgroup *cgrp)
        number_of_cpusets++;
-        if (!test_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags))
+        if (!test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags))
                goto out_unlock;
        /*
@@ -2004,7 +1999,7 @@ static int cpuset_css_online(struct cgroup *cgrp)
         * (and likewise for mems) to the new cgroup.
         */
        rcu_read_lock();
-        cpuset_for_each_child(tmp_cs, pos_cg, parent) {
+        cpuset_for_each_child(tmp_cs, pos_css, parent) {
                if (is_mem_exclusive(tmp_cs) || is_cpu_exclusive(tmp_cs)) {
                        rcu_read_unlock();
                        goto out_unlock;
@@ -2021,9 +2016,15 @@ out_unlock:
        return 0;
 }
-static void cpuset_css_offline(struct cgroup *cgrp)
+/*
+ * If the cpuset being removed has its flag 'sched_load_balance'
+ * enabled, then simulate turning sched_load_balance off, which
+ * will call rebuild_sched_domains_locked().
+ */
+static void cpuset_css_offline(struct cgroup_subsys_state *css)
 {
-        struct cpuset *cs = cgroup_cs(cgrp);
+        struct cpuset *cs = css_cs(css);
        mutex_lock(&cpuset_mutex);
@@ -2036,15 +2037,9 @@ static void cpuset_css_offline(struct cgroup *cgrp)
        mutex_unlock(&cpuset_mutex);
 }
-/*
+static void cpuset_css_free(struct cgroup_subsys_state *css)
- * If the cpuset being removed has its flag 'sched_load_balance'
- * enabled, then simulate turning sched_load_balance off, which
- * will call rebuild_sched_domains_locked().
- */
-static void cpuset_css_free(struct cgroup *cgrp)
 {
-        struct cpuset *cs = cgroup_cs(cgrp);
+        struct cpuset *cs = css_cs(css);
        free_cpumask_var(cs->cpus_allowed);
        kfree(cs);
@@ -2251,11 +2246,11 @@ static void cpuset_hotplug_workfn(struct work_struct *work)
        /* if cpus or mems changed, we need to propagate to descendants */
        if (cpus_updated || mems_updated) {
                struct cpuset *cs;
-                struct cgroup *pos_cgrp;
+                struct cgroup_subsys_state *pos_css;
                rcu_read_lock();
-                cpuset_for_each_descendant_pre(cs, pos_cgrp, &top_cpuset) {
+                cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) {
-                        if (!css_tryget(&cs->css))
+                        if (cs == &top_cpuset || !css_tryget(&cs->css))
                                continue;
                        rcu_read_unlock();
@@ -2344,7 +2339,7 @@ void cpuset_cpus_allowed(struct task_struct *tsk, struct cpumask *pmask)
 void cpuset_cpus_allowed_fallback(struct task_struct *tsk)
 {
-        const struct cpuset *cpus_cs;
+        struct cpuset *cpus_cs;
        rcu_read_lock();
        cpus_cs = effective_cpumask_cpuset(task_cs(tsk));
@@ -2417,7 +2412,7 @@ int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask)
 * callback_mutex.  If no ancestor is mem_exclusive or mem_hardwall
 * (an unusual configuration), then returns the root cpuset.
 */
-static const struct cpuset *nearest_hardwall_ancestor(const struct cpuset *cs)
+static struct cpuset *nearest_hardwall_ancestor(struct cpuset *cs)
 {
        while (!(is_mem_exclusive(cs) || is_mem_hardwall(cs)) && parent_cs(cs))
                cs = parent_cs(cs);
@@ -2487,7 +2482,7 @@ static const struct cpuset *nearest_hardwall_ancestor(const struct cpuset *cs)
 */
 int __cpuset_node_allowed_softwall(int node, gfp_t gfp_mask)
 {
-        const struct cpuset *cs;        /* current cpuset ancestors */
+        struct cpuset *cs;              /* current cpuset ancestors */
        int allowed;                    /* is allocation in zone z allowed? */
        if (in_interrupt() || (gfp_mask & __GFP_THISNODE))
@@ -2725,7 +2720,7 @@ int proc_cpuset_show(struct seq_file *m, void *unused_v)
                goto out_free;
        rcu_read_lock();
-        css = task_subsys_state(tsk, cpuset_subsys_id);
+        css = task_css(tsk, cpuset_subsys_id);
        retval = cgroup_path(css->cgroup, buf, PAGE_SIZE);
        rcu_read_unlock();
        if (retval < 0)
diff --git a/kernel/events/callchain.c b/kernel/events/callchain.c
index c77206184b8b..97b67df8fbfe 100644
--- a/kernel/events/callchain.c
+++ b/kernel/events/callchain.c
@@ -116,6 +116,9 @@ int get_callchain_buffers(void)
        err = alloc_callchain_buffers();
 exit:
+        if (err)
+                atomic_dec(&nr_callchain_events);
        mutex_unlock(&callchain_mutex);
        return err;
diff --git a/kernel/events/core.c b/kernel/events/core.c
index eba8fb5834ae..2207efc941d1 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -145,6 +145,7 @@ static DEFINE_PER_CPU(atomic_t, perf_branch_stack_events);
 static atomic_t nr_mmap_events __read_mostly;
 static atomic_t nr_comm_events __read_mostly;
 static atomic_t nr_task_events __read_mostly;
+static atomic_t nr_freq_events __read_mostly;
 static LIST_HEAD(pmus);
 static DEFINE_MUTEX(pmus_lock);
@@ -340,8 +341,8 @@ struct perf_cgroup {
 static inline struct perf_cgroup *
 perf_cgroup_from_task(struct task_struct *task)
 {
-        return container_of(task_subsys_state(task, perf_subsys_id),
+        return container_of(task_css(task, perf_subsys_id),
-                        struct perf_cgroup, css);
+                            struct perf_cgroup, css);
 }
 static inline bool
@@ -591,7 +592,9 @@ static inline int perf_cgroup_connect(int fd, struct perf_event *event,
        if (!f.file)
                return -EBADF;
-        css = cgroup_css_from_dir(f.file, perf_subsys_id);
+        rcu_read_lock();
+        css = css_from_dir(f.file->f_dentry, &perf_subsys);
        if (IS_ERR(css)) {
                ret = PTR_ERR(css);
                goto out;
@@ -617,6 +620,7 @@ static inline int perf_cgroup_connect(int fd, struct perf_event *event,
                ret = -EINVAL;
        }
 out:
+        rcu_read_unlock();
        fdput(f);
        return ret;
 }
@@ -869,12 +873,8 @@ static void perf_pmu_rotate_start(struct pmu *pmu)
        WARN_ON(!irqs_disabled());
-        if (list_empty(&cpuctx->rotation_list)) {
+        if (list_empty(&cpuctx->rotation_list))
-                int was_empty = list_empty(head);
                list_add(&cpuctx->rotation_list, head);
-                if (was_empty)
-                        tick_nohz_full_kick();
-        }
 }
 static void get_ctx(struct perf_event_context *ctx)
@@ -1216,6 +1216,9 @@ static void perf_event__id_header_size(struct perf_event *event)
        if (sample_type & PERF_SAMPLE_TIME)
                size += sizeof(data->time);
+        if (sample_type & PERF_SAMPLE_IDENTIFIER)
+                size += sizeof(data->id);
        if (sample_type & PERF_SAMPLE_ID)
                size += sizeof(data->id);
@@ -2712,7 +2715,7 @@ static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx,
                hwc = &event->hw;
-                if (needs_unthr && hwc->interrupts == MAX_INTERRUPTS) {
+                if (hwc->interrupts == MAX_INTERRUPTS) {
                        hwc->interrupts = 0;
                        perf_log_throttle(event, 1);
                        event->pmu->start(event, 0);
@@ -2811,10 +2814,11 @@ done:
 #ifdef CONFIG_NO_HZ_FULL
 bool perf_event_can_stop_tick(void)
 {
-        if (list_empty(&__get_cpu_var(rotation_list)))
+        if (atomic_read(&nr_freq_events) ||
-                return true;
+            __this_cpu_read(perf_throttled_count))
-        else
                return false;
+        else
+                return true;
 }
 #endif
@@ -3128,36 +3132,63 @@ static void free_event_rcu(struct rcu_head *head)
 static void ring_buffer_put(struct ring_buffer *rb);
 static void ring_buffer_detach(struct perf_event *event, struct ring_buffer *rb);
-static void free_event(struct perf_event *event)
+static void unaccount_event_cpu(struct perf_event *event, int cpu)
 {
-        irq_work_sync(&event->pending);
+        if (event->parent)
+                return;
+        if (has_branch_stack(event)) {
+                if (!(event->attach_state & PERF_ATTACH_TASK))
+                        atomic_dec(&per_cpu(perf_branch_stack_events, cpu));
+        }
+        if (is_cgroup_event(event))
+                atomic_dec(&per_cpu(perf_cgroup_events, cpu));
+}
+static void unaccount_event(struct perf_event *event)
+{
+        if (event->parent)
+                return;
+        if (event->attach_state & PERF_ATTACH_TASK)
+                static_key_slow_dec_deferred(&perf_sched_events);
+        if (event->attr.mmap || event->attr.mmap_data)
+                atomic_dec(&nr_mmap_events);
+        if (event->attr.comm)
+                atomic_dec(&nr_comm_events);
+        if (event->attr.task)
+                atomic_dec(&nr_task_events);
+        if (event->attr.freq)
+                atomic_dec(&nr_freq_events);
+        if (is_cgroup_event(event))
+                static_key_slow_dec_deferred(&perf_sched_events);
+        if (has_branch_stack(event))
+                static_key_slow_dec_deferred(&perf_sched_events);
+        unaccount_event_cpu(event, event->cpu);
+}
+static void __free_event(struct perf_event *event)
+{
        if (!event->parent) {
-                if (event->attach_state & PERF_ATTACH_TASK)
-                        static_key_slow_dec_deferred(&perf_sched_events);
-                if (event->attr.mmap || event->attr.mmap_data)
-                        atomic_dec(&nr_mmap_events);
-                if (event->attr.comm)
-                        atomic_dec(&nr_comm_events);
-                if (event->attr.task)
-                        atomic_dec(&nr_task_events);
                if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
                        put_callchain_buffers();
-                if (is_cgroup_event(event)) {
-                        atomic_dec(&per_cpu(perf_cgroup_events, event->cpu));
-                        static_key_slow_dec_deferred(&perf_sched_events);
-                }
-                if (has_branch_stack(event)) {
-                        static_key_slow_dec_deferred(&perf_sched_events);
-                        /* is system-wide event */
-                        if (!(event->attach_state & PERF_ATTACH_TASK)) {
-                                atomic_dec(&per_cpu(perf_branch_stack_events,
-                                                    event->cpu));
-                        }
-                }
        }
+        if (event->destroy)
+                event->destroy(event);
+        if (event->ctx)
+                put_ctx(event->ctx);
+        call_rcu(&event->rcu_head, free_event_rcu);
+}
+static void free_event(struct perf_event *event)
+{
+        irq_work_sync(&event->pending);
+        unaccount_event(event);
        if (event->rb) {
                struct ring_buffer *rb;
@@ -3180,13 +3211,8 @@ static void free_event(struct perf_event *event)
        if (is_cgroup_event(event))
                perf_detach_cgroup(event);
-        if (event->destroy)
-                event->destroy(event);
-        if (event->ctx)
+        __free_event(event);
-                put_ctx(event->ctx);
-        call_rcu(&event->rcu_head, free_event_rcu);
 }
 int perf_event_release_kernel(struct perf_event *event)
@@ -3544,6 +3570,15 @@ static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
        case PERF_EVENT_IOC_PERIOD:
                return perf_event_period(event, (u64 __user *)arg);
+        case PERF_EVENT_IOC_ID:
+        {
+                u64 id = primary_event_id(event);
+                if (copy_to_user((void __user *)arg, &id, sizeof(id)))
+                        return -EFAULT;
+                return 0;
+        }
        case PERF_EVENT_IOC_SET_OUTPUT:
        {
                int ret;
@@ -3641,6 +3676,10 @@ void perf_event_update_userpage(struct perf_event *event)
        u64 enabled, running, now;
        rcu_read_lock();
+        rb = rcu_dereference(event->rb);
+        if (!rb)
+                goto unlock;
        /*
         * compute total_time_enabled, total_time_running
         * based on snapshot values taken when the event
@@ -3651,12 +3690,8 @@ void perf_event_update_userpage(struct perf_event *event)
         * NMI context
         */
        calc_timer_values(event, &now, &enabled, &running);
-        rb = rcu_dereference(event->rb);
-        if (!rb)
-                goto unlock;
        userpg = rb->user_page;
        /*
         * Disable preemption so as to not let the corresponding user-space
         * spin too long if we get preempted.
@@ -4251,7 +4286,7 @@ static void __perf_event_header__init_id(struct perf_event_header *header,
        if (sample_type & PERF_SAMPLE_TIME)
                data->time = perf_clock();
-        if (sample_type & PERF_SAMPLE_ID)
+        if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER))
                data->id = primary_event_id(event);
        if (sample_type & PERF_SAMPLE_STREAM_ID)
@@ -4290,6 +4325,9 @@ static void __perf_event__output_id_sample(struct perf_output_handle *handle,
        if (sample_type & PERF_SAMPLE_CPU)
                perf_output_put(handle, data->cpu_entry);
+        if (sample_type & PERF_SAMPLE_IDENTIFIER)
+                perf_output_put(handle, data->id);
 }
 void perf_event__output_id_sample(struct perf_event *event,
@@ -4355,7 +4393,8 @@ static void perf_output_read_group(struct perf_output_handle *handle,
        list_for_each_entry(sub, &leader->sibling_list, group_entry) {
                n = 0;
-                if (sub != event)
+                if ((sub != event) &&
+                    (sub->state == PERF_EVENT_STATE_ACTIVE))
                        sub->pmu->read(sub);
                values[n++] = perf_event_count(sub);
@@ -4402,6 +4441,9 @@ void perf_output_sample(struct perf_output_handle *handle,
        perf_output_put(handle, *header);
+        if (sample_type & PERF_SAMPLE_IDENTIFIER)
+                perf_output_put(handle, data->id);
        if (sample_type & PERF_SAMPLE_IP)
                perf_output_put(handle, data->ip);
@@ -4462,20 +4504,6 @@ void perf_output_sample(struct perf_output_handle *handle,
                }
        }
-        if (!event->attr.watermark) {
-                int wakeup_events = event->attr.wakeup_events;
-                if (wakeup_events) {
-                        struct ring_buffer *rb = handle->rb;
-                        int events = local_inc_return(&rb->events);
-                        if (events >= wakeup_events) {
-                                local_sub(wakeup_events, &rb->events);
-                                local_inc(&rb->wakeup);
-                        }
-                }
-        }
        if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
                if (data->br_stack) {
                        size_t size;
@@ -4511,16 +4539,31 @@ void perf_output_sample(struct perf_output_handle *handle,
                }
        }
-        if (sample_type & PERF_SAMPLE_STACK_USER)
+        if (sample_type & PERF_SAMPLE_STACK_USER) {
                perf_output_sample_ustack(handle,
                                          data->stack_user_size,
                                          data->regs_user.regs);
+        }
        if (sample_type & PERF_SAMPLE_WEIGHT)
                perf_output_put(handle, data->weight);
        if (sample_type & PERF_SAMPLE_DATA_SRC)
                perf_output_put(handle, data->data_src.val);
+        if (!event->attr.watermark) {
+                int wakeup_events = event->attr.wakeup_events;
+                if (wakeup_events) {
+                        struct ring_buffer *rb = handle->rb;
+                        int events = local_inc_return(&rb->events);
+                        if (events >= wakeup_events) {
+                                local_sub(wakeup_events, &rb->events);
+                                local_inc(&rb->wakeup);
+                        }
+                }
+        }
 }
 void perf_prepare_sample(struct perf_event_header *header,
@@ -4680,12 +4723,10 @@ perf_event_read_event(struct perf_event *event,
        perf_output_end(&handle);
 }
-typedef int  (perf_event_aux_match_cb)(struct perf_event *event, void *data);
 typedef void (perf_event_aux_output_cb)(struct perf_event *event, void *data);
 static void
 perf_event_aux_ctx(struct perf_event_context *ctx,
-                   perf_event_aux_match_cb match,
                   perf_event_aux_output_cb output,
                   void *data)
 {
@@ -4696,15 +4737,12 @@ perf_event_aux_ctx(struct perf_event_context *ctx,
                        continue;
                if (!event_filter_match(event))
                        continue;
-                if (match(event, data))
+                output(event, data);
-                        output(event, data);
        }
 }
 static void
-perf_event_aux(perf_event_aux_match_cb match,
+perf_event_aux(perf_event_aux_output_cb output, void *data,
-               perf_event_aux_output_cb output,
-               void *data,
               struct perf_event_context *task_ctx)
 {
        struct perf_cpu_context *cpuctx;
@@ -4717,7 +4755,7 @@ perf_event_aux(perf_event_aux_match_cb match,
                cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
                if (cpuctx->unique_pmu != pmu)
                        goto next;
-                perf_event_aux_ctx(&cpuctx->ctx, match, output, data);
+                perf_event_aux_ctx(&cpuctx->ctx, output, data);
                if (task_ctx)
                        goto next;
                ctxn = pmu->task_ctx_nr;
@@ -4725,14 +4763,14 @@ perf_event_aux(perf_event_aux_match_cb match,
                        goto next;
                ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
                if (ctx)
-                        perf_event_aux_ctx(ctx, match, output, data);
+                        perf_event_aux_ctx(ctx, output, data);
 next:
                put_cpu_ptr(pmu->pmu_cpu_context);
        }
        if (task_ctx) {
                preempt_disable();
-                perf_event_aux_ctx(task_ctx, match, output, data);
+                perf_event_aux_ctx(task_ctx, output, data);
                preempt_enable();
        }
        rcu_read_unlock();
@@ -4741,7 +4779,7 @@ next:
 /*
 * task tracking -- fork/exit
 *
- * enabled by: attr.comm | attr.mmap | attr.mmap_data | attr.task
+ * enabled by: attr.comm | attr.mmap | attr.mmap2 | attr.mmap_data | attr.task
 */
 struct perf_task_event {
@@ -4759,6 +4797,13 @@ struct perf_task_event {
        } event_id;
 };
+static int perf_event_task_match(struct perf_event *event)
+{
+        return event->attr.comm  || event->attr.mmap ||
+               event->attr.mmap2 || event->attr.mmap_data ||
+               event->attr.task;
+}
 static void perf_event_task_output(struct perf_event *event,
                                   void *data)
 {
@@ -4768,6 +4813,9 @@ static void perf_event_task_output(struct perf_event *event,
        struct task_struct *task = task_event->task;
        int ret, size = task_event->event_id.header.size;
+        if (!perf_event_task_match(event))
+                return;
        perf_event_header__init_id(&task_event->event_id.header, &sample, event);
        ret = perf_output_begin(&handle, event,
@@ -4790,13 +4838,6 @@ out:
        task_event->event_id.header.size = size;
 }
-static int perf_event_task_match(struct perf_event *event,
-                                 void *data __maybe_unused)
-{
-        return event->attr.comm || event->attr.mmap ||
-               event->attr.mmap_data || event->attr.task;
-}
 static void perf_event_task(struct task_struct *task,
                              struct perf_event_context *task_ctx,
                              int new)
@@ -4825,8 +4866,7 @@ static void perf_event_task(struct task_struct *task,
                },
        };
-        perf_event_aux(perf_event_task_match,
+        perf_event_aux(perf_event_task_output,
-                       perf_event_task_output,
                       &task_event,
                       task_ctx);
 }
@@ -4853,6 +4893,11 @@ struct perf_comm_event {
        } event_id;
 };
+static int perf_event_comm_match(struct perf_event *event)
+{
+        return event->attr.comm;
+}
 static void perf_event_comm_output(struct perf_event *event,
                                   void *data)
 {
@@ -4862,6 +4907,9 @@ static void perf_event_comm_output(struct perf_event *event,
        int size = comm_event->event_id.header.size;
        int ret;
+        if (!perf_event_comm_match(event))
+                return;
        perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
        ret = perf_output_begin(&handle, event,
                                comm_event->event_id.header.size);
@@ -4883,12 +4931,6 @@ out:
        comm_event->event_id.header.size = size;
 }
-static int perf_event_comm_match(struct perf_event *event,
-                                 void *data __maybe_unused)
-{
-        return event->attr.comm;
-}
 static void perf_event_comm_event(struct perf_comm_event *comm_event)
 {
        char comm[TASK_COMM_LEN];
@@ -4903,8 +4945,7 @@ static void perf_event_comm_event(struct perf_comm_event *comm_event)
        comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
-        perf_event_aux(perf_event_comm_match,
+        perf_event_aux(perf_event_comm_output,
-                       perf_event_comm_output,
                       comm_event,
                       NULL);
 }
@@ -4955,6 +4996,9 @@ struct perf_mmap_event {
        const char              *file_name;
        int                     file_size;
+        int                     maj, min;
+        u64                     ino;
+        u64                     ino_generation;
        struct {
                struct perf_event_header        header;
@@ -4967,6 +5011,17 @@ struct perf_mmap_event {
        } event_id;
 };
+static int perf_event_mmap_match(struct perf_event *event,
+                                 void *data)
+{
+        struct perf_mmap_event *mmap_event = data;
+        struct vm_area_struct *vma = mmap_event->vma;
+        int executable = vma->vm_flags & VM_EXEC;
+        return (!executable && event->attr.mmap_data) ||
+               (executable && (event->attr.mmap || event->attr.mmap2));
+}
 static void perf_event_mmap_output(struct perf_event *event,
                                   void *data)
 {
@@ -4976,6 +5031,16 @@ static void perf_event_mmap_output(struct perf_event *event,
        int size = mmap_event->event_id.header.size;
        int ret;
+        if (!perf_event_mmap_match(event, data))
+                return;
+        if (event->attr.mmap2) {
+                mmap_event->event_id.header.type = PERF_RECORD_MMAP2;
+                mmap_event->event_id.header.size += sizeof(mmap_event->maj);
+                mmap_event->event_id.header.size += sizeof(mmap_event->min);
+                mmap_event->event_id.header.size += sizeof(mmap_event->ino);
+        }
        perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
        ret = perf_output_begin(&handle, event,
                                mmap_event->event_id.header.size);
@@ -4986,6 +5051,14 @@ static void perf_event_mmap_output(struct perf_event *event,
        mmap_event->event_id.tid = perf_event_tid(event, current);
        perf_output_put(&handle, mmap_event->event_id);
+        if (event->attr.mmap2) {
+                perf_output_put(&handle, mmap_event->maj);
+                perf_output_put(&handle, mmap_event->min);
+                perf_output_put(&handle, mmap_event->ino);
+                perf_output_put(&handle, mmap_event->ino_generation);
+        }
        __output_copy(&handle, mmap_event->file_name,
                                   mmap_event->file_size);
@@ -4996,21 +5069,12 @@ out:
        mmap_event->event_id.header.size = size;
 }
-static int perf_event_mmap_match(struct perf_event *event,
-                                 void *data)
-{
-        struct perf_mmap_event *mmap_event = data;
-        struct vm_area_struct *vma = mmap_event->vma;
-        int executable = vma->vm_flags & VM_EXEC;
-        return (!executable && event->attr.mmap_data) ||
-               (executable && event->attr.mmap);
-}
 static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
 {
        struct vm_area_struct *vma = mmap_event->vma;
        struct file *file = vma->vm_file;
+        int maj = 0, min = 0;
+        u64 ino = 0, gen = 0;
        unsigned int size;
        char tmp[16];
        char *buf = NULL;
@@ -5019,6 +5083,8 @@ static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
        memset(tmp, 0, sizeof(tmp));
        if (file) {
+                struct inode *inode;
+                dev_t dev;
                /*
                 * d_path works from the end of the rb backwards, so we
                 * need to add enough zero bytes after the string to handle
@@ -5034,6 +5100,13 @@ static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
                        name = strncpy(tmp, "//toolong", sizeof(tmp));
                        goto got_name;
                }
+                inode = file_inode(vma->vm_file);
+                dev = inode->i_sb->s_dev;
+                ino = inode->i_ino;
+                gen = inode->i_generation;
+                maj = MAJOR(dev);
+                min = MINOR(dev);
        } else {
                if (arch_vma_name(mmap_event->vma)) {
                        name = strncpy(tmp, arch_vma_name(mmap_event->vma),
@@ -5064,14 +5137,17 @@ got_name:
        mmap_event->file_name = name;
        mmap_event->file_size = size;
+        mmap_event->maj = maj;
+        mmap_event->min = min;
+        mmap_event->ino = ino;
+        mmap_event->ino_generation = gen;
        if (!(vma->vm_flags & VM_EXEC))
                mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA;
        mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
-        perf_event_aux(perf_event_mmap_match,
+        perf_event_aux(perf_event_mmap_output,
-                       perf_event_mmap_output,
                       mmap_event,
                       NULL);
@@ -5101,6 +5177,10 @@ void perf_event_mmap(struct vm_area_struct *vma)
                        .len    = vma->vm_end - vma->vm_start,
                        .pgoff  = (u64)vma->vm_pgoff << PAGE_SHIFT,
                },
+                /* .maj (attr_mmap2 only) */
+                /* .min (attr_mmap2 only) */
+                /* .ino (attr_mmap2 only) */
+                /* .ino_generation (attr_mmap2 only) */
        };
        perf_event_mmap_event(&mmap_event);
@@ -5178,6 +5258,7 @@ static int __perf_event_overflow(struct perf_event *event,
                        __this_cpu_inc(perf_throttled_count);
                        hwc->interrupts = MAX_INTERRUPTS;
                        perf_log_throttle(event, 0);
+                        tick_nohz_full_kick();
                        ret = 1;
                }
        }
@@ -6234,8 +6315,6 @@ perf_event_mux_interval_ms_store(struct device *dev,
        return count;
 }
-#define __ATTR_RW(attr) __ATTR(attr, 0644, attr##_show, attr##_store)
 static struct device_attribute pmu_dev_attrs[] = {
        __ATTR_RO(type),
        __ATTR_RW(perf_event_mux_interval_ms),
@@ -6445,6 +6524,44 @@ unlock:
        return pmu;
 }
+static void account_event_cpu(struct perf_event *event, int cpu)
+{
+        if (event->parent)
+                return;
+        if (has_branch_stack(event)) {
+                if (!(event->attach_state & PERF_ATTACH_TASK))
+                        atomic_inc(&per_cpu(perf_branch_stack_events, cpu));
+        }
+        if (is_cgroup_event(event))
+                atomic_inc(&per_cpu(perf_cgroup_events, cpu));
+}
+static void account_event(struct perf_event *event)
+{
+        if (event->parent)
+                return;
+        if (event->attach_state & PERF_ATTACH_TASK)
+                static_key_slow_inc(&perf_sched_events.key);
+        if (event->attr.mmap || event->attr.mmap_data)
+                atomic_inc(&nr_mmap_events);
+        if (event->attr.comm)
+                atomic_inc(&nr_comm_events);
+        if (event->attr.task)
+                atomic_inc(&nr_task_events);
+        if (event->attr.freq) {
+                if (atomic_inc_return(&nr_freq_events) == 1)
+                        tick_nohz_full_kick_all();
+        }
+        if (has_branch_stack(event))
+                static_key_slow_inc(&perf_sched_events.key);
+        if (is_cgroup_event(event))
+                static_key_slow_inc(&perf_sched_events.key);
+        account_event_cpu(event, event->cpu);
+}
 /*
 * Allocate and initialize a event structure
 */
@@ -6459,7 +6576,7 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu,
        struct pmu *pmu;
        struct perf_event *event;
        struct hw_perf_event *hwc;
-        long err;
+        long err = -EINVAL;
        if ((unsigned)cpu >= nr_cpu_ids) {
                if (!task || cpu != -1)
@@ -6542,49 +6659,35 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu,
         * we currently do not support PERF_FORMAT_GROUP on inherited events
         */
        if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
-                goto done;
+                goto err_ns;
        pmu = perf_init_event(event);
-done:
-        err = 0;
        if (!pmu)
-                err = -EINVAL;
+                goto err_ns;
-        else if (IS_ERR(pmu))
+        else if (IS_ERR(pmu)) {
                err = PTR_ERR(pmu);
+                goto err_ns;
-        if (err) {
-                if (event->ns)
-                        put_pid_ns(event->ns);
-                kfree(event);
-                return ERR_PTR(err);
        }
        if (!event->parent) {
-                if (event->attach_state & PERF_ATTACH_TASK)
-                        static_key_slow_inc(&perf_sched_events.key);
-                if (event->attr.mmap || event->attr.mmap_data)
-                        atomic_inc(&nr_mmap_events);
-                if (event->attr.comm)
-                        atomic_inc(&nr_comm_events);
-                if (event->attr.task)
-                        atomic_inc(&nr_task_events);
                if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
                        err = get_callchain_buffers();
-                        if (err) {
+                        if (err)
-                                free_event(event);
+                                goto err_pmu;
-                                return ERR_PTR(err);
-                        }
-                }
-                if (has_branch_stack(event)) {
-                        static_key_slow_inc(&perf_sched_events.key);
-                        if (!(event->attach_state & PERF_ATTACH_TASK))
-                                atomic_inc(&per_cpu(perf_branch_stack_events,
-                                                    event->cpu));
                }
        }
        return event;
+err_pmu:
+        if (event->destroy)
+                event->destroy(event);
+err_ns:
+        if (event->ns)
+                put_pid_ns(event->ns);
+        kfree(event);
+        return ERR_PTR(err);
 }
 static int perf_copy_attr(struct perf_event_attr __user *uattr,
@@ -6866,17 +6969,14 @@ SYSCALL_DEFINE5(perf_event_open,
        if (flags & PERF_FLAG_PID_CGROUP) {
                err = perf_cgroup_connect(pid, event, &attr, group_leader);
-                if (err)
+                if (err) {
-                        goto err_alloc;
+                        __free_event(event);
-                /*
+                        goto err_task;
-                 * one more event:
+                }
-                 * - that has cgroup constraint on event->cpu
-                 * - that may need work on context switch
-                 */
-                atomic_inc(&per_cpu(perf_cgroup_events, event->cpu));
-                static_key_slow_inc(&perf_sched_events.key);
        }
+        account_event(event);
        /*
         * Special case software events and allow them to be part of
         * any hardware group.
@@ -7072,6 +7172,8 @@ perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
                goto err;
        }
+        account_event(event);
        ctx = find_get_context(event->pmu, task, cpu);
        if (IS_ERR(ctx)) {
                err = PTR_ERR(ctx);
@@ -7108,6 +7210,7 @@ void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu)
        list_for_each_entry_safe(event, tmp, &src_ctx->event_list,
                                 event_entry) {
                perf_remove_from_context(event);
+                unaccount_event_cpu(event, src_cpu);
                put_ctx(src_ctx);
                list_add(&event->event_entry, &events);
        }
@@ -7120,6 +7223,7 @@ void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu)
                list_del(&event->event_entry);
                if (event->state >= PERF_EVENT_STATE_OFF)
                        event->state = PERF_EVENT_STATE_INACTIVE;
+                account_event_cpu(event, dst_cpu);
                perf_install_in_context(dst_ctx, event, dst_cpu);
                get_ctx(dst_ctx);
        }
@@ -7630,7 +7734,7 @@ static void __init perf_event_init_all_cpus(void)
        }
 }
-static void __cpuinit perf_event_init_cpu(int cpu)
+static void perf_event_init_cpu(int cpu)
 {
        struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
@@ -7719,7 +7823,7 @@ static struct notifier_block perf_reboot_notifier = {
        .priority = INT_MIN,
 };
-static int __cpuinit
+static int
 perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
 {
        unsigned int cpu = (long)hcpu;
@@ -7800,7 +7904,8 @@ unlock:
 device_initcall(perf_event_sysfs_init);
 #ifdef CONFIG_CGROUP_PERF
-static struct cgroup_subsys_state *perf_cgroup_css_alloc(struct cgroup *cont)
+static struct cgroup_subsys_state *
+perf_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
 {
        struct perf_cgroup *jc;
@@ -7817,11 +7922,10 @@ static struct cgroup_subsys_state *perf_cgroup_css_alloc(struct cgroup *cont)
        return &jc->css;
 }
-static void perf_cgroup_css_free(struct cgroup *cont)
+static void perf_cgroup_css_free(struct cgroup_subsys_state *css)
 {
-        struct perf_cgroup *jc;
+        struct perf_cgroup *jc = container_of(css, struct perf_cgroup, css);
-        jc = container_of(cgroup_subsys_state(cont, perf_subsys_id),
-                          struct perf_cgroup, css);
        free_percpu(jc->info);
        kfree(jc);
 }
@@ -7833,15 +7937,17 @@ static int __perf_cgroup_move(void *info)
        return 0;
 }
-static void perf_cgroup_attach(struct cgroup *cgrp, struct cgroup_taskset *tset)
+static void perf_cgroup_attach(struct cgroup_subsys_state *css,
+                               struct cgroup_taskset *tset)
 {
        struct task_struct *task;
-        cgroup_taskset_for_each(task, cgrp, tset)
+        cgroup_taskset_for_each(task, css, tset)
                task_function_call(task, __perf_cgroup_move, task);
 }
-static void perf_cgroup_exit(struct cgroup *cgrp, struct cgroup *old_cgrp,
+static void perf_cgroup_exit(struct cgroup_subsys_state *css,
+                             struct cgroup_subsys_state *old_css,
                             struct task_struct *task)
 {
        /*
diff --git a/kernel/fork.c b/kernel/fork.c
index 66635c80a813..bf46287c91a4 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -1177,7 +1177,8 @@ static struct task_struct *copy_process(unsigned long clone_flags,
         * don't allow the creation of threads.
         */
        if ((clone_flags & (CLONE_VM|CLONE_NEWPID)) &&
-            (task_active_pid_ns(current) != current->nsproxy->pid_ns))
+            (task_active_pid_ns(current) !=
+             current->nsproxy->pid_ns_for_children))
                return ERR_PTR(-EINVAL);
        retval = security_task_create(clone_flags);
@@ -1351,7 +1352,7 @@ static struct task_struct *copy_process(unsigned long clone_flags,
        if (pid != &init_struct_pid) {
                retval = -ENOMEM;
-                pid = alloc_pid(p->nsproxy->pid_ns);
+                pid = alloc_pid(p->nsproxy->pid_ns_for_children);
                if (!pid)
                        goto bad_fork_cleanup_io;
        }
@@ -1546,7 +1547,7 @@ static inline void init_idle_pids(struct pid_link *links)
        }
 }
-struct task_struct * __cpuinit fork_idle(int cpu)
+struct task_struct *fork_idle(int cpu)
 {
        struct task_struct *task;
        task = copy_process(CLONE_VM, 0, 0, NULL, &init_struct_pid, 0);
@@ -1679,6 +1680,12 @@ SYSCALL_DEFINE5(clone, unsigned long, newsp, unsigned long, clone_flags,
                 int __user *, parent_tidptr,
                 int __user *, child_tidptr,
                 int, tls_val)
+#elif defined(CONFIG_CLONE_BACKWARDS3)
+SYSCALL_DEFINE6(clone, unsigned long, clone_flags, unsigned long, newsp,
+                int, stack_size,
+                int __user *, parent_tidptr,
+                int __user *, child_tidptr,
+                int, tls_val)
 #else
 SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
                 int __user *, parent_tidptr,
diff --git a/kernel/freezer.c b/kernel/freezer.c
index 8b2afc1c9df0..b462fa197517 100644
--- a/kernel/freezer.c
+++ b/kernel/freezer.c
@@ -33,7 +33,7 @@ static DEFINE_SPINLOCK(freezer_lock);
 */
 bool freezing_slow_path(struct task_struct *p)
 {
-        if (p->flags & PF_NOFREEZE)
+        if (p->flags & (PF_NOFREEZE | PF_SUSPEND_TASK))
                return false;
        if (pm_nosig_freezing || cgroup_freezing(p))
diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c
index f0f4fe29cd21..383319bae3f7 100644
--- a/kernel/hrtimer.c
+++ b/kernel/hrtimer.c
@@ -1659,7 +1659,7 @@ SYSCALL_DEFINE2(nanosleep, struct timespec __user *, rqtp,
 /*
 * Functions related to boot-time initialization:
 */
-static void __cpuinit init_hrtimers_cpu(int cpu)
+static void init_hrtimers_cpu(int cpu)
 {
        struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);
        int i;
@@ -1740,7 +1740,7 @@ static void migrate_hrtimers(int scpu)
 #endif /* CONFIG_HOTPLUG_CPU */
-static int __cpuinit hrtimer_cpu_notify(struct notifier_block *self,
+static int hrtimer_cpu_notify(struct notifier_block *self,
                                        unsigned long action, void *hcpu)
 {
        int scpu = (long)hcpu;
@@ -1773,7 +1773,7 @@ static int __cpuinit hrtimer_cpu_notify(struct notifier_block *self,
        return NOTIFY_OK;
 }
-static struct notifier_block __cpuinitdata hrtimers_nb = {
+static struct notifier_block hrtimers_nb = {
        .notifier_call = hrtimer_cpu_notify,
 };
diff --git a/kernel/hung_task.c b/kernel/hung_task.c
index 6df614912b9d..3e97fb126e6b 100644
--- a/kernel/hung_task.c
+++ b/kernel/hung_task.c
@@ -15,6 +15,7 @@
 #include <linux/lockdep.h>
 #include <linux/export.h>
 #include <linux/sysctl.h>
+#include <linux/utsname.h>
 /*
 * The number of tasks checked:
@@ -99,10 +100,14 @@ static void check_hung_task(struct task_struct *t, unsigned long timeout)
         * Ok, the task did not get scheduled for more than 2 minutes,
         * complain:
         */
-        printk(KERN_ERR "INFO: task %s:%d blocked for more than "
+        pr_err("INFO: task %s:%d blocked for more than %ld seconds.\n",
-                        "%ld seconds.\n", t->comm, t->pid, timeout);
+                t->comm, t->pid, timeout);
-        printk(KERN_ERR "\"echo 0 > /proc/sys/kernel/hung_task_timeout_secs\""
+        pr_err("      %s %s %.*s\n",
-                        " disables this message.\n");
+                print_tainted(), init_utsname()->release,
+                (int)strcspn(init_utsname()->version, " "),
+                init_utsname()->version);
+        pr_err("\"echo 0 > /proc/sys/kernel/hung_task_timeout_secs\""
+                " disables this message.\n");
        sched_show_task(t);
        debug_show_held_locks(t);
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 60f48fa0fd0d..297a9247a3b3 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -13,6 +13,7 @@
 #include <linux/sort.h>
 #include <linux/err.h>
 #include <linux/static_key.h>
+#include <linux/jump_label_ratelimit.h>
 #ifdef HAVE_JUMP_LABEL
diff --git a/kernel/lglock.c b/kernel/lglock.c
index 6535a667a5a7..86ae2aebf004 100644
--- a/kernel/lglock.c
+++ b/kernel/lglock.c
@@ -21,7 +21,7 @@ void lg_local_lock(struct lglock *lg)
        arch_spinlock_t *lock;
        preempt_disable();
-        rwlock_acquire_read(&lg->lock_dep_map, 0, 0, _RET_IP_);
+        lock_acquire_shared(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_);
        lock = this_cpu_ptr(lg->lock);
        arch_spin_lock(lock);
 }
@@ -31,7 +31,7 @@ void lg_local_unlock(struct lglock *lg)
 {
        arch_spinlock_t *lock;
-        rwlock_release(&lg->lock_dep_map, 1, _RET_IP_);
+        lock_release(&lg->lock_dep_map, 1, _RET_IP_);
        lock = this_cpu_ptr(lg->lock);
        arch_spin_unlock(lock);
        preempt_enable();
@@ -43,7 +43,7 @@ void lg_local_lock_cpu(struct lglock *lg, int cpu)
        arch_spinlock_t *lock;
        preempt_disable();
-        rwlock_acquire_read(&lg->lock_dep_map, 0, 0, _RET_IP_);
+        lock_acquire_shared(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_);
        lock = per_cpu_ptr(lg->lock, cpu);
        arch_spin_lock(lock);
 }
@@ -53,7 +53,7 @@ void lg_local_unlock_cpu(struct lglock *lg, int cpu)
 {
        arch_spinlock_t *lock;
-        rwlock_release(&lg->lock_dep_map, 1, _RET_IP_);
+        lock_release(&lg->lock_dep_map, 1, _RET_IP_);
        lock = per_cpu_ptr(lg->lock, cpu);
        arch_spin_unlock(lock);
        preempt_enable();
@@ -65,7 +65,7 @@ void lg_global_lock(struct lglock *lg)
        int i;
        preempt_disable();
-        rwlock_acquire(&lg->lock_dep_map, 0, 0, _RET_IP_);
+        lock_acquire_exclusive(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_);
        for_each_possible_cpu(i) {
                arch_spinlock_t *lock;
                lock = per_cpu_ptr(lg->lock, i);
@@ -78,7 +78,7 @@ void lg_global_unlock(struct lglock *lg)
 {
        int i;
-        rwlock_release(&lg->lock_dep_map, 1, _RET_IP_);
+        lock_release(&lg->lock_dep_map, 1, _RET_IP_);
        for_each_possible_cpu(i) {
                arch_spinlock_t *lock;
                lock = per_cpu_ptr(lg->lock, i);
diff --git a/kernel/mutex.c b/kernel/mutex.c
index ff05f4bd86eb..6d647aedffea 100644
--- a/kernel/mutex.c
+++ b/kernel/mutex.c
@@ -209,11 +209,13 @@ int mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner)
 */
 static inline int mutex_can_spin_on_owner(struct mutex *lock)
 {
+        struct task_struct *owner;
        int retval = 1;
        rcu_read_lock();
-        if (lock->owner)
+        owner = ACCESS_ONCE(lock->owner);
-                retval = lock->owner->on_cpu;
+        if (owner)
+                retval = owner->on_cpu;
        rcu_read_unlock();
        /*
         * if lock->owner is not set, the mutex owner may have just acquired
@@ -461,7 +463,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
                         * performed the optimistic spinning cannot be done.
                         */
                        if (ACCESS_ONCE(ww->ctx))
-                                break;
+                                goto slowpath;
                }
                /*
@@ -472,7 +474,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
                owner = ACCESS_ONCE(lock->owner);
                if (owner && !mutex_spin_on_owner(lock, owner)) {
                        mspin_unlock(MLOCK(lock), &node);
-                        break;
+                        goto slowpath;
                }
                if ((atomic_read(&lock->count) == 1) &&
@@ -499,7 +501,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
                 * the owner complete.
                 */
                if (!owner && (need_resched() || rt_task(task)))
-                        break;
+                        goto slowpath;
                /*
                 * The cpu_relax() call is a compiler barrier which forces
@@ -513,6 +515,10 @@ slowpath:
 #endif
        spin_lock_mutex(&lock->wait_lock, flags);
+        /* once more, can we acquire the lock? */
+        if (MUTEX_SHOW_NO_WAITER(lock) && (atomic_xchg(&lock->count, 0) == 1))
+                goto skip_wait;
        debug_mutex_lock_common(lock, &waiter);
        debug_mutex_add_waiter(lock, &waiter, task_thread_info(task));
@@ -520,9 +526,6 @@ slowpath:
        list_add_tail(&waiter.list, &lock->wait_list);
        waiter.task = task;
-        if (MUTEX_SHOW_NO_WAITER(lock) && (atomic_xchg(&lock->count, -1) == 1))
-                goto done;
        lock_contended(&lock->dep_map, ip);
        for (;;) {
@@ -536,7 +539,7 @@ slowpath:
                 * other waiters:
                 */
                if (MUTEX_SHOW_NO_WAITER(lock) &&
-                   (atomic_xchg(&lock->count, -1) == 1))
+                    (atomic_xchg(&lock->count, -1) == 1))
                        break;
                /*
@@ -561,24 +564,25 @@ slowpath:
                schedule_preempt_disabled();
                spin_lock_mutex(&lock->wait_lock, flags);
        }
+        mutex_remove_waiter(lock, &waiter, current_thread_info());
+        /* set it to 0 if there are no waiters left: */
+        if (likely(list_empty(&lock->wait_list)))
+                atomic_set(&lock->count, 0);
+        debug_mutex_free_waiter(&waiter);
-done:
+skip_wait:
+        /* got the lock - cleanup and rejoice! */
        lock_acquired(&lock->dep_map, ip);
-        /* got the lock - rejoice! */
-        mutex_remove_waiter(lock, &waiter, current_thread_info());
        mutex_set_owner(lock);
        if (!__builtin_constant_p(ww_ctx == NULL)) {
-                struct ww_mutex *ww = container_of(lock,
+                struct ww_mutex *ww = container_of(lock, struct ww_mutex, base);
-                                                      struct ww_mutex,
-                                                      base);
                struct mutex_waiter *cur;
                /*
                 * This branch gets optimized out for the common case,
                 * and is only important for ww_mutex_lock.
                 */
                ww_mutex_lock_acquired(ww, ww_ctx);
                ww->ctx = ww_ctx;
@@ -592,15 +596,8 @@ done:
                }
        }
-        /* set it to 0 if there are no waiters left: */
-        if (likely(list_empty(&lock->wait_list)))
-                atomic_set(&lock->count, 0);
        spin_unlock_mutex(&lock->wait_lock, flags);
-        debug_mutex_free_waiter(&waiter);
        preempt_enable();
        return 0;
 err:
@@ -686,7 +683,7 @@ __ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
        might_sleep();
        ret =  __mutex_lock_common(&lock->base, TASK_UNINTERRUPTIBLE,
                                   0, &ctx->dep_map, _RET_IP_, ctx);
-        if (!ret && ctx->acquired > 0)
+        if (!ret && ctx->acquired > 1)
                return ww_mutex_deadlock_injection(lock, ctx);
        return ret;
@@ -702,7 +699,7 @@ __ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
        ret = __mutex_lock_common(&lock->base, TASK_INTERRUPTIBLE,
                                  0, &ctx->dep_map, _RET_IP_, ctx);
-        if (!ret && ctx->acquired > 0)
+        if (!ret && ctx->acquired > 1)
                return ww_mutex_deadlock_injection(lock, ctx);
        return ret;
diff --git a/kernel/nsproxy.c b/kernel/nsproxy.c
index 364ceab15f0c..997cbb951a3b 100644
--- a/kernel/nsproxy.c
+++ b/kernel/nsproxy.c
@@ -29,15 +29,15 @@
 static struct kmem_cache *nsproxy_cachep;
 struct nsproxy init_nsproxy = {
-        .count  = ATOMIC_INIT(1),
+        .count                  = ATOMIC_INIT(1),
-        .uts_ns = &init_uts_ns,
+        .uts_ns                 = &init_uts_ns,
 #if defined(CONFIG_POSIX_MQUEUE) || defined(CONFIG_SYSVIPC)
-        .ipc_ns = &init_ipc_ns,
+        .ipc_ns                 = &init_ipc_ns,
 #endif
-        .mnt_ns = NULL,
+        .mnt_ns                 = NULL,
-        .pid_ns = &init_pid_ns,
+        .pid_ns_for_children    = &init_pid_ns,
 #ifdef CONFIG_NET
-        .net_ns = &init_net,
+        .net_ns                 = &init_net,
 #endif
 };
@@ -85,9 +85,10 @@ static struct nsproxy *create_new_namespaces(unsigned long flags,
                goto out_ipc;
        }
-        new_nsp->pid_ns = copy_pid_ns(flags, user_ns, tsk->nsproxy->pid_ns);
+        new_nsp->pid_ns_for_children =
-        if (IS_ERR(new_nsp->pid_ns)) {
+                copy_pid_ns(flags, user_ns, tsk->nsproxy->pid_ns_for_children);
-                err = PTR_ERR(new_nsp->pid_ns);
+        if (IS_ERR(new_nsp->pid_ns_for_children)) {
+                err = PTR_ERR(new_nsp->pid_ns_for_children);
                goto out_pid;
        }
@@ -100,8 +101,8 @@ static struct nsproxy *create_new_namespaces(unsigned long flags,
        return new_nsp;
 out_net:
-        if (new_nsp->pid_ns)
+        if (new_nsp->pid_ns_for_children)
-                put_pid_ns(new_nsp->pid_ns);
+                put_pid_ns(new_nsp->pid_ns_for_children);
 out_pid:
        if (new_nsp->ipc_ns)
                put_ipc_ns(new_nsp->ipc_ns);
@@ -174,8 +175,8 @@ void free_nsproxy(struct nsproxy *ns)
                put_uts_ns(ns->uts_ns);
        if (ns->ipc_ns)
                put_ipc_ns(ns->ipc_ns);
-        if (ns->pid_ns)
+        if (ns->pid_ns_for_children)
-                put_pid_ns(ns->pid_ns);
+                put_pid_ns(ns->pid_ns_for_children);
        put_net(ns->net_ns);
        kmem_cache_free(nsproxy_cachep, ns);
 }
diff --git a/kernel/pid_namespace.c b/kernel/pid_namespace.c
index 6917e8edb48e..601bb361c235 100644
--- a/kernel/pid_namespace.c
+++ b/kernel/pid_namespace.c
@@ -349,8 +349,8 @@ static int pidns_install(struct nsproxy *nsproxy, void *ns)
        if (ancestor != active)
                return -EINVAL;
-        put_pid_ns(nsproxy->pid_ns);
+        put_pid_ns(nsproxy->pid_ns_for_children);
-        nsproxy->pid_ns = get_pid_ns(new);
+        nsproxy->pid_ns_for_children = get_pid_ns(new);
        return 0;
 }
diff --git a/kernel/power/autosleep.c b/kernel/power/autosleep.c
index c6422ffeda9a..9012ecf7b814 100644
--- a/kernel/power/autosleep.c
+++ b/kernel/power/autosleep.c
@@ -32,7 +32,8 @@ static void try_to_suspend(struct work_struct *work)
        mutex_lock(&autosleep_lock);
-        if (!pm_save_wakeup_count(initial_count)) {
+        if (!pm_save_wakeup_count(initial_count) ||
+                system_state != SYSTEM_RUNNING) {
                mutex_unlock(&autosleep_lock);
                goto out;
        }
diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c
index b26f5f1e773e..3085e62a80a5 100644
--- a/kernel/power/hibernate.c
+++ b/kernel/power/hibernate.c
@@ -39,7 +39,7 @@ static int resume_delay;
 static char resume_file[256] = CONFIG_PM_STD_PARTITION;
 dev_t swsusp_resume_device;
 sector_t swsusp_resume_block;
-int in_suspend __nosavedata;
+__visible int in_suspend __nosavedata;
 enum {
        HIBERNATION_INVALID,
diff --git a/kernel/power/process.c b/kernel/power/process.c
index fc0df8486449..06ec8869dbf1 100644
--- a/kernel/power/process.c
+++ b/kernel/power/process.c
@@ -109,6 +109,8 @@ static int try_to_freeze_tasks(bool user_only)
 /**
 * freeze_processes - Signal user space processes to enter the refrigerator.
+ * The current thread will not be frozen.  The same process that calls
+ * freeze_processes must later call thaw_processes.
 *
 * On success, returns 0.  On failure, -errno and system is fully thawed.
 */
@@ -120,6 +122,9 @@ int freeze_processes(void)
        if (error)
                return error;
+        /* Make sure this task doesn't get frozen */
+        current->flags |= PF_SUSPEND_TASK;
        if (!pm_freezing)
                atomic_inc(&system_freezing_cnt);
@@ -168,6 +173,7 @@ int freeze_kernel_threads(void)
 void thaw_processes(void)
 {
        struct task_struct *g, *p;
+        struct task_struct *curr = current;
        if (pm_freezing)
                atomic_dec(&system_freezing_cnt);
@@ -182,10 +188,15 @@ void thaw_processes(void)
        read_lock(&tasklist_lock);
        do_each_thread(g, p) {
+                /* No other threads should have PF_SUSPEND_TASK set */
+                WARN_ON((p != curr) && (p->flags & PF_SUSPEND_TASK));
                __thaw_task(p);
        } while_each_thread(g, p);
        read_unlock(&tasklist_lock);
+        WARN_ON(!(curr->flags & PF_SUSPEND_TASK));
+        curr->flags &= ~PF_SUSPEND_TASK;
        usermodehelper_enable();
        schedule();
diff --git a/kernel/power/qos.c b/kernel/power/qos.c
index 06fe28589e9c..a394297f8b2f 100644
--- a/kernel/power/qos.c
+++ b/kernel/power/qos.c
@@ -296,6 +296,17 @@ int pm_qos_request_active(struct pm_qos_request *req)
 }
 EXPORT_SYMBOL_GPL(pm_qos_request_active);
+static void __pm_qos_update_request(struct pm_qos_request *req,
+                           s32 new_value)
+{
+        trace_pm_qos_update_request(req->pm_qos_class, new_value);
+        if (new_value != req->node.prio)
+                pm_qos_update_target(
+                        pm_qos_array[req->pm_qos_class]->constraints,
+                        &req->node, PM_QOS_UPDATE_REQ, new_value);
+}
 /**
 * pm_qos_work_fn - the timeout handler of pm_qos_update_request_timeout
 * @work: work struct for the delayed work (timeout)
@@ -308,7 +319,7 @@ static void pm_qos_work_fn(struct work_struct *work)
                                                  struct pm_qos_request,
                                                  work);
-        pm_qos_update_request(req, PM_QOS_DEFAULT_VALUE);
+        __pm_qos_update_request(req, PM_QOS_DEFAULT_VALUE);
 }
 /**
@@ -364,12 +375,7 @@ void pm_qos_update_request(struct pm_qos_request *req,
        }
        cancel_delayed_work_sync(&req->work);
+        __pm_qos_update_request(req, new_value);
-        trace_pm_qos_update_request(req->pm_qos_class, new_value);
-        if (new_value != req->node.prio)
-                pm_qos_update_target(
-                        pm_qos_array[req->pm_qos_class]->constraints,
-                        &req->node, PM_QOS_UPDATE_REQ, new_value);
 }
 EXPORT_SYMBOL_GPL(pm_qos_update_request);
diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c
index ece04223bb1e..62ee437b5c7e 100644
--- a/kernel/power/suspend.c
+++ b/kernel/power/suspend.c
@@ -210,6 +210,7 @@ static int suspend_enter(suspend_state_t state, bool *wakeup)
                goto Platform_wake;
        }
+        ftrace_stop();
        error = disable_nonboot_cpus();
        if (error || suspend_test(TEST_CPUS))
                goto Enable_cpus;
@@ -232,6 +233,7 @@ static int suspend_enter(suspend_state_t state, bool *wakeup)
 Enable_cpus:
        enable_nonboot_cpus();
+        ftrace_start();
 Platform_wake:
        if (need_suspend_ops(state) && suspend_ops->wake)
@@ -265,7 +267,6 @@ int suspend_devices_and_enter(suspend_state_t state)
                        goto Close;
        }
        suspend_console();
-        ftrace_stop();
        suspend_test_start();
        error = dpm_suspend_start(PMSG_SUSPEND);
        if (error) {
@@ -285,7 +286,6 @@ int suspend_devices_and_enter(suspend_state_t state)
        suspend_test_start();
        dpm_resume_end(PMSG_RESUME);
        suspend_test_finish("resume devices");
-        ftrace_start();
        resume_console();
 Close:
        if (need_suspend_ops(state) && suspend_ops->end)
diff --git a/kernel/printk/Makefile b/kernel/printk/Makefile
new file mode 100644
index 000000000000..85405bdcf2b3
--- /dev/null
+++ b/kernel/printk/Makefile
@@ -0,0 +1,2 @@
+obj-y   = printk.o
+obj-$(CONFIG_A11Y_BRAILLE_CONSOLE)      += braille.o
diff --git a/kernel/printk/braille.c b/kernel/printk/braille.c
new file mode 100644
index 000000000000..276762f3a460
--- /dev/null
+++ b/kernel/printk/braille.c
@@ -0,0 +1,49 @@
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#include <linux/kernel.h>
+#include <linux/console.h>
+#include <linux/string.h>
+#include "console_cmdline.h"
+#include "braille.h"
+char *_braille_console_setup(char **str, char **brl_options)
+{
+        if (!memcmp(*str, "brl,", 4)) {
+                *brl_options = "";
+                *str += 4;
+        } else if (!memcmp(str, "brl=", 4)) {
+                *brl_options = *str + 4;
+                *str = strchr(*brl_options, ',');
+                if (!*str)
+                        pr_err("need port name after brl=\n");
+                else
+                        *((*str)++) = 0;
+        } else
+                return NULL;
+        return *str;
+}
+int
+_braille_register_console(struct console *console, struct console_cmdline *c)
+{
+        int rtn = 0;
+        if (c->brl_options) {
+                console->flags |= CON_BRL;
+                rtn = braille_register_console(console, c->index, c->options,
+                                               c->brl_options);
+        }
+        return rtn;
+}
+int
+_braille_unregister_console(struct console *console)
+{
+        if (console->flags & CON_BRL)
+                return braille_unregister_console(console);
+        return 0;
+}
diff --git a/kernel/printk/braille.h b/kernel/printk/braille.h
new file mode 100644
index 000000000000..769d771145c8
--- /dev/null
+++ b/kernel/printk/braille.h
@@ -0,0 +1,48 @@
+#ifndef _PRINTK_BRAILLE_H
+#define _PRINTK_BRAILLE_H
+#ifdef CONFIG_A11Y_BRAILLE_CONSOLE
+static inline void
+braille_set_options(struct console_cmdline *c, char *brl_options)
+{
+        c->brl_options = brl_options;
+}
+char *
+_braille_console_setup(char **str, char **brl_options);
+int
+_braille_register_console(struct console *console, struct console_cmdline *c);
+int
+_braille_unregister_console(struct console *console);
+#else
+static inline void
+braille_set_options(struct console_cmdline *c, char *brl_options)
+{
+}
+static inline char *
+_braille_console_setup(char **str, char **brl_options)
+{
+        return NULL;
+}
+static inline int
+_braille_register_console(struct console *console, struct console_cmdline *c)
+{
+        return 0;
+}
+static inline int
+_braille_unregister_console(struct console *console)
+{
+        return 0;
+}
+#endif
+#endif
diff --git a/kernel/printk/console_cmdline.h b/kernel/printk/console_cmdline.h
new file mode 100644
index 000000000000..cbd69d842341
--- /dev/null
+++ b/kernel/printk/console_cmdline.h
@@ -0,0 +1,14 @@
+#ifndef _CONSOLE_CMDLINE_H
+#define _CONSOLE_CMDLINE_H
+struct console_cmdline
+{
+        char    name[8];                        /* Name of the driver       */
+        int     index;                          /* Minor dev. to use        */
+        char    *options;                       /* Options for the driver   */
+#ifdef CONFIG_A11Y_BRAILLE_CONSOLE
+        char    *brl_options;                   /* Options for braille driver */
+#endif
+};
+#endif
diff --git a/kernel/printk.c b/kernel/printk/printk.c
index d37d45c90ae6..b4e8500afdb3 100644
--- a/kernel/printk.c
+++ b/kernel/printk/printk.c
@@ -51,6 +51,9 @@
 #define CREATE_TRACE_POINTS
 #include <trace/events/printk.h>
+#include "console_cmdline.h"
+#include "braille.h"
 /* printk's without a loglevel use this.. */
 #define DEFAULT_MESSAGE_LOGLEVEL CONFIG_DEFAULT_MESSAGE_LOGLEVEL
@@ -105,19 +108,11 @@ static struct console *exclusive_console;
 /*
 *      Array of consoles built from command line options (console=)
 */
-struct console_cmdline
-{
-        char    name[8];                        /* Name of the driver       */
-        int     index;                          /* Minor dev. to use        */
-        char    *options;                       /* Options for the driver   */
-#ifdef CONFIG_A11Y_BRAILLE_CONSOLE
-        char    *brl_options;                   /* Options for braille driver */
-#endif
-};
 #define MAX_CMDLINECONSOLES 8
 static struct console_cmdline console_cmdline[MAX_CMDLINECONSOLES];
 static int selected_console = -1;
 static int preferred_console = -1;
 int console_set_on_cmdline;
@@ -178,7 +173,7 @@ static int console_may_schedule;
 *         67                           "g"
 *   0032     00 00 00                  padding to next message header
 *
- * The 'struct log' buffer header must never be directly exported to
+ * The 'struct printk_log' buffer header must never be directly exported to
 * userspace, it is a kernel-private implementation detail that might
 * need to be changed in the future, when the requirements change.
 *
@@ -200,7 +195,7 @@ enum log_flags {
        LOG_CONT        = 8,    /* text is a fragment of a continuation line */
 };
-struct log {
+struct printk_log {
        u64 ts_nsec;            /* timestamp in nanoseconds */
        u16 len;                /* length of entire record */
        u16 text_len;           /* length of text buffer */
@@ -248,7 +243,7 @@ static u32 clear_idx;
 #if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)
 #define LOG_ALIGN 4
 #else
-#define LOG_ALIGN __alignof__(struct log)
+#define LOG_ALIGN __alignof__(struct printk_log)
 #endif
 #define __LOG_BUF_LEN (1 << CONFIG_LOG_BUF_SHIFT)
 static char __log_buf[__LOG_BUF_LEN] __aligned(LOG_ALIGN);
@@ -259,35 +254,35 @@ static u32 log_buf_len = __LOG_BUF_LEN;
 static volatile unsigned int logbuf_cpu = UINT_MAX;
 /* human readable text of the record */
-static char *log_text(const struct log *msg)
+static char *log_text(const struct printk_log *msg)
 {
-        return (char *)msg + sizeof(struct log);
+        return (char *)msg + sizeof(struct printk_log);
 }
 /* optional key/value pair dictionary attached to the record */
-static char *log_dict(const struct log *msg)
+static char *log_dict(const struct printk_log *msg)
 {
-        return (char *)msg + sizeof(struct log) + msg->text_len;
+        return (char *)msg + sizeof(struct printk_log) + msg->text_len;
 }
 /* get record by index; idx must point to valid msg */
-static struct log *log_from_idx(u32 idx)
+static struct printk_log *log_from_idx(u32 idx)
 {
-        struct log *msg = (struct log *)(log_buf + idx);
+        struct printk_log *msg = (struct printk_log *)(log_buf + idx);
        /*
         * A length == 0 record is the end of buffer marker. Wrap around and
         * read the message at the start of the buffer.
         */
        if (!msg->len)
-                return (struct log *)log_buf;
+                return (struct printk_log *)log_buf;
        return msg;
 }
 /* get next record; idx must point to valid msg */
 static u32 log_next(u32 idx)
 {
-        struct log *msg = (struct log *)(log_buf + idx);
+        struct printk_log *msg = (struct printk_log *)(log_buf + idx);
        /* length == 0 indicates the end of the buffer; wrap */
        /*
@@ -296,7 +291,7 @@ static u32 log_next(u32 idx)
         * return the one after that.
         */
        if (!msg->len) {
-                msg = (struct log *)log_buf;
+                msg = (struct printk_log *)log_buf;
                return msg->len;
        }
        return idx + msg->len;
@@ -308,11 +303,11 @@ static void log_store(int facility, int level,
                      const char *dict, u16 dict_len,
                      const char *text, u16 text_len)
 {
-        struct log *msg;
+        struct printk_log *msg;
        u32 size, pad_len;
        /* number of '\0' padding bytes to next message */
-        size = sizeof(struct log) + text_len + dict_len;
+        size = sizeof(struct printk_log) + text_len + dict_len;
        pad_len = (-size) & (LOG_ALIGN - 1);
        size += pad_len;
@@ -324,7 +319,7 @@ static void log_store(int facility, int level,
                else
                        free = log_first_idx - log_next_idx;
-                if (free > size + sizeof(struct log))
+                if (free > size + sizeof(struct printk_log))
                        break;
                /* drop old messages until we have enough contiuous space */
@@ -332,18 +327,18 @@ static void log_store(int facility, int level,
                log_first_seq++;
        }
-        if (log_next_idx + size + sizeof(struct log) >= log_buf_len) {
+        if (log_next_idx + size + sizeof(struct printk_log) >= log_buf_len) {
                /*
                 * This message + an additional empty header does not fit
                 * at the end of the buffer. Add an empty header with len == 0
                 * to signify a wrap around.
                 */
-                memset(log_buf + log_next_idx, 0, sizeof(struct log));
+                memset(log_buf + log_next_idx, 0, sizeof(struct printk_log));
                log_next_idx = 0;
        }
        /* fill message */
-        msg = (struct log *)(log_buf + log_next_idx);
+        msg = (struct printk_log *)(log_buf + log_next_idx);
        memcpy(log_text(msg), text, text_len);
        msg->text_len = text_len;
        memcpy(log_dict(msg), dict, dict_len);
@@ -356,7 +351,7 @@ static void log_store(int facility, int level,
        else
                msg->ts_nsec = local_clock();
        memset(log_dict(msg) + dict_len, 0, pad_len);
-        msg->len = sizeof(struct log) + text_len + dict_len + pad_len;
+        msg->len = sizeof(struct printk_log) + text_len + dict_len + pad_len;
        /* insert message */
        log_next_idx += msg->len;
@@ -479,7 +474,7 @@ static ssize_t devkmsg_read(struct file *file, char __user *buf,
                            size_t count, loff_t *ppos)
 {
        struct devkmsg_user *user = file->private_data;
-        struct log *msg;
+        struct printk_log *msg;
        u64 ts_usec;
        size_t i;
        char cont = '-';
@@ -724,14 +719,14 @@ void log_buf_kexec_setup(void)
        VMCOREINFO_SYMBOL(log_first_idx);
        VMCOREINFO_SYMBOL(log_next_idx);
        /*
-         * Export struct log size and field offsets. User space tools can
+         * Export struct printk_log size and field offsets. User space tools can
         * parse it and detect any changes to structure down the line.
         */
-        VMCOREINFO_STRUCT_SIZE(log);
+        VMCOREINFO_STRUCT_SIZE(printk_log);
-        VMCOREINFO_OFFSET(log, ts_nsec);
+        VMCOREINFO_OFFSET(printk_log, ts_nsec);
-        VMCOREINFO_OFFSET(log, len);
+        VMCOREINFO_OFFSET(printk_log, len);
-        VMCOREINFO_OFFSET(log, text_len);
+        VMCOREINFO_OFFSET(printk_log, text_len);
-        VMCOREINFO_OFFSET(log, dict_len);
+        VMCOREINFO_OFFSET(printk_log, dict_len);
 }
 #endif
@@ -884,7 +879,7 @@ static size_t print_time(u64 ts, char *buf)
                       (unsigned long)ts, rem_nsec / 1000);
 }
-static size_t print_prefix(const struct log *msg, bool syslog, char *buf)
+static size_t print_prefix(const struct printk_log *msg, bool syslog, char *buf)
 {
        size_t len = 0;
        unsigned int prefix = (msg->facility << 3) | msg->level;
@@ -907,7 +902,7 @@ static size_t print_prefix(const struct log *msg, bool syslog, char *buf)
        return len;
 }
-static size_t msg_print_text(const struct log *msg, enum log_flags prev,
+static size_t msg_print_text(const struct printk_log *msg, enum log_flags prev,
                             bool syslog, char *buf, size_t size)
 {
        const char *text = log_text(msg);
@@ -969,7 +964,7 @@ static size_t msg_print_text(const struct log *msg, enum log_flags prev,
 static int syslog_print(char __user *buf, int size)
 {
        char *text;
-        struct log *msg;
+        struct printk_log *msg;
        int len = 0;
        text = kmalloc(LOG_LINE_MAX + PREFIX_MAX, GFP_KERNEL);
@@ -1060,7 +1055,7 @@ static int syslog_print_all(char __user *buf, int size, bool clear)
                idx = clear_idx;
                prev = 0;
                while (seq < log_next_seq) {
-                        struct log *msg = log_from_idx(idx);
+                        struct printk_log *msg = log_from_idx(idx);
                        len += msg_print_text(msg, prev, true, NULL, 0);
                        prev = msg->flags;
@@ -1073,7 +1068,7 @@ static int syslog_print_all(char __user *buf, int size, bool clear)
                idx = clear_idx;
                prev = 0;
                while (len > size && seq < log_next_seq) {
-                        struct log *msg = log_from_idx(idx);
+                        struct printk_log *msg = log_from_idx(idx);
                        len -= msg_print_text(msg, prev, true, NULL, 0);
                        prev = msg->flags;
@@ -1087,7 +1082,7 @@ static int syslog_print_all(char __user *buf, int size, bool clear)
                len = 0;
                prev = 0;
                while (len >= 0 && seq < next_seq) {
-                        struct log *msg = log_from_idx(idx);
+                        struct printk_log *msg = log_from_idx(idx);
                        int textlen;
                        textlen = msg_print_text(msg, prev, true, text,
@@ -1233,7 +1228,7 @@ int do_syslog(int type, char __user *buf, int len, bool from_file)
                        error = 0;
                        while (seq < log_next_seq) {
-                                struct log *msg = log_from_idx(idx);
+                                struct printk_log *msg = log_from_idx(idx);
                                error += msg_print_text(msg, prev, true, NULL, 0);
                                idx = log_next(idx);
@@ -1719,10 +1714,10 @@ static struct cont {
        u8 level;
        bool flushed:1;
 } cont;
-static struct log *log_from_idx(u32 idx) { return NULL; }
+static struct printk_log *log_from_idx(u32 idx) { return NULL; }
 static u32 log_next(u32 idx) { return 0; }
 static void call_console_drivers(int level, const char *text, size_t len) {}
-static size_t msg_print_text(const struct log *msg, enum log_flags prev,
+static size_t msg_print_text(const struct printk_log *msg, enum log_flags prev,
                             bool syslog, char *buf, size_t size) { return 0; }
 static size_t cont_print_text(char *text, size_t size) { return 0; }
@@ -1761,23 +1756,23 @@ static int __add_preferred_console(char *name, int idx, char *options,
         *      See if this tty is not yet registered, and
         *      if we have a slot free.
         */
-        for (i = 0; i < MAX_CMDLINECONSOLES && console_cmdline[i].name[0]; i++)
+        for (i = 0, c = console_cmdline;
-                if (strcmp(console_cmdline[i].name, name) == 0 &&
+             i < MAX_CMDLINECONSOLES && c->name[0];
-                          console_cmdline[i].index == idx) {
+             i++, c++) {
-                                if (!brl_options)
+                if (strcmp(c->name, name) == 0 && c->index == idx) {
-                                        selected_console = i;
+                        if (!brl_options)
-                                return 0;
+                                selected_console = i;
+                        return 0;
                }
+        }
        if (i == MAX_CMDLINECONSOLES)
                return -E2BIG;
        if (!brl_options)
                selected_console = i;
-        c = &console_cmdline[i];
        strlcpy(c->name, name, sizeof(c->name));
        c->options = options;
-#ifdef CONFIG_A11Y_BRAILLE_CONSOLE
+        braille_set_options(c, brl_options);
-        c->brl_options = brl_options;
-#endif
        c->index = idx;
        return 0;
 }
@@ -1790,20 +1785,8 @@ static int __init console_setup(char *str)
        char *s, *options, *brl_options = NULL;
        int idx;
-#ifdef CONFIG_A11Y_BRAILLE_CONSOLE
+        if (_braille_console_setup(&str, &brl_options))
-        if (!memcmp(str, "brl,", 4)) {
+                return 1;
-                brl_options = "";
-                str += 4;
-        } else if (!memcmp(str, "brl=", 4)) {
-                brl_options = str + 4;
-                str = strchr(brl_options, ',');
-                if (!str) {
-                        printk(KERN_ERR "need port name after brl=\n");
-                        return 1;
-                }
-                *(str++) = 0;
-        }
-#endif
        /*
         * Decode str into name, index, options.
@@ -1858,15 +1841,15 @@ int update_console_cmdline(char *name, int idx, char *name_new, int idx_new, cha
        struct console_cmdline *c;
        int i;
-        for (i = 0; i < MAX_CMDLINECONSOLES && console_cmdline[i].name[0]; i++)
+        for (i = 0, c = console_cmdline;
-                if (strcmp(console_cmdline[i].name, name) == 0 &&
+             i < MAX_CMDLINECONSOLES && c->name[0];
-                          console_cmdline[i].index == idx) {
+             i++, c++)
-                                c = &console_cmdline[i];
+                if (strcmp(c->name, name) == 0 && c->index == idx) {
-                                strlcpy(c->name, name_new, sizeof(c->name));
+                        strlcpy(c->name, name_new, sizeof(c->name));
-                                c->name[sizeof(c->name) - 1] = 0;
+                        c->name[sizeof(c->name) - 1] = 0;
-                                c->options = options;
+                        c->options = options;
-                                c->index = idx_new;
+                        c->index = idx_new;
-                                return i;
+                        return i;
                }
        /* not found */
        return -1;
@@ -1921,7 +1904,7 @@ void resume_console(void)
 * called when a new CPU comes online (or fails to come up), and ensures
 * that any such output gets printed.
 */
-static int __cpuinit console_cpu_notify(struct notifier_block *self,
+static int console_cpu_notify(struct notifier_block *self,
        unsigned long action, void *hcpu)
 {
        switch (action) {
@@ -2046,7 +2029,7 @@ void console_unlock(void)
        console_cont_flush(text, sizeof(text));
 again:
        for (;;) {
-                struct log *msg;
+                struct printk_log *msg;
                size_t len;
                int level;
@@ -2241,6 +2224,14 @@ void register_console(struct console *newcon)
        int i;
        unsigned long flags;
        struct console *bcon = NULL;
+        struct console_cmdline *c;
+        if (console_drivers)
+                for_each_console(bcon)
+                        if (WARN(bcon == newcon,
+                                        "console '%s%d' already registered\n",
+                                        bcon->name, bcon->index))
+                                return;
        /*
         * before we register a new CON_BOOT console, make sure we don't
@@ -2288,30 +2279,25 @@ void register_console(struct console *newcon)
         *      See if this console matches one we selected on
         *      the command line.
         */
-        for (i = 0; i < MAX_CMDLINECONSOLES && console_cmdline[i].name[0];
+        for (i = 0, c = console_cmdline;
-                        i++) {
+             i < MAX_CMDLINECONSOLES && c->name[0];
-                if (strcmp(console_cmdline[i].name, newcon->name) != 0)
+             i++, c++) {
+                if (strcmp(c->name, newcon->name) != 0)
                        continue;
                if (newcon->index >= 0 &&
-                    newcon->index != console_cmdline[i].index)
+                    newcon->index != c->index)
                        continue;
                if (newcon->index < 0)
-                        newcon->index = console_cmdline[i].index;
+                        newcon->index = c->index;
-#ifdef CONFIG_A11Y_BRAILLE_CONSOLE
-                if (console_cmdline[i].brl_options) {
+                if (_braille_register_console(newcon, c))
-                        newcon->flags |= CON_BRL;
-                        braille_register_console(newcon,
-                                        console_cmdline[i].index,
-                                        console_cmdline[i].options,
-                                        console_cmdline[i].brl_options);
                        return;
-                }
-#endif
                if (newcon->setup &&
                    newcon->setup(newcon, console_cmdline[i].options) != 0)
                        break;
                newcon->flags |= CON_ENABLED;
-                newcon->index = console_cmdline[i].index;
+                newcon->index = c->index;
                if (i == selected_console) {
                        newcon->flags |= CON_CONSDEV;
                        preferred_console = selected_console;
@@ -2394,13 +2380,13 @@ EXPORT_SYMBOL(register_console);
 int unregister_console(struct console *console)
 {
        struct console *a, *b;
-        int res = 1;
+        int res;
-#ifdef CONFIG_A11Y_BRAILLE_CONSOLE
+        res = _braille_unregister_console(console);
-        if (console->flags & CON_BRL)
+        if (res)
-                return braille_unregister_console(console);
+                return res;
-#endif
+        res = 1;
        console_lock();
        if (console_drivers == console) {
                console_drivers=console->next;
@@ -2666,7 +2652,7 @@ void kmsg_dump(enum kmsg_dump_reason reason)
 bool kmsg_dump_get_line_nolock(struct kmsg_dumper *dumper, bool syslog,
                               char *line, size_t size, size_t *len)
 {
-        struct log *msg;
+        struct printk_log *msg;
        size_t l = 0;
        bool ret = false;
@@ -2778,7 +2764,7 @@ bool kmsg_dump_get_buffer(struct kmsg_dumper *dumper, bool syslog,
        idx = dumper->cur_idx;
        prev = 0;
        while (seq < dumper->next_seq) {
-                struct log *msg = log_from_idx(idx);
+                struct printk_log *msg = log_from_idx(idx);
                l += msg_print_text(msg, prev, true, NULL, 0);
                idx = log_next(idx);
@@ -2791,7 +2777,7 @@ bool kmsg_dump_get_buffer(struct kmsg_dumper *dumper, bool syslog,
        idx = dumper->cur_idx;
        prev = 0;
        while (l > size && seq < dumper->next_seq) {
-                struct log *msg = log_from_idx(idx);
+                struct printk_log *msg = log_from_idx(idx);
                l -= msg_print_text(msg, prev, true, NULL, 0);
                idx = log_next(idx);
@@ -2806,7 +2792,7 @@ bool kmsg_dump_get_buffer(struct kmsg_dumper *dumper, bool syslog,
        l = 0;
        prev = 0;
        while (seq < dumper->next_seq) {
-                struct log *msg = log_from_idx(idx);
+                struct printk_log *msg = log_from_idx(idx);
                l += msg_print_text(msg, prev, syslog, buf + l, size - l);
                idx = log_next(idx);
diff --git a/kernel/profile.c b/kernel/profile.c
index 0bf400737660..6631e1ef55ab 100644
--- a/kernel/profile.c
+++ b/kernel/profile.c
@@ -331,7 +331,7 @@ out:
        put_cpu();
 }
-static int __cpuinit profile_cpu_callback(struct notifier_block *info,
+static int profile_cpu_callback(struct notifier_block *info,
                                        unsigned long action, void *__cpu)
 {
        int node, cpu = (unsigned long)__cpu;
diff --git a/kernel/ptrace.c b/kernel/ptrace.c
index 4041f5747e73..a146ee327f6a 100644
--- a/kernel/ptrace.c
+++ b/kernel/ptrace.c
@@ -469,7 +469,6 @@ static int ptrace_detach(struct task_struct *child, unsigned int data)
        /* Architecture-specific hardware disable .. */
        ptrace_disable(child);
        clear_tsk_thread_flag(child, TIF_SYSCALL_TRACE);
-        flush_ptrace_hw_breakpoint(child);
        write_lock_irq(&tasklist_lock);
        /*
diff --git a/kernel/rcu.h b/kernel/rcu.h
index 7f8e7590e3e5..77131966c4ad 100644
--- a/kernel/rcu.h
+++ b/kernel/rcu.h
@@ -67,12 +67,15 @@
 extern struct debug_obj_descr rcuhead_debug_descr;
-static inline void debug_rcu_head_queue(struct rcu_head *head)
+static inline int debug_rcu_head_queue(struct rcu_head *head)
 {
-        debug_object_activate(head, &rcuhead_debug_descr);
+        int r1;
+        r1 = debug_object_activate(head, &rcuhead_debug_descr);
        debug_object_active_state(head, &rcuhead_debug_descr,
                                  STATE_RCU_HEAD_READY,
                                  STATE_RCU_HEAD_QUEUED);
+        return r1;
 }
 static inline void debug_rcu_head_unqueue(struct rcu_head *head)
@@ -83,8 +86,9 @@ static inline void debug_rcu_head_unqueue(struct rcu_head *head)
        debug_object_deactivate(head, &rcuhead_debug_descr);
 }
 #else   /* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
-static inline void debug_rcu_head_queue(struct rcu_head *head)
+static inline int debug_rcu_head_queue(struct rcu_head *head)
 {
+        return 0;
 }
 static inline void debug_rcu_head_unqueue(struct rcu_head *head)
@@ -94,7 +98,7 @@ static inline void debug_rcu_head_unqueue(struct rcu_head *head)
 extern void kfree(const void *);
-static inline bool __rcu_reclaim(char *rn, struct rcu_head *head)
+static inline bool __rcu_reclaim(const char *rn, struct rcu_head *head)
 {
        unsigned long offset = (unsigned long)head->func;
diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c
index cce6ba8bbace..33eb4620aa17 100644
--- a/kernel/rcupdate.c
+++ b/kernel/rcupdate.c
@@ -212,43 +212,6 @@ static inline void debug_rcu_head_free(struct rcu_head *head)
 }
 /*
- * fixup_init is called when:
- * - an active object is initialized
- */
-static int rcuhead_fixup_init(void *addr, enum debug_obj_state state)
-{
-        struct rcu_head *head = addr;
-        switch (state) {
-        case ODEBUG_STATE_ACTIVE:
-                /*
-                 * Ensure that queued callbacks are all executed.
-                 * If we detect that we are nested in a RCU read-side critical
-                 * section, we should simply fail, otherwise we would deadlock.
-                 * In !PREEMPT configurations, there is no way to tell if we are
-                 * in a RCU read-side critical section or not, so we never
-                 * attempt any fixup and just print a warning.
-                 */
-#ifndef CONFIG_PREEMPT
-                WARN_ON_ONCE(1);
-                return 0;
-#endif
-                if (rcu_preempt_depth() != 0 || preempt_count() != 0 ||
-                    irqs_disabled()) {
-                        WARN_ON_ONCE(1);
-                        return 0;
-                }
-                rcu_barrier();
-                rcu_barrier_sched();
-                rcu_barrier_bh();
-                debug_object_init(head, &rcuhead_debug_descr);
-                return 1;
-        default:
-                return 0;
-        }
-}
-/*
 * fixup_activate is called when:
 * - an active object is activated
 * - an unknown object is activated (might be a statically initialized object)
@@ -268,69 +231,8 @@ static int rcuhead_fixup_activate(void *addr, enum debug_obj_state state)
                debug_object_init(head, &rcuhead_debug_descr);
                debug_object_activate(head, &rcuhead_debug_descr);
                return 0;
-        case ODEBUG_STATE_ACTIVE:
-                /*
-                 * Ensure that queued callbacks are all executed.
-                 * If we detect that we are nested in a RCU read-side critical
-                 * section, we should simply fail, otherwise we would deadlock.
-                 * In !PREEMPT configurations, there is no way to tell if we are
-                 * in a RCU read-side critical section or not, so we never
-                 * attempt any fixup and just print a warning.
-                 */
-#ifndef CONFIG_PREEMPT
-                WARN_ON_ONCE(1);
-                return 0;
-#endif
-                if (rcu_preempt_depth() != 0 || preempt_count() != 0 ||
-                    irqs_disabled()) {
-                        WARN_ON_ONCE(1);
-                        return 0;
-                }
-                rcu_barrier();
-                rcu_barrier_sched();
-                rcu_barrier_bh();
-                debug_object_activate(head, &rcuhead_debug_descr);
-                return 1;
        default:
-                return 0;
-        }
-}
-/*
- * fixup_free is called when:
- * - an active object is freed
- */
-static int rcuhead_fixup_free(void *addr, enum debug_obj_state state)
-{
-        struct rcu_head *head = addr;
-        switch (state) {
-        case ODEBUG_STATE_ACTIVE:
-                /*
-                 * Ensure that queued callbacks are all executed.
-                 * If we detect that we are nested in a RCU read-side critical
-                 * section, we should simply fail, otherwise we would deadlock.
-                 * In !PREEMPT configurations, there is no way to tell if we are
-                 * in a RCU read-side critical section or not, so we never
-                 * attempt any fixup and just print a warning.
-                 */
-#ifndef CONFIG_PREEMPT
-                WARN_ON_ONCE(1);
-                return 0;
-#endif
-                if (rcu_preempt_depth() != 0 || preempt_count() != 0 ||
-                    irqs_disabled()) {
-                        WARN_ON_ONCE(1);
-                        return 0;
-                }
-                rcu_barrier();
-                rcu_barrier_sched();
-                rcu_barrier_bh();
-                debug_object_free(head, &rcuhead_debug_descr);
                return 1;
-        default:
-                return 0;
        }
 }
@@ -369,15 +271,13 @@ EXPORT_SYMBOL_GPL(destroy_rcu_head_on_stack);
 struct debug_obj_descr rcuhead_debug_descr = {
        .name = "rcu_head",
-        .fixup_init = rcuhead_fixup_init,
        .fixup_activate = rcuhead_fixup_activate,
-        .fixup_free = rcuhead_fixup_free,
 };
 EXPORT_SYMBOL_GPL(rcuhead_debug_descr);
 #endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
 #if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU) || defined(CONFIG_RCU_TRACE)
-void do_trace_rcu_torture_read(char *rcutorturename, struct rcu_head *rhp,
+void do_trace_rcu_torture_read(const char *rcutorturename, struct rcu_head *rhp,
                               unsigned long secs,
                               unsigned long c_old, unsigned long c)
 {
diff --git a/kernel/rcutiny.c b/kernel/rcutiny.c
index aa344111de3e..9ed6075dc562 100644
--- a/kernel/rcutiny.c
+++ b/kernel/rcutiny.c
@@ -264,7 +264,7 @@ void rcu_check_callbacks(int cpu, int user)
 */
 static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp)
 {
-        char *rn = NULL;
+        const char *rn = NULL;
        struct rcu_head *next, *list;
        unsigned long flags;
        RCU_TRACE(int cb_count = 0);
diff --git a/kernel/rcutiny_plugin.h b/kernel/rcutiny_plugin.h
index 0cd385acccfa..280d06cae352 100644
--- a/kernel/rcutiny_plugin.h
+++ b/kernel/rcutiny_plugin.h
@@ -36,7 +36,7 @@ struct rcu_ctrlblk {
        RCU_TRACE(unsigned long gp_start); /* Start time for stalls. */
        RCU_TRACE(unsigned long ticks_this_gp); /* Statistic for stalls. */
        RCU_TRACE(unsigned long jiffies_stall); /* Jiffies at next stall. */
-        RCU_TRACE(char *name);          /* Name of RCU type. */
+        RCU_TRACE(const char *name);    /* Name of RCU type. */
 };
 /* Definition for rcupdate control block. */
diff --git a/kernel/rcutorture.c b/kernel/rcutorture.c
index b1fa5510388d..be63101c6175 100644
--- a/kernel/rcutorture.c
+++ b/kernel/rcutorture.c
@@ -52,72 +52,78 @@
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and Josh Triplett <josh@freedesktop.org>");
-static int nreaders = -1;       /* # reader threads, defaults to 2*ncpus */
+static int fqs_duration;
-static int nfakewriters = 4;    /* # fake writer threads */
-static int stat_interval = 60;  /* Interval between stats, in seconds. */
-                                /*  Zero means "only at end of test". */
-static bool verbose;            /* Print more debug info. */
-static bool test_no_idle_hz = true;
-                                /* Test RCU support for tickless idle CPUs. */
-static int shuffle_interval = 3; /* Interval between shuffles (in sec)*/
-static int stutter = 5;         /* Start/stop testing interval (in sec) */
-static int irqreader = 1;       /* RCU readers from irq (timers). */
-static int fqs_duration;        /* Duration of bursts (us), 0 to disable. */
-static int fqs_holdoff;         /* Hold time within burst (us). */
-static int fqs_stutter = 3;     /* Wait time between bursts (s). */
-static int n_barrier_cbs;       /* Number of callbacks to test RCU barriers. */
-static int onoff_interval;      /* Wait time between CPU hotplugs, 0=disable. */
-static int onoff_holdoff;       /* Seconds after boot before CPU hotplugs. */
-static int shutdown_secs;       /* Shutdown time (s).  <=0 for no shutdown. */
-static int stall_cpu;           /* CPU-stall duration (s).  0 for no stall. */
-static int stall_cpu_holdoff = 10; /* Time to wait until stall (s).  */
-static int test_boost = 1;      /* Test RCU prio boost: 0=no, 1=maybe, 2=yes. */
-static int test_boost_interval = 7; /* Interval between boost tests, seconds. */
-static int test_boost_duration = 4; /* Duration of each boost test, seconds. */
-static char *torture_type = "rcu"; /* What RCU implementation to torture. */
-module_param(nreaders, int, 0444);
-MODULE_PARM_DESC(nreaders, "Number of RCU reader threads");
-module_param(nfakewriters, int, 0444);
-MODULE_PARM_DESC(nfakewriters, "Number of RCU fake writer threads");
-module_param(stat_interval, int, 0644);
-MODULE_PARM_DESC(stat_interval, "Number of seconds between stats printk()s");
-module_param(verbose, bool, 0444);
-MODULE_PARM_DESC(verbose, "Enable verbose debugging printk()s");
-module_param(test_no_idle_hz, bool, 0444);
-MODULE_PARM_DESC(test_no_idle_hz, "Test support for tickless idle CPUs");
-module_param(shuffle_interval, int, 0444);
-MODULE_PARM_DESC(shuffle_interval, "Number of seconds between shuffles");
-module_param(stutter, int, 0444);
-MODULE_PARM_DESC(stutter, "Number of seconds to run/halt test");
-module_param(irqreader, int, 0444);
-MODULE_PARM_DESC(irqreader, "Allow RCU readers from irq handlers");
 module_param(fqs_duration, int, 0444);
-MODULE_PARM_DESC(fqs_duration, "Duration of fqs bursts (us)");
+MODULE_PARM_DESC(fqs_duration, "Duration of fqs bursts (us), 0 to disable");
+static int fqs_holdoff;
 module_param(fqs_holdoff, int, 0444);
 MODULE_PARM_DESC(fqs_holdoff, "Holdoff time within fqs bursts (us)");
+static int fqs_stutter = 3;
 module_param(fqs_stutter, int, 0444);
 MODULE_PARM_DESC(fqs_stutter, "Wait time between fqs bursts (s)");
+static bool gp_exp;
+module_param(gp_exp, bool, 0444);
+MODULE_PARM_DESC(gp_exp, "Use expedited GP wait primitives");
+static bool gp_normal;
+module_param(gp_normal, bool, 0444);
+MODULE_PARM_DESC(gp_normal, "Use normal (non-expedited) GP wait primitives");
+static int irqreader = 1;
+module_param(irqreader, int, 0444);
+MODULE_PARM_DESC(irqreader, "Allow RCU readers from irq handlers");
+static int n_barrier_cbs;
 module_param(n_barrier_cbs, int, 0444);
 MODULE_PARM_DESC(n_barrier_cbs, "# of callbacks/kthreads for barrier testing");
-module_param(onoff_interval, int, 0444);
+static int nfakewriters = 4;
-MODULE_PARM_DESC(onoff_interval, "Time between CPU hotplugs (s), 0=disable");
+module_param(nfakewriters, int, 0444);
+MODULE_PARM_DESC(nfakewriters, "Number of RCU fake writer threads");
+static int nreaders = -1;
+module_param(nreaders, int, 0444);
+MODULE_PARM_DESC(nreaders, "Number of RCU reader threads");
+static int object_debug;
+module_param(object_debug, int, 0444);
+MODULE_PARM_DESC(object_debug, "Enable debug-object double call_rcu() testing");
+static int onoff_holdoff;
 module_param(onoff_holdoff, int, 0444);
 MODULE_PARM_DESC(onoff_holdoff, "Time after boot before CPU hotplugs (s)");
+static int onoff_interval;
+module_param(onoff_interval, int, 0444);
+MODULE_PARM_DESC(onoff_interval, "Time between CPU hotplugs (s), 0=disable");
+static int shuffle_interval = 3;
+module_param(shuffle_interval, int, 0444);
+MODULE_PARM_DESC(shuffle_interval, "Number of seconds between shuffles");
+static int shutdown_secs;
 module_param(shutdown_secs, int, 0444);
-MODULE_PARM_DESC(shutdown_secs, "Shutdown time (s), zero to disable.");
+MODULE_PARM_DESC(shutdown_secs, "Shutdown time (s), <= zero to disable.");
+static int stall_cpu;
 module_param(stall_cpu, int, 0444);
 MODULE_PARM_DESC(stall_cpu, "Stall duration (s), zero to disable.");
+static int stall_cpu_holdoff = 10;
 module_param(stall_cpu_holdoff, int, 0444);
 MODULE_PARM_DESC(stall_cpu_holdoff, "Time to wait before starting stall (s).");
+static int stat_interval = 60;
+module_param(stat_interval, int, 0644);
+MODULE_PARM_DESC(stat_interval, "Number of seconds between stats printk()s");
+static int stutter = 5;
+module_param(stutter, int, 0444);
+MODULE_PARM_DESC(stutter, "Number of seconds to run/halt test");
+static int test_boost = 1;
 module_param(test_boost, int, 0444);
 MODULE_PARM_DESC(test_boost, "Test RCU prio boost: 0=no, 1=maybe, 2=yes.");
-module_param(test_boost_interval, int, 0444);
+static int test_boost_duration = 4;
-MODULE_PARM_DESC(test_boost_interval, "Interval between boost tests, seconds.");
 module_param(test_boost_duration, int, 0444);
 MODULE_PARM_DESC(test_boost_duration, "Duration of each boost test, seconds.");
+static int test_boost_interval = 7;
+module_param(test_boost_interval, int, 0444);
+MODULE_PARM_DESC(test_boost_interval, "Interval between boost tests, seconds.");
+static bool test_no_idle_hz = true;
+module_param(test_no_idle_hz, bool, 0444);
+MODULE_PARM_DESC(test_no_idle_hz, "Test support for tickless idle CPUs");
+static char *torture_type = "rcu";
 module_param(torture_type, charp, 0444);
-MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, srcu)");
+MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, ...)");
+static bool verbose;
+module_param(verbose, bool, 0444);
+MODULE_PARM_DESC(verbose, "Enable verbose debugging printk()s");
 #define TORTURE_FLAG "-torture:"
 #define PRINTK_STRING(s) \
@@ -267,7 +273,7 @@ rcutorture_shutdown_notify(struct notifier_block *unused1,
 * Absorb kthreads into a kernel function that won't return, so that
 * they won't ever access module text or data again.
 */
-static void rcutorture_shutdown_absorb(char *title)
+static void rcutorture_shutdown_absorb(const char *title)
 {
        if (ACCESS_ONCE(fullstop) == FULLSTOP_SHUTDOWN) {
                pr_notice(
@@ -337,7 +343,7 @@ rcu_random(struct rcu_random_state *rrsp)
 }
 static void
-rcu_stutter_wait(char *title)
+rcu_stutter_wait(const char *title)
 {
        while (stutter_pause_test || !rcutorture_runnable) {
                if (rcutorture_runnable)
@@ -360,13 +366,14 @@ struct rcu_torture_ops {
        int (*completed)(void);
        void (*deferred_free)(struct rcu_torture *p);
        void (*sync)(void);
+        void (*exp_sync)(void);
        void (*call)(struct rcu_head *head, void (*func)(struct rcu_head *rcu));
        void (*cb_barrier)(void);
        void (*fqs)(void);
        int (*stats)(char *page);
        int irq_capable;
        int can_boost;
-        char *name;
+        const char *name;
 };
 static struct rcu_torture_ops *cur_ops;
@@ -443,81 +450,27 @@ static void rcu_torture_deferred_free(struct rcu_torture *p)
        call_rcu(&p->rtort_rcu, rcu_torture_cb);
 }
-static struct rcu_torture_ops rcu_ops = {
-        .init           = NULL,
-        .readlock       = rcu_torture_read_lock,
-        .read_delay     = rcu_read_delay,
-        .readunlock     = rcu_torture_read_unlock,
-        .completed      = rcu_torture_completed,
-        .deferred_free  = rcu_torture_deferred_free,
-        .sync           = synchronize_rcu,
-        .call           = call_rcu,
-        .cb_barrier     = rcu_barrier,
-        .fqs            = rcu_force_quiescent_state,
-        .stats          = NULL,
-        .irq_capable    = 1,
-        .can_boost      = rcu_can_boost(),
-        .name           = "rcu"
-};
-static void rcu_sync_torture_deferred_free(struct rcu_torture *p)
-{
-        int i;
-        struct rcu_torture *rp;
-        struct rcu_torture *rp1;
-        cur_ops->sync();
-        list_add(&p->rtort_free, &rcu_torture_removed);
-        list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) {
-                i = rp->rtort_pipe_count;
-                if (i > RCU_TORTURE_PIPE_LEN)
-                        i = RCU_TORTURE_PIPE_LEN;
-                atomic_inc(&rcu_torture_wcount[i]);
-                if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
-                        rp->rtort_mbtest = 0;
-                        list_del(&rp->rtort_free);
-                        rcu_torture_free(rp);
-                }
-        }
-}
 static void rcu_sync_torture_init(void)
 {
        INIT_LIST_HEAD(&rcu_torture_removed);
 }
-static struct rcu_torture_ops rcu_sync_ops = {
+static struct rcu_torture_ops rcu_ops = {
        .init           = rcu_sync_torture_init,
        .readlock       = rcu_torture_read_lock,
        .read_delay     = rcu_read_delay,
        .readunlock     = rcu_torture_read_unlock,
        .completed      = rcu_torture_completed,
-        .deferred_free  = rcu_sync_torture_deferred_free,
+        .deferred_free  = rcu_torture_deferred_free,
        .sync           = synchronize_rcu,
-        .call           = NULL,
+        .exp_sync       = synchronize_rcu_expedited,
-        .cb_barrier     = NULL,
+        .call           = call_rcu,
-        .fqs            = rcu_force_quiescent_state,
+        .cb_barrier     = rcu_barrier,
-        .stats          = NULL,
-        .irq_capable    = 1,
-        .can_boost      = rcu_can_boost(),
-        .name           = "rcu_sync"
-};
-static struct rcu_torture_ops rcu_expedited_ops = {
-        .init           = rcu_sync_torture_init,
-        .readlock       = rcu_torture_read_lock,
-        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
-        .readunlock     = rcu_torture_read_unlock,
-        .completed      = rcu_no_completed,
-        .deferred_free  = rcu_sync_torture_deferred_free,
-        .sync           = synchronize_rcu_expedited,
-        .call           = NULL,
-        .cb_barrier     = NULL,
        .fqs            = rcu_force_quiescent_state,
        .stats          = NULL,
        .irq_capable    = 1,
        .can_boost      = rcu_can_boost(),
-        .name           = "rcu_expedited"
+        .name           = "rcu"
 };
 /*
@@ -546,13 +499,14 @@ static void rcu_bh_torture_deferred_free(struct rcu_torture *p)
 }
 static struct rcu_torture_ops rcu_bh_ops = {
-        .init           = NULL,
+        .init           = rcu_sync_torture_init,
        .readlock       = rcu_bh_torture_read_lock,
        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
        .readunlock     = rcu_bh_torture_read_unlock,
        .completed      = rcu_bh_torture_completed,
        .deferred_free  = rcu_bh_torture_deferred_free,
        .sync           = synchronize_rcu_bh,
+        .exp_sync       = synchronize_rcu_bh_expedited,
        .call           = call_rcu_bh,
        .cb_barrier     = rcu_barrier_bh,
        .fqs            = rcu_bh_force_quiescent_state,
@@ -561,38 +515,6 @@ static struct rcu_torture_ops rcu_bh_ops = {
        .name           = "rcu_bh"
 };
-static struct rcu_torture_ops rcu_bh_sync_ops = {
-        .init           = rcu_sync_torture_init,
-        .readlock       = rcu_bh_torture_read_lock,
-        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
-        .readunlock     = rcu_bh_torture_read_unlock,
-        .completed      = rcu_bh_torture_completed,
-        .deferred_free  = rcu_sync_torture_deferred_free,
-        .sync           = synchronize_rcu_bh,
-        .call           = NULL,
-        .cb_barrier     = NULL,
-        .fqs            = rcu_bh_force_quiescent_state,
-        .stats          = NULL,
-        .irq_capable    = 1,
-        .name           = "rcu_bh_sync"
-};
-static struct rcu_torture_ops rcu_bh_expedited_ops = {
-        .init           = rcu_sync_torture_init,
-        .readlock       = rcu_bh_torture_read_lock,
-        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
-        .readunlock     = rcu_bh_torture_read_unlock,
-        .completed      = rcu_bh_torture_completed,
-        .deferred_free  = rcu_sync_torture_deferred_free,
-        .sync           = synchronize_rcu_bh_expedited,
-        .call           = NULL,
-        .cb_barrier     = NULL,
-        .fqs            = rcu_bh_force_quiescent_state,
-        .stats          = NULL,
-        .irq_capable    = 1,
-        .name           = "rcu_bh_expedited"
-};
 /*
 * Definitions for srcu torture testing.
 */
@@ -667,6 +589,11 @@ static int srcu_torture_stats(char *page)
        return cnt;
 }
+static void srcu_torture_synchronize_expedited(void)
+{
+        synchronize_srcu_expedited(&srcu_ctl);
+}
 static struct rcu_torture_ops srcu_ops = {
        .init           = rcu_sync_torture_init,
        .readlock       = srcu_torture_read_lock,
@@ -675,45 +602,13 @@ static struct rcu_torture_ops srcu_ops = {
        .completed      = srcu_torture_completed,
        .deferred_free  = srcu_torture_deferred_free,
        .sync           = srcu_torture_synchronize,
+        .exp_sync       = srcu_torture_synchronize_expedited,
        .call           = srcu_torture_call,
        .cb_barrier     = srcu_torture_barrier,
        .stats          = srcu_torture_stats,
        .name           = "srcu"
 };
-static struct rcu_torture_ops srcu_sync_ops = {
-        .init           = rcu_sync_torture_init,
-        .readlock       = srcu_torture_read_lock,
-        .read_delay     = srcu_read_delay,
-        .readunlock     = srcu_torture_read_unlock,
-        .completed      = srcu_torture_completed,
-        .deferred_free  = rcu_sync_torture_deferred_free,
-        .sync           = srcu_torture_synchronize,
-        .call           = NULL,
-        .cb_barrier     = NULL,
-        .stats          = srcu_torture_stats,
-        .name           = "srcu_sync"
-};
-static void srcu_torture_synchronize_expedited(void)
-{
-        synchronize_srcu_expedited(&srcu_ctl);
-}
-static struct rcu_torture_ops srcu_expedited_ops = {
-        .init           = rcu_sync_torture_init,
-        .readlock       = srcu_torture_read_lock,
-        .read_delay     = srcu_read_delay,
-        .readunlock     = srcu_torture_read_unlock,
-        .completed      = srcu_torture_completed,
-        .deferred_free  = rcu_sync_torture_deferred_free,
-        .sync           = srcu_torture_synchronize_expedited,
-        .call           = NULL,
-        .cb_barrier     = NULL,
-        .stats          = srcu_torture_stats,
-        .name           = "srcu_expedited"
-};
 /*
 * Definitions for sched torture testing.
 */
@@ -742,6 +637,8 @@ static struct rcu_torture_ops sched_ops = {
        .completed      = rcu_no_completed,
        .deferred_free  = rcu_sched_torture_deferred_free,
        .sync           = synchronize_sched,
+        .exp_sync       = synchronize_sched_expedited,
+        .call           = call_rcu_sched,
        .cb_barrier     = rcu_barrier_sched,
        .fqs            = rcu_sched_force_quiescent_state,
        .stats          = NULL,
@@ -749,35 +646,6 @@ static struct rcu_torture_ops sched_ops = {
        .name           = "sched"
 };
-static struct rcu_torture_ops sched_sync_ops = {
-        .init           = rcu_sync_torture_init,
-        .readlock       = sched_torture_read_lock,
-        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
-        .readunlock     = sched_torture_read_unlock,
-        .completed      = rcu_no_completed,
-        .deferred_free  = rcu_sync_torture_deferred_free,
-        .sync           = synchronize_sched,
-        .cb_barrier     = NULL,
-        .fqs            = rcu_sched_force_quiescent_state,
-        .stats          = NULL,
-        .name           = "sched_sync"
-};
-static struct rcu_torture_ops sched_expedited_ops = {
-        .init           = rcu_sync_torture_init,
-        .readlock       = sched_torture_read_lock,
-        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
-        .readunlock     = sched_torture_read_unlock,
-        .completed      = rcu_no_completed,
-        .deferred_free  = rcu_sync_torture_deferred_free,
-        .sync           = synchronize_sched_expedited,
-        .cb_barrier     = NULL,
-        .fqs            = rcu_sched_force_quiescent_state,
-        .stats          = NULL,
-        .irq_capable    = 1,
-        .name           = "sched_expedited"
-};
 /*
 * RCU torture priority-boost testing.  Runs one real-time thread per
 * CPU for moderate bursts, repeatedly registering RCU callbacks and
@@ -927,9 +795,10 @@ rcu_torture_fqs(void *arg)
 static int
 rcu_torture_writer(void *arg)
 {
+        bool exp;
        int i;
-        long oldbatch = rcu_batches_completed();
        struct rcu_torture *rp;
+        struct rcu_torture *rp1;
        struct rcu_torture *old_rp;
        static DEFINE_RCU_RANDOM(rand);
@@ -954,10 +823,33 @@ rcu_torture_writer(void *arg)
                                i = RCU_TORTURE_PIPE_LEN;
                        atomic_inc(&rcu_torture_wcount[i]);
                        old_rp->rtort_pipe_count++;
-                        cur_ops->deferred_free(old_rp);
+                        if (gp_normal == gp_exp)
+                                exp = !!(rcu_random(&rand) & 0x80);
+                        else
+                                exp = gp_exp;
+                        if (!exp) {
+                                cur_ops->deferred_free(old_rp);
+                        } else {
+                                cur_ops->exp_sync();
+                                list_add(&old_rp->rtort_free,
+                                         &rcu_torture_removed);
+                                list_for_each_entry_safe(rp, rp1,
+                                                         &rcu_torture_removed,
+                                                         rtort_free) {
+                                        i = rp->rtort_pipe_count;
+                                        if (i > RCU_TORTURE_PIPE_LEN)
+                                                i = RCU_TORTURE_PIPE_LEN;
+                                        atomic_inc(&rcu_torture_wcount[i]);
+                                        if (++rp->rtort_pipe_count >=
+                                            RCU_TORTURE_PIPE_LEN) {
+                                                rp->rtort_mbtest = 0;
+                                                list_del(&rp->rtort_free);
+                                                rcu_torture_free(rp);
+                                        }
+                                 }
+                        }
                }
                rcutorture_record_progress(++rcu_torture_current_version);
-                oldbatch = cur_ops->completed();
                rcu_stutter_wait("rcu_torture_writer");
        } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
        VERBOSE_PRINTK_STRING("rcu_torture_writer task stopping");
@@ -983,10 +875,18 @@ rcu_torture_fakewriter(void *arg)
                schedule_timeout_uninterruptible(1 + rcu_random(&rand)%10);
                udelay(rcu_random(&rand) & 0x3ff);
                if (cur_ops->cb_barrier != NULL &&
-                    rcu_random(&rand) % (nfakewriters * 8) == 0)
+                    rcu_random(&rand) % (nfakewriters * 8) == 0) {
                        cur_ops->cb_barrier();
-                else
+                } else if (gp_normal == gp_exp) {
+                        if (rcu_random(&rand) & 0x80)
+                                cur_ops->sync();
+                        else
+                                cur_ops->exp_sync();
+                } else if (gp_normal) {
                        cur_ops->sync();
+                } else {
+                        cur_ops->exp_sync();
+                }
                rcu_stutter_wait("rcu_torture_fakewriter");
        } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
@@ -1364,7 +1264,7 @@ rcu_torture_stutter(void *arg)
 }
 static inline void
-rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, char *tag)
+rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag)
 {
        pr_alert("%s" TORTURE_FLAG
                 "--- %s: nreaders=%d nfakewriters=%d "
@@ -1476,7 +1376,7 @@ rcu_torture_shutdown(void *arg)
 * Execute random CPU-hotplug operations at the interval specified
 * by the onoff_interval.
 */
-static int __cpuinit
+static int
 rcu_torture_onoff(void *arg)
 {
        int cpu;
@@ -1534,7 +1434,13 @@ rcu_torture_onoff(void *arg)
                                         torture_type, cpu);
                        starttime = jiffies;
                        n_online_attempts++;
-                        if (cpu_up(cpu) == 0) {
+                        ret = cpu_up(cpu);
+                        if (ret) {
+                                if (verbose)
+                                        pr_alert("%s" TORTURE_FLAG
+                                                 "rcu_torture_onoff task: online %d failed: errno %d\n",
+                                                 torture_type, cpu, ret);
+                        } else {
                                if (verbose)
                                        pr_alert("%s" TORTURE_FLAG
                                                 "rcu_torture_onoff task: onlined %d\n",
@@ -1558,7 +1464,7 @@ rcu_torture_onoff(void *arg)
        return 0;
 }
-static int __cpuinit
+static int
 rcu_torture_onoff_init(void)
 {
        int ret;
@@ -1601,7 +1507,7 @@ static void rcu_torture_onoff_cleanup(void)
 * CPU-stall kthread.  It waits as specified by stall_cpu_holdoff, then
 * induces a CPU stall for the time specified by stall_cpu.
 */
-static int __cpuinit rcu_torture_stall(void *args)
+static int rcu_torture_stall(void *args)
 {
        unsigned long stop_at;
@@ -1934,6 +1840,62 @@ rcu_torture_cleanup(void)
                rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS");
 }
+#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
+static void rcu_torture_leak_cb(struct rcu_head *rhp)
+{
+}
+static void rcu_torture_err_cb(struct rcu_head *rhp)
+{
+        /*
+         * This -might- happen due to race conditions, but is unlikely.
+         * The scenario that leads to this happening is that the
+         * first of the pair of duplicate callbacks is queued,
+         * someone else starts a grace period that includes that
+         * callback, then the second of the pair must wait for the
+         * next grace period.  Unlikely, but can happen.  If it
+         * does happen, the debug-objects subsystem won't have splatted.
+         */
+        pr_alert("rcutorture: duplicated callback was invoked.\n");
+}
+#endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
+/*
+ * Verify that double-free causes debug-objects to complain, but only
+ * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y.  Otherwise, say that the test
+ * cannot be carried out.
+ */
+static void rcu_test_debug_objects(void)
+{
+#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
+        struct rcu_head rh1;
+        struct rcu_head rh2;
+        init_rcu_head_on_stack(&rh1);
+        init_rcu_head_on_stack(&rh2);
+        pr_alert("rcutorture: WARN: Duplicate call_rcu() test starting.\n");
+        /* Try to queue the rh2 pair of callbacks for the same grace period. */
+        preempt_disable(); /* Prevent preemption from interrupting test. */
+        rcu_read_lock(); /* Make it impossible to finish a grace period. */
+        call_rcu(&rh1, rcu_torture_leak_cb); /* Start grace period. */
+        local_irq_disable(); /* Make it harder to start a new grace period. */
+        call_rcu(&rh2, rcu_torture_leak_cb);
+        call_rcu(&rh2, rcu_torture_err_cb); /* Duplicate callback. */
+        local_irq_enable();
+        rcu_read_unlock();
+        preempt_enable();
+        /* Wait for them all to get done so we can safely return. */
+        rcu_barrier();
+        pr_alert("rcutorture: WARN: Duplicate call_rcu() test complete.\n");
+        destroy_rcu_head_on_stack(&rh1);
+        destroy_rcu_head_on_stack(&rh2);
+#else /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
+        pr_alert("rcutorture: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_rcu()\n");
+#endif /* #else #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
+}
 static int __init
 rcu_torture_init(void)
 {
@@ -1941,11 +1903,9 @@ rcu_torture_init(void)
        int cpu;
        int firsterr = 0;
        int retval;
-        static struct rcu_torture_ops *torture_ops[] =
+        static struct rcu_torture_ops *torture_ops[] = {
-                { &rcu_ops, &rcu_sync_ops, &rcu_expedited_ops,
+                &rcu_ops, &rcu_bh_ops, &srcu_ops, &sched_ops,
-                  &rcu_bh_ops, &rcu_bh_sync_ops, &rcu_bh_expedited_ops,
+        };
-                  &srcu_ops, &srcu_sync_ops, &srcu_expedited_ops,
-                  &sched_ops, &sched_sync_ops, &sched_expedited_ops, };
        mutex_lock(&fullstop_mutex);
@@ -2163,6 +2123,8 @@ rcu_torture_init(void)
                firsterr = retval;
                goto unwind;
        }
+        if (object_debug)
+                rcu_test_debug_objects();
        rcutorture_record_test_transition();
        mutex_unlock(&fullstop_mutex);
        return 0;
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index e08abb9461ac..32618b3fe4e6 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -53,18 +53,38 @@
 #include <linux/delay.h>
 #include <linux/stop_machine.h>
 #include <linux/random.h>
+#include <linux/ftrace_event.h>
+#include <linux/suspend.h>
 #include "rcutree.h"
 #include <trace/events/rcu.h>
 #include "rcu.h"
+/*
+ * Strings used in tracepoints need to be exported via the
+ * tracing system such that tools like perf and trace-cmd can
+ * translate the string address pointers to actual text.
+ */
+#define TPS(x)  tracepoint_string(x)
 /* Data structures. */
 static struct lock_class_key rcu_node_class[RCU_NUM_LVLS];
 static struct lock_class_key rcu_fqs_class[RCU_NUM_LVLS];
-#define RCU_STATE_INITIALIZER(sname, sabbr, cr) { \
+/*
+ * In order to export the rcu_state name to the tracing tools, it
+ * needs to be added in the __tracepoint_string section.
+ * This requires defining a separate variable tp_<sname>_varname
+ * that points to the string being used, and this will allow
+ * the tracing userspace tools to be able to decipher the string
+ * address to the matching string.
+ */
+#define RCU_STATE_INITIALIZER(sname, sabbr, cr) \
+static char sname##_varname[] = #sname; \
+static const char *tp_##sname##_varname __used __tracepoint_string = sname##_varname; \
+struct rcu_state sname##_state = { \
        .level = { &sname##_state.node[0] }, \
        .call = cr, \
        .fqs_state = RCU_GP_IDLE, \
@@ -75,16 +95,13 @@ static struct lock_class_key rcu_fqs_class[RCU_NUM_LVLS];
        .orphan_donetail = &sname##_state.orphan_donelist, \
        .barrier_mutex = __MUTEX_INITIALIZER(sname##_state.barrier_mutex), \
        .onoff_mutex = __MUTEX_INITIALIZER(sname##_state.onoff_mutex), \
-        .name = #sname, \
+        .name = sname##_varname, \
        .abbr = sabbr, \
-}
+}; \
+DEFINE_PER_CPU(struct rcu_data, sname##_data)
-struct rcu_state rcu_sched_state =
-        RCU_STATE_INITIALIZER(rcu_sched, 's', call_rcu_sched);
-DEFINE_PER_CPU(struct rcu_data, rcu_sched_data);
-struct rcu_state rcu_bh_state = RCU_STATE_INITIALIZER(rcu_bh, 'b', call_rcu_bh);
+RCU_STATE_INITIALIZER(rcu_sched, 's', call_rcu_sched);
-DEFINE_PER_CPU(struct rcu_data, rcu_bh_data);
+RCU_STATE_INITIALIZER(rcu_bh, 'b', call_rcu_bh);
 static struct rcu_state *rcu_state;
 LIST_HEAD(rcu_struct_flavors);
@@ -178,7 +195,7 @@ void rcu_sched_qs(int cpu)
        struct rcu_data *rdp = &per_cpu(rcu_sched_data, cpu);
        if (rdp->passed_quiesce == 0)
-                trace_rcu_grace_period("rcu_sched", rdp->gpnum, "cpuqs");
+                trace_rcu_grace_period(TPS("rcu_sched"), rdp->gpnum, TPS("cpuqs"));
        rdp->passed_quiesce = 1;
 }
@@ -187,7 +204,7 @@ void rcu_bh_qs(int cpu)
        struct rcu_data *rdp = &per_cpu(rcu_bh_data, cpu);
        if (rdp->passed_quiesce == 0)
-                trace_rcu_grace_period("rcu_bh", rdp->gpnum, "cpuqs");
+                trace_rcu_grace_period(TPS("rcu_bh"), rdp->gpnum, TPS("cpuqs"));
        rdp->passed_quiesce = 1;
 }
@@ -198,16 +215,20 @@ void rcu_bh_qs(int cpu)
 */
 void rcu_note_context_switch(int cpu)
 {
-        trace_rcu_utilization("Start context switch");
+        trace_rcu_utilization(TPS("Start context switch"));
        rcu_sched_qs(cpu);
        rcu_preempt_note_context_switch(cpu);
-        trace_rcu_utilization("End context switch");
+        trace_rcu_utilization(TPS("End context switch"));
 }
 EXPORT_SYMBOL_GPL(rcu_note_context_switch);
 DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
        .dynticks_nesting = DYNTICK_TASK_EXIT_IDLE,
        .dynticks = ATOMIC_INIT(1),
+#ifdef CONFIG_NO_HZ_FULL_SYSIDLE
+        .dynticks_idle_nesting = DYNTICK_TASK_NEST_VALUE,
+        .dynticks_idle = ATOMIC_INIT(1),
+#endif /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
 };
 static long blimit = 10;        /* Maximum callbacks per rcu_do_batch. */
@@ -226,7 +247,10 @@ module_param(jiffies_till_next_fqs, ulong, 0644);
 static void rcu_start_gp_advanced(struct rcu_state *rsp, struct rcu_node *rnp,
                                  struct rcu_data *rdp);
-static void force_qs_rnp(struct rcu_state *rsp, int (*f)(struct rcu_data *));
+static void force_qs_rnp(struct rcu_state *rsp,
+                         int (*f)(struct rcu_data *rsp, bool *isidle,
+                                  unsigned long *maxj),
+                         bool *isidle, unsigned long *maxj);
 static void force_quiescent_state(struct rcu_state *rsp);
 static int rcu_pending(int cpu);
@@ -345,11 +369,11 @@ static struct rcu_node *rcu_get_root(struct rcu_state *rsp)
 static void rcu_eqs_enter_common(struct rcu_dynticks *rdtp, long long oldval,
                                bool user)
 {
-        trace_rcu_dyntick("Start", oldval, rdtp->dynticks_nesting);
+        trace_rcu_dyntick(TPS("Start"), oldval, rdtp->dynticks_nesting);
        if (!user && !is_idle_task(current)) {
                struct task_struct *idle = idle_task(smp_processor_id());
-                trace_rcu_dyntick("Error on entry: not idle task", oldval, 0);
+                trace_rcu_dyntick(TPS("Error on entry: not idle task"), oldval, 0);
                ftrace_dump(DUMP_ORIG);
                WARN_ONCE(1, "Current pid: %d comm: %s / Idle pid: %d comm: %s",
                          current->pid, current->comm,
@@ -411,6 +435,7 @@ void rcu_idle_enter(void)
        local_irq_save(flags);
        rcu_eqs_enter(false);
+        rcu_sysidle_enter(&__get_cpu_var(rcu_dynticks), 0);
        local_irq_restore(flags);
 }
 EXPORT_SYMBOL_GPL(rcu_idle_enter);
@@ -428,27 +453,6 @@ void rcu_user_enter(void)
 {
        rcu_eqs_enter(1);
 }
-/**
- * rcu_user_enter_after_irq - inform RCU that we are going to resume userspace
- * after the current irq returns.
- *
- * This is similar to rcu_user_enter() but in the context of a non-nesting
- * irq. After this call, RCU enters into idle mode when the interrupt
- * returns.
- */
-void rcu_user_enter_after_irq(void)
-{
-        unsigned long flags;
-        struct rcu_dynticks *rdtp;
-        local_irq_save(flags);
-        rdtp = &__get_cpu_var(rcu_dynticks);
-        /* Ensure this irq is interrupting a non-idle RCU state.  */
-        WARN_ON_ONCE(!(rdtp->dynticks_nesting & DYNTICK_TASK_MASK));
-        rdtp->dynticks_nesting = 1;
-        local_irq_restore(flags);
-}
 #endif /* CONFIG_RCU_USER_QS */
 /**
@@ -479,9 +483,10 @@ void rcu_irq_exit(void)
        rdtp->dynticks_nesting--;
        WARN_ON_ONCE(rdtp->dynticks_nesting < 0);
        if (rdtp->dynticks_nesting)
-                trace_rcu_dyntick("--=", oldval, rdtp->dynticks_nesting);
+                trace_rcu_dyntick(TPS("--="), oldval, rdtp->dynticks_nesting);
        else
                rcu_eqs_enter_common(rdtp, oldval, true);
+        rcu_sysidle_enter(rdtp, 1);
        local_irq_restore(flags);
 }
@@ -501,11 +506,11 @@ static void rcu_eqs_exit_common(struct rcu_dynticks *rdtp, long long oldval,
        smp_mb__after_atomic_inc();  /* See above. */
        WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
        rcu_cleanup_after_idle(smp_processor_id());
-        trace_rcu_dyntick("End", oldval, rdtp->dynticks_nesting);
+        trace_rcu_dyntick(TPS("End"), oldval, rdtp->dynticks_nesting);
        if (!user && !is_idle_task(current)) {
                struct task_struct *idle = idle_task(smp_processor_id());
-                trace_rcu_dyntick("Error on exit: not idle task",
+                trace_rcu_dyntick(TPS("Error on exit: not idle task"),
                                  oldval, rdtp->dynticks_nesting);
                ftrace_dump(DUMP_ORIG);
                WARN_ONCE(1, "Current pid: %d comm: %s / Idle pid: %d comm: %s",
@@ -550,6 +555,7 @@ void rcu_idle_exit(void)
        local_irq_save(flags);
        rcu_eqs_exit(false);
+        rcu_sysidle_exit(&__get_cpu_var(rcu_dynticks), 0);
        local_irq_restore(flags);
 }
 EXPORT_SYMBOL_GPL(rcu_idle_exit);
@@ -565,28 +571,6 @@ void rcu_user_exit(void)
 {
        rcu_eqs_exit(1);
 }
-/**
- * rcu_user_exit_after_irq - inform RCU that we won't resume to userspace
- * idle mode after the current non-nesting irq returns.
- *
- * This is similar to rcu_user_exit() but in the context of an irq.
- * This is called when the irq has interrupted a userspace RCU idle mode
- * context. When the current non-nesting interrupt returns after this call,
- * the CPU won't restore the RCU idle mode.
- */
-void rcu_user_exit_after_irq(void)
-{
-        unsigned long flags;
-        struct rcu_dynticks *rdtp;
-        local_irq_save(flags);
-        rdtp = &__get_cpu_var(rcu_dynticks);
-        /* Ensure we are interrupting an RCU idle mode. */
-        WARN_ON_ONCE(rdtp->dynticks_nesting & DYNTICK_TASK_NEST_MASK);
-        rdtp->dynticks_nesting += DYNTICK_TASK_EXIT_IDLE;
-        local_irq_restore(flags);
-}
 #endif /* CONFIG_RCU_USER_QS */
 /**
@@ -620,9 +604,10 @@ void rcu_irq_enter(void)
        rdtp->dynticks_nesting++;
        WARN_ON_ONCE(rdtp->dynticks_nesting == 0);
        if (oldval)
-                trace_rcu_dyntick("++=", oldval, rdtp->dynticks_nesting);
+                trace_rcu_dyntick(TPS("++="), oldval, rdtp->dynticks_nesting);
        else
                rcu_eqs_exit_common(rdtp, oldval, true);
+        rcu_sysidle_exit(rdtp, 1);
        local_irq_restore(flags);
 }
@@ -746,9 +731,11 @@ static int rcu_is_cpu_rrupt_from_idle(void)
 * credit them with an implicit quiescent state.  Return 1 if this CPU
 * is in dynticks idle mode, which is an extended quiescent state.
 */
-static int dyntick_save_progress_counter(struct rcu_data *rdp)
+static int dyntick_save_progress_counter(struct rcu_data *rdp,
+                                         bool *isidle, unsigned long *maxj)
 {
        rdp->dynticks_snap = atomic_add_return(0, &rdp->dynticks->dynticks);
+        rcu_sysidle_check_cpu(rdp, isidle, maxj);
        return (rdp->dynticks_snap & 0x1) == 0;
 }
@@ -758,7 +745,8 @@ static int dyntick_save_progress_counter(struct rcu_data *rdp)
 * idle state since the last call to dyntick_save_progress_counter()
 * for this same CPU, or by virtue of having been offline.
 */
-static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)
+static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
+                                    bool *isidle, unsigned long *maxj)
 {
        unsigned int curr;
        unsigned int snap;
@@ -775,7 +763,7 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)
         * of the current RCU grace period.
         */
        if ((curr & 0x1) == 0 || UINT_CMP_GE(curr, snap + 2)) {
-                trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, "dti");
+                trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti"));
                rdp->dynticks_fqs++;
                return 1;
        }
@@ -795,7 +783,7 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)
                return 0;  /* Grace period is not old enough. */
        barrier();
        if (cpu_is_offline(rdp->cpu)) {
-                trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, "ofl");
+                trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("ofl"));
                rdp->offline_fqs++;
                return 1;
        }
@@ -1032,7 +1020,7 @@ static unsigned long rcu_cbs_completed(struct rcu_state *rsp,
 * rcu_nocb_wait_gp().
 */
 static void trace_rcu_future_gp(struct rcu_node *rnp, struct rcu_data *rdp,
-                                unsigned long c, char *s)
+                                unsigned long c, const char *s)
 {
        trace_rcu_future_grace_period(rdp->rsp->name, rnp->gpnum,
                                      rnp->completed, c, rnp->level,
@@ -1058,9 +1046,9 @@ rcu_start_future_gp(struct rcu_node *rnp, struct rcu_data *rdp)
         * grace period is already marked as needed, return to the caller.
         */
        c = rcu_cbs_completed(rdp->rsp, rnp);
-        trace_rcu_future_gp(rnp, rdp, c, "Startleaf");
+        trace_rcu_future_gp(rnp, rdp, c, TPS("Startleaf"));
        if (rnp->need_future_gp[c & 0x1]) {
-                trace_rcu_future_gp(rnp, rdp, c, "Prestartleaf");
+                trace_rcu_future_gp(rnp, rdp, c, TPS("Prestartleaf"));
                return c;
        }
@@ -1074,7 +1062,7 @@ rcu_start_future_gp(struct rcu_node *rnp, struct rcu_data *rdp)
        if (rnp->gpnum != rnp->completed ||
            ACCESS_ONCE(rnp->gpnum) != ACCESS_ONCE(rnp->completed)) {
                rnp->need_future_gp[c & 0x1]++;
-                trace_rcu_future_gp(rnp, rdp, c, "Startedleaf");
+                trace_rcu_future_gp(rnp, rdp, c, TPS("Startedleaf"));
                return c;
        }
@@ -1102,7 +1090,7 @@ rcu_start_future_gp(struct rcu_node *rnp, struct rcu_data *rdp)
         * recorded, trace and leave.
         */
        if (rnp_root->need_future_gp[c & 0x1]) {
-                trace_rcu_future_gp(rnp, rdp, c, "Prestartedroot");
+                trace_rcu_future_gp(rnp, rdp, c, TPS("Prestartedroot"));
                goto unlock_out;
        }
@@ -1111,9 +1099,9 @@ rcu_start_future_gp(struct rcu_node *rnp, struct rcu_data *rdp)
        /* If a grace period is not already in progress, start one. */
        if (rnp_root->gpnum != rnp_root->completed) {
-                trace_rcu_future_gp(rnp, rdp, c, "Startedleafroot");
+                trace_rcu_future_gp(rnp, rdp, c, TPS("Startedleafroot"));
        } else {
-                trace_rcu_future_gp(rnp, rdp, c, "Startedroot");
+                trace_rcu_future_gp(rnp, rdp, c, TPS("Startedroot"));
                rcu_start_gp_advanced(rdp->rsp, rnp_root, rdp);
        }
 unlock_out:
@@ -1137,7 +1125,8 @@ static int rcu_future_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp)
        rcu_nocb_gp_cleanup(rsp, rnp);
        rnp->need_future_gp[c & 0x1] = 0;
        needmore = rnp->need_future_gp[(c + 1) & 0x1];
-        trace_rcu_future_gp(rnp, rdp, c, needmore ? "CleanupMore" : "Cleanup");
+        trace_rcu_future_gp(rnp, rdp, c,
+                            needmore ? TPS("CleanupMore") : TPS("Cleanup"));
        return needmore;
 }
@@ -1205,9 +1194,9 @@ static void rcu_accelerate_cbs(struct rcu_state *rsp, struct rcu_node *rnp,
        /* Trace depending on how much we were able to accelerate. */
        if (!*rdp->nxttail[RCU_WAIT_TAIL])
-                trace_rcu_grace_period(rsp->name, rdp->gpnum, "AccWaitCB");
+                trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("AccWaitCB"));
        else
-                trace_rcu_grace_period(rsp->name, rdp->gpnum, "AccReadyCB");
+                trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("AccReadyCB"));
 }
 /*
@@ -1273,7 +1262,7 @@ static void __note_gp_changes(struct rcu_state *rsp, struct rcu_node *rnp, struc
                /* Remember that we saw this grace-period completion. */
                rdp->completed = rnp->completed;
-                trace_rcu_grace_period(rsp->name, rdp->gpnum, "cpuend");
+                trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuend"));
        }
        if (rdp->gpnum != rnp->gpnum) {
@@ -1283,7 +1272,7 @@ static void __note_gp_changes(struct rcu_state *rsp, struct rcu_node *rnp, struc
                 * go looking for one.
                 */
                rdp->gpnum = rnp->gpnum;
-                trace_rcu_grace_period(rsp->name, rdp->gpnum, "cpustart");
+                trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpustart"));
                rdp->passed_quiesce = 0;
                rdp->qs_pending = !!(rnp->qsmask & rdp->grpmask);
                zero_cpu_stall_ticks(rdp);
@@ -1315,6 +1304,7 @@ static int rcu_gp_init(struct rcu_state *rsp)
        struct rcu_data *rdp;
        struct rcu_node *rnp = rcu_get_root(rsp);
+        rcu_bind_gp_kthread();
        raw_spin_lock_irq(&rnp->lock);
        rsp->gp_flags = 0; /* Clear all flags: New grace period. */
@@ -1326,7 +1316,7 @@ static int rcu_gp_init(struct rcu_state *rsp)
        /* Advance to a new grace period and initialize state. */
        rsp->gpnum++;
-        trace_rcu_grace_period(rsp->name, rsp->gpnum, "start");
+        trace_rcu_grace_period(rsp->name, rsp->gpnum, TPS("start"));
        record_gp_stall_check_time(rsp);
        raw_spin_unlock_irq(&rnp->lock);
@@ -1379,16 +1369,25 @@ static int rcu_gp_init(struct rcu_state *rsp)
 int rcu_gp_fqs(struct rcu_state *rsp, int fqs_state_in)
 {
        int fqs_state = fqs_state_in;
+        bool isidle = false;
+        unsigned long maxj;
        struct rcu_node *rnp = rcu_get_root(rsp);
        rsp->n_force_qs++;
        if (fqs_state == RCU_SAVE_DYNTICK) {
                /* Collect dyntick-idle snapshots. */
-                force_qs_rnp(rsp, dyntick_save_progress_counter);
+                if (is_sysidle_rcu_state(rsp)) {
+                        isidle = 1;
+                        maxj = jiffies - ULONG_MAX / 4;
+                }
+                force_qs_rnp(rsp, dyntick_save_progress_counter,
+                             &isidle, &maxj);
+                rcu_sysidle_report_gp(rsp, isidle, maxj);
                fqs_state = RCU_FORCE_QS;
        } else {
                /* Handle dyntick-idle and offline CPUs. */
-                force_qs_rnp(rsp, rcu_implicit_dynticks_qs);
+                isidle = 0;
+                force_qs_rnp(rsp, rcu_implicit_dynticks_qs, &isidle, &maxj);
        }
        /* Clear flag to prevent immediate re-entry. */
        if (ACCESS_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) {
@@ -1448,7 +1447,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
        rcu_nocb_gp_set(rnp, nocb);
        rsp->completed = rsp->gpnum; /* Declare grace period done. */
-        trace_rcu_grace_period(rsp->name, rsp->completed, "end");
+        trace_rcu_grace_period(rsp->name, rsp->completed, TPS("end"));
        rsp->fqs_state = RCU_GP_IDLE;
        rdp = this_cpu_ptr(rsp->rda);
        rcu_advance_cbs(rsp, rnp, rdp);  /* Reduce false positives below. */
@@ -1558,10 +1557,12 @@ rcu_start_gp_advanced(struct rcu_state *rsp, struct rcu_node *rnp,
        /*
         * We can't do wakeups while holding the rnp->lock, as that
-         * could cause possible deadlocks with the rq->lock. Deter
+         * could cause possible deadlocks with the rq->lock. Defer
-         * the wakeup to interrupt context.
+         * the wakeup to interrupt context.  And don't bother waking
+         * up the running kthread.
         */
-        irq_work_queue(&rsp->wakeup_work);
+        if (current != rsp->gp_kthread)
+                irq_work_queue(&rsp->wakeup_work);
 }
 /*
@@ -1857,7 +1858,7 @@ static void rcu_cleanup_dying_cpu(struct rcu_state *rsp)
        RCU_TRACE(mask = rdp->grpmask);
        trace_rcu_grace_period(rsp->name,
                               rnp->gpnum + 1 - !!(rnp->qsmask & mask),
-                               "cpuofl");
+                               TPS("cpuofl"));
 }
 /*
@@ -2044,7 +2045,7 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp)
 */
 void rcu_check_callbacks(int cpu, int user)
 {
-        trace_rcu_utilization("Start scheduler-tick");
+        trace_rcu_utilization(TPS("Start scheduler-tick"));
        increment_cpu_stall_ticks();
        if (user || rcu_is_cpu_rrupt_from_idle()) {
@@ -2077,7 +2078,7 @@ void rcu_check_callbacks(int cpu, int user)
        rcu_preempt_check_callbacks(cpu);
        if (rcu_pending(cpu))
                invoke_rcu_core();
-        trace_rcu_utilization("End scheduler-tick");
+        trace_rcu_utilization(TPS("End scheduler-tick"));
 }
 /*
@@ -2087,7 +2088,10 @@ void rcu_check_callbacks(int cpu, int user)
 *
 * The caller must have suppressed start of new grace periods.
 */
-static void force_qs_rnp(struct rcu_state *rsp, int (*f)(struct rcu_data *))
+static void force_qs_rnp(struct rcu_state *rsp,
+                         int (*f)(struct rcu_data *rsp, bool *isidle,
+                                  unsigned long *maxj),
+                         bool *isidle, unsigned long *maxj)
 {
        unsigned long bit;
        int cpu;
@@ -2110,9 +2114,12 @@ static void force_qs_rnp(struct rcu_state *rsp, int (*f)(struct rcu_data *))
                cpu = rnp->grplo;
                bit = 1;
                for (; cpu <= rnp->grphi; cpu++, bit <<= 1) {
-                        if ((rnp->qsmask & bit) != 0 &&
+                        if ((rnp->qsmask & bit) != 0) {
-                            f(per_cpu_ptr(rsp->rda, cpu)))
+                                if ((rnp->qsmaskinit & bit) != 0)
-                                mask |= bit;
+                                        *isidle = 0;
+                                if (f(per_cpu_ptr(rsp->rda, cpu), isidle, maxj))
+                                        mask |= bit;
+                        }
                }
                if (mask != 0) {
@@ -2208,10 +2215,10 @@ static void rcu_process_callbacks(struct softirq_action *unused)
        if (cpu_is_offline(smp_processor_id()))
                return;
-        trace_rcu_utilization("Start RCU core");
+        trace_rcu_utilization(TPS("Start RCU core"));
        for_each_rcu_flavor(rsp)
                __rcu_process_callbacks(rsp);
-        trace_rcu_utilization("End RCU core");
+        trace_rcu_utilization(TPS("End RCU core"));
 }
 /*
@@ -2287,6 +2294,13 @@ static void __call_rcu_core(struct rcu_state *rsp, struct rcu_data *rdp,
 }
 /*
+ * RCU callback function to leak a callback.
+ */
+static void rcu_leak_callback(struct rcu_head *rhp)
+{
+}
+/*
 * Helper function for call_rcu() and friends.  The cpu argument will
 * normally be -1, indicating "currently running CPU".  It may specify
 * a CPU only if that CPU is a no-CBs CPU.  Currently, only _rcu_barrier()
@@ -2300,7 +2314,12 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
        struct rcu_data *rdp;
        WARN_ON_ONCE((unsigned long)head & 0x3); /* Misaligned rcu_head! */
-        debug_rcu_head_queue(head);
+        if (debug_rcu_head_queue(head)) {
+                /* Probable double call_rcu(), so leak the callback. */
+                ACCESS_ONCE(head->func) = rcu_leak_callback;
+                WARN_ONCE(1, "__call_rcu(): Leaked duplicate callback\n");
+                return;
+        }
        head->func = func;
        head->next = NULL;
@@ -2720,7 +2739,7 @@ static int rcu_cpu_has_callbacks(int cpu, bool *all_lazy)
 * Helper function for _rcu_barrier() tracing.  If tracing is disabled,
 * the compiler is expected to optimize this away.
 */
-static void _rcu_barrier_trace(struct rcu_state *rsp, char *s,
+static void _rcu_barrier_trace(struct rcu_state *rsp, const char *s,
                               int cpu, unsigned long done)
 {
        trace_rcu_barrier(rsp->name, s, cpu,
@@ -2785,9 +2804,20 @@ static void _rcu_barrier(struct rcu_state *rsp)
         * transition.  The "if" expression below therefore rounds the old
         * value up to the next even number and adds two before comparing.
         */
-        snap_done = ACCESS_ONCE(rsp->n_barrier_done);
+        snap_done = rsp->n_barrier_done;
        _rcu_barrier_trace(rsp, "Check", -1, snap_done);
-        if (ULONG_CMP_GE(snap_done, ((snap + 1) & ~0x1) + 2)) {
+        /*
+         * If the value in snap is odd, we needed to wait for the current
+         * rcu_barrier() to complete, then wait for the next one, in other
+         * words, we need the value of snap_done to be three larger than
+         * the value of snap.  On the other hand, if the value in snap is
+         * even, we only had to wait for the next rcu_barrier() to complete,
+         * in other words, we need the value of snap_done to be only two
+         * greater than the value of snap.  The "(snap + 3) & ~0x1" computes
+         * this for us (thank you, Linus!).
+         */
+        if (ULONG_CMP_GE(snap_done, (snap + 3) & ~0x1)) {
                _rcu_barrier_trace(rsp, "EarlyExit", -1, snap_done);
                smp_mb(); /* caller's subsequent code after above check. */
                mutex_unlock(&rsp->barrier_mutex);
@@ -2910,7 +2940,7 @@ rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp)
 * can accept some slop in the rsp->completed access due to the fact
 * that this CPU cannot possibly have any RCU callbacks in flight yet.
 */
-static void __cpuinit
+static void
 rcu_init_percpu_data(int cpu, struct rcu_state *rsp, int preemptible)
 {
        unsigned long flags;
@@ -2930,6 +2960,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp, int preemptible)
        rdp->blimit = blimit;
        init_callback_list(rdp);  /* Re-enable callbacks on this CPU. */
        rdp->dynticks->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
+        rcu_sysidle_init_percpu_data(rdp->dynticks);
        atomic_set(&rdp->dynticks->dynticks,
                   (atomic_read(&rdp->dynticks->dynticks) & ~0x1) + 1);
        raw_spin_unlock(&rnp->lock);            /* irqs remain disabled. */
@@ -2952,7 +2983,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp, int preemptible)
                        rdp->completed = rnp->completed;
                        rdp->passed_quiesce = 0;
                        rdp->qs_pending = 0;
-                        trace_rcu_grace_period(rsp->name, rdp->gpnum, "cpuonl");
+                        trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuonl"));
                }
                raw_spin_unlock(&rnp->lock); /* irqs already disabled. */
                rnp = rnp->parent;
@@ -2962,7 +2993,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp, int preemptible)
        mutex_unlock(&rsp->onoff_mutex);
 }
-static void __cpuinit rcu_prepare_cpu(int cpu)
+static void rcu_prepare_cpu(int cpu)
 {
        struct rcu_state *rsp;
@@ -2974,7 +3005,7 @@ static void __cpuinit rcu_prepare_cpu(int cpu)
 /*
 * Handle CPU online/offline notification events.
 */
-static int __cpuinit rcu_cpu_notify(struct notifier_block *self,
+static int rcu_cpu_notify(struct notifier_block *self,
                                    unsigned long action, void *hcpu)
 {
        long cpu = (long)hcpu;
@@ -2982,7 +3013,7 @@ static int __cpuinit rcu_cpu_notify(struct notifier_block *self,
        struct rcu_node *rnp = rdp->mynode;
        struct rcu_state *rsp;
-        trace_rcu_utilization("Start CPU hotplug");
+        trace_rcu_utilization(TPS("Start CPU hotplug"));
        switch (action) {
        case CPU_UP_PREPARE:
        case CPU_UP_PREPARE_FROZEN:
@@ -3011,7 +3042,26 @@ static int __cpuinit rcu_cpu_notify(struct notifier_block *self,
        default:
                break;
        }
-        trace_rcu_utilization("End CPU hotplug");
+        trace_rcu_utilization(TPS("End CPU hotplug"));
+        return NOTIFY_OK;
+}
+static int rcu_pm_notify(struct notifier_block *self,
+                         unsigned long action, void *hcpu)
+{
+        switch (action) {
+        case PM_HIBERNATION_PREPARE:
+        case PM_SUSPEND_PREPARE:
+                if (nr_cpu_ids <= 256) /* Expediting bad for large systems. */
+                        rcu_expedited = 1;
+                break;
+        case PM_POST_HIBERNATION:
+        case PM_POST_SUSPEND:
+                rcu_expedited = 0;
+                break;
+        default:
+                break;
+        }
        return NOTIFY_OK;
 }
@@ -3256,6 +3306,7 @@ void __init rcu_init(void)
         * or the scheduler are operational.
         */
        cpu_notifier(rcu_cpu_notify, 0);
+        pm_notifier(rcu_pm_notify, 0);
        for_each_online_cpu(cpu)
                rcu_cpu_notify(NULL, CPU_UP_PREPARE, (void *)(long)cpu);
 }
diff --git a/kernel/rcutree.h b/kernel/rcutree.h
index 4a39d364493c..5f97eab602cd 100644
--- a/kernel/rcutree.h
+++ b/kernel/rcutree.h
@@ -88,6 +88,14 @@ struct rcu_dynticks {
                                    /* Process level is worth LLONG_MAX/2. */
        int dynticks_nmi_nesting;   /* Track NMI nesting level. */
        atomic_t dynticks;          /* Even value for idle, else odd. */
+#ifdef CONFIG_NO_HZ_FULL_SYSIDLE
+        long long dynticks_idle_nesting;
+                                    /* irq/process nesting level from idle. */
+        atomic_t dynticks_idle;     /* Even value for idle, else odd. */
+                                    /*  "Idle" excludes userspace execution. */
+        unsigned long dynticks_idle_jiffies;
+                                    /* End of last non-NMI non-idle period. */
+#endif /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
 #ifdef CONFIG_RCU_FAST_NO_HZ
        bool all_lazy;              /* Are all CPU's CBs lazy? */
        unsigned long nonlazy_posted;
@@ -445,7 +453,7 @@ struct rcu_state {
                                                /*  for CPU stalls. */
        unsigned long gp_max;                   /* Maximum GP duration in */
                                                /*  jiffies. */
-        char *name;                             /* Name of structure. */
+        const char *name;                       /* Name of structure. */
        char abbr;                              /* Abbreviated name. */
        struct list_head flavors;               /* List of RCU flavors. */
        struct irq_work wakeup_work;            /* Postponed wakeups */
@@ -521,10 +529,10 @@ static void invoke_rcu_callbacks_kthread(void);
 static bool rcu_is_callbacks_kthread(void);
 #ifdef CONFIG_RCU_BOOST
 static void rcu_preempt_do_callbacks(void);
-static int __cpuinit rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
+static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
                                                 struct rcu_node *rnp);
 #endif /* #ifdef CONFIG_RCU_BOOST */
-static void __cpuinit rcu_prepare_kthreads(int cpu);
+static void rcu_prepare_kthreads(int cpu);
 static void rcu_cleanup_after_idle(int cpu);
 static void rcu_prepare_for_idle(int cpu);
 static void rcu_idle_count_callbacks_posted(void);
@@ -545,6 +553,15 @@ static void rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp);
 static void rcu_spawn_nocb_kthreads(struct rcu_state *rsp);
 static void rcu_kick_nohz_cpu(int cpu);
 static bool init_nocb_callback_list(struct rcu_data *rdp);
+static void rcu_sysidle_enter(struct rcu_dynticks *rdtp, int irq);
+static void rcu_sysidle_exit(struct rcu_dynticks *rdtp, int irq);
+static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
+                                  unsigned long *maxj);
+static bool is_sysidle_rcu_state(struct rcu_state *rsp);
+static void rcu_sysidle_report_gp(struct rcu_state *rsp, int isidle,
+                                  unsigned long maxj);
+static void rcu_bind_gp_kthread(void);
+static void rcu_sysidle_init_percpu_data(struct rcu_dynticks *rdtp);
 #endif /* #ifndef RCU_TREE_NONCORE */
diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h
index 63098a59216e..130c97b027f2 100644
--- a/kernel/rcutree_plugin.h
+++ b/kernel/rcutree_plugin.h
@@ -28,7 +28,7 @@
 #include <linux/gfp.h>
 #include <linux/oom.h>
 #include <linux/smpboot.h>
-#include <linux/tick.h>
+#include "time/tick-internal.h"
 #define RCU_KTHREAD_PRIO 1
@@ -110,9 +110,7 @@ static void __init rcu_bootup_announce_oddness(void)
 #ifdef CONFIG_TREE_PREEMPT_RCU
-struct rcu_state rcu_preempt_state =
+RCU_STATE_INITIALIZER(rcu_preempt, 'p', call_rcu);
-        RCU_STATE_INITIALIZER(rcu_preempt, 'p', call_rcu);
-DEFINE_PER_CPU(struct rcu_data, rcu_preempt_data);
 static struct rcu_state *rcu_state = &rcu_preempt_state;
 static int rcu_preempted_readers_exp(struct rcu_node *rnp);
@@ -169,7 +167,7 @@ static void rcu_preempt_qs(int cpu)
        struct rcu_data *rdp = &per_cpu(rcu_preempt_data, cpu);
        if (rdp->passed_quiesce == 0)
-                trace_rcu_grace_period("rcu_preempt", rdp->gpnum, "cpuqs");
+                trace_rcu_grace_period(TPS("rcu_preempt"), rdp->gpnum, TPS("cpuqs"));
        rdp->passed_quiesce = 1;
        current->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_NEED_QS;
 }
@@ -388,7 +386,7 @@ void rcu_read_unlock_special(struct task_struct *t)
                np = rcu_next_node_entry(t, rnp);
                list_del_init(&t->rcu_node_entry);
                t->rcu_blocked_node = NULL;
-                trace_rcu_unlock_preempted_task("rcu_preempt",
+                trace_rcu_unlock_preempted_task(TPS("rcu_preempt"),
                                                rnp->gpnum, t->pid);
                if (&t->rcu_node_entry == rnp->gp_tasks)
                        rnp->gp_tasks = np;
@@ -412,7 +410,7 @@ void rcu_read_unlock_special(struct task_struct *t)
                 */
                empty_exp_now = !rcu_preempted_readers_exp(rnp);
                if (!empty && !rcu_preempt_blocked_readers_cgp(rnp)) {
-                        trace_rcu_quiescent_state_report("preempt_rcu",
+                        trace_rcu_quiescent_state_report(TPS("preempt_rcu"),
                                                         rnp->gpnum,
                                                         0, rnp->qsmask,
                                                         rnp->level,
@@ -1250,12 +1248,12 @@ static int rcu_boost_kthread(void *arg)
        int spincnt = 0;
        int more2boost;
-        trace_rcu_utilization("Start boost kthread@init");
+        trace_rcu_utilization(TPS("Start boost kthread@init"));
        for (;;) {
                rnp->boost_kthread_status = RCU_KTHREAD_WAITING;
-                trace_rcu_utilization("End boost kthread@rcu_wait");
+                trace_rcu_utilization(TPS("End boost kthread@rcu_wait"));
                rcu_wait(rnp->boost_tasks || rnp->exp_tasks);
-                trace_rcu_utilization("Start boost kthread@rcu_wait");
+                trace_rcu_utilization(TPS("Start boost kthread@rcu_wait"));
                rnp->boost_kthread_status = RCU_KTHREAD_RUNNING;
                more2boost = rcu_boost(rnp);
                if (more2boost)
@@ -1264,14 +1262,14 @@ static int rcu_boost_kthread(void *arg)
                        spincnt = 0;
                if (spincnt > 10) {
                        rnp->boost_kthread_status = RCU_KTHREAD_YIELDING;
-                        trace_rcu_utilization("End boost kthread@rcu_yield");
+                        trace_rcu_utilization(TPS("End boost kthread@rcu_yield"));
                        schedule_timeout_interruptible(2);
-                        trace_rcu_utilization("Start boost kthread@rcu_yield");
+                        trace_rcu_utilization(TPS("Start boost kthread@rcu_yield"));
                        spincnt = 0;
                }
        }
        /* NOTREACHED */
-        trace_rcu_utilization("End boost kthread@notreached");
+        trace_rcu_utilization(TPS("End boost kthread@notreached"));
        return 0;
 }
@@ -1352,7 +1350,7 @@ static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
 * already exist.  We only create this kthread for preemptible RCU.
 * Returns zero if all is well, a negated errno otherwise.
 */
-static int __cpuinit rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
+static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
                                                 struct rcu_node *rnp)
 {
        int rnp_index = rnp - &rsp->node[0];
@@ -1419,7 +1417,7 @@ static void rcu_cpu_kthread(unsigned int cpu)
        int spincnt;
        for (spincnt = 0; spincnt < 10; spincnt++) {
-                trace_rcu_utilization("Start CPU kthread@rcu_wait");
+                trace_rcu_utilization(TPS("Start CPU kthread@rcu_wait"));
                local_bh_disable();
                *statusp = RCU_KTHREAD_RUNNING;
                this_cpu_inc(rcu_cpu_kthread_loops);
@@ -1431,15 +1429,15 @@ static void rcu_cpu_kthread(unsigned int cpu)
                        rcu_kthread_do_work();
                local_bh_enable();
                if (*workp == 0) {
-                        trace_rcu_utilization("End CPU kthread@rcu_wait");
+                        trace_rcu_utilization(TPS("End CPU kthread@rcu_wait"));
                        *statusp = RCU_KTHREAD_WAITING;
                        return;
                }
        }
        *statusp = RCU_KTHREAD_YIELDING;
-        trace_rcu_utilization("Start CPU kthread@rcu_yield");
+        trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield"));
        schedule_timeout_interruptible(2);
-        trace_rcu_utilization("End CPU kthread@rcu_yield");
+        trace_rcu_utilization(TPS("End CPU kthread@rcu_yield"));
        *statusp = RCU_KTHREAD_WAITING;
 }
@@ -1507,7 +1505,7 @@ static int __init rcu_spawn_kthreads(void)
 }
 early_initcall(rcu_spawn_kthreads);
-static void __cpuinit rcu_prepare_kthreads(int cpu)
+static void rcu_prepare_kthreads(int cpu)
 {
        struct rcu_data *rdp = per_cpu_ptr(rcu_state->rda, cpu);
        struct rcu_node *rnp = rdp->mynode;
@@ -1549,7 +1547,7 @@ static int __init rcu_scheduler_really_started(void)
 }
 early_initcall(rcu_scheduler_really_started);
-static void __cpuinit rcu_prepare_kthreads(int cpu)
+static void rcu_prepare_kthreads(int cpu)
 {
 }
@@ -2202,7 +2200,7 @@ static void rcu_nocb_wait_gp(struct rcu_data *rdp)
         * Wait for the grace period.  Do so interruptibly to avoid messing
         * up the load average.
         */
-        trace_rcu_future_gp(rnp, rdp, c, "StartWait");
+        trace_rcu_future_gp(rnp, rdp, c, TPS("StartWait"));
        for (;;) {
                wait_event_interruptible(
                        rnp->nocb_gp_wq[c & 0x1],
@@ -2210,9 +2208,9 @@ static void rcu_nocb_wait_gp(struct rcu_data *rdp)
                if (likely(d))
                        break;
                flush_signals(current);
-                trace_rcu_future_gp(rnp, rdp, c, "ResumeWait");
+                trace_rcu_future_gp(rnp, rdp, c, TPS("ResumeWait"));
        }
-        trace_rcu_future_gp(rnp, rdp, c, "EndWait");
+        trace_rcu_future_gp(rnp, rdp, c, TPS("EndWait"));
        smp_mb(); /* Ensure that CB invocation happens after GP end. */
 }
@@ -2375,3 +2373,425 @@ static void rcu_kick_nohz_cpu(int cpu)
                smp_send_reschedule(cpu);
 #endif /* #ifdef CONFIG_NO_HZ_FULL */
 }
+#ifdef CONFIG_NO_HZ_FULL_SYSIDLE
+/*
+ * Define RCU flavor that holds sysidle state.  This needs to be the
+ * most active flavor of RCU.
+ */
+#ifdef CONFIG_PREEMPT_RCU
+static struct rcu_state *rcu_sysidle_state = &rcu_preempt_state;
+#else /* #ifdef CONFIG_PREEMPT_RCU */
+static struct rcu_state *rcu_sysidle_state = &rcu_sched_state;
+#endif /* #else #ifdef CONFIG_PREEMPT_RCU */
+static int full_sysidle_state;          /* Current system-idle state. */
+#define RCU_SYSIDLE_NOT         0       /* Some CPU is not idle. */
+#define RCU_SYSIDLE_SHORT       1       /* All CPUs idle for brief period. */
+#define RCU_SYSIDLE_LONG        2       /* All CPUs idle for long enough. */
+#define RCU_SYSIDLE_FULL        3       /* All CPUs idle, ready for sysidle. */
+#define RCU_SYSIDLE_FULL_NOTED  4       /* Actually entered sysidle state. */
+/*
+ * Invoked to note exit from irq or task transition to idle.  Note that
+ * usermode execution does -not- count as idle here!  After all, we want
+ * to detect full-system idle states, not RCU quiescent states and grace
+ * periods.  The caller must have disabled interrupts.
+ */
+static void rcu_sysidle_enter(struct rcu_dynticks *rdtp, int irq)
+{
+        unsigned long j;
+        /* Adjust nesting, check for fully idle. */
+        if (irq) {
+                rdtp->dynticks_idle_nesting--;
+                WARN_ON_ONCE(rdtp->dynticks_idle_nesting < 0);
+                if (rdtp->dynticks_idle_nesting != 0)
+                        return;  /* Still not fully idle. */
+        } else {
+                if ((rdtp->dynticks_idle_nesting & DYNTICK_TASK_NEST_MASK) ==
+                    DYNTICK_TASK_NEST_VALUE) {
+                        rdtp->dynticks_idle_nesting = 0;
+                } else {
+                        rdtp->dynticks_idle_nesting -= DYNTICK_TASK_NEST_VALUE;
+                        WARN_ON_ONCE(rdtp->dynticks_idle_nesting < 0);
+                        return;  /* Still not fully idle. */
+                }
+        }
+        /* Record start of fully idle period. */
+        j = jiffies;
+        ACCESS_ONCE(rdtp->dynticks_idle_jiffies) = j;
+        smp_mb__before_atomic_inc();
+        atomic_inc(&rdtp->dynticks_idle);
+        smp_mb__after_atomic_inc();
+        WARN_ON_ONCE(atomic_read(&rdtp->dynticks_idle) & 0x1);
+}
+/*
+ * Unconditionally force exit from full system-idle state.  This is
+ * invoked when a normal CPU exits idle, but must be called separately
+ * for the timekeeping CPU (tick_do_timer_cpu).  The reason for this
+ * is that the timekeeping CPU is permitted to take scheduling-clock
+ * interrupts while the system is in system-idle state, and of course
+ * rcu_sysidle_exit() has no way of distinguishing a scheduling-clock
+ * interrupt from any other type of interrupt.
+ */
+void rcu_sysidle_force_exit(void)
+{
+        int oldstate = ACCESS_ONCE(full_sysidle_state);
+        int newoldstate;
+        /*
+         * Each pass through the following loop attempts to exit full
+         * system-idle state.  If contention proves to be a problem,
+         * a trylock-based contention tree could be used here.
+         */
+        while (oldstate > RCU_SYSIDLE_SHORT) {
+                newoldstate = cmpxchg(&full_sysidle_state,
+                                      oldstate, RCU_SYSIDLE_NOT);
+                if (oldstate == newoldstate &&
+                    oldstate == RCU_SYSIDLE_FULL_NOTED) {
+                        rcu_kick_nohz_cpu(tick_do_timer_cpu);
+                        return; /* We cleared it, done! */
+                }
+                oldstate = newoldstate;
+        }
+        smp_mb(); /* Order initial oldstate fetch vs. later non-idle work. */
+}
+/*
+ * Invoked to note entry to irq or task transition from idle.  Note that
+ * usermode execution does -not- count as idle here!  The caller must
+ * have disabled interrupts.
+ */
+static void rcu_sysidle_exit(struct rcu_dynticks *rdtp, int irq)
+{
+        /* Adjust nesting, check for already non-idle. */
+        if (irq) {
+                rdtp->dynticks_idle_nesting++;
+                WARN_ON_ONCE(rdtp->dynticks_idle_nesting <= 0);
+                if (rdtp->dynticks_idle_nesting != 1)
+                        return; /* Already non-idle. */
+        } else {
+                /*
+                 * Allow for irq misnesting.  Yes, it really is possible
+                 * to enter an irq handler then never leave it, and maybe
+                 * also vice versa.  Handle both possibilities.
+                 */
+                if (rdtp->dynticks_idle_nesting & DYNTICK_TASK_NEST_MASK) {
+                        rdtp->dynticks_idle_nesting += DYNTICK_TASK_NEST_VALUE;
+                        WARN_ON_ONCE(rdtp->dynticks_idle_nesting <= 0);
+                        return; /* Already non-idle. */
+                } else {
+                        rdtp->dynticks_idle_nesting = DYNTICK_TASK_EXIT_IDLE;
+                }
+        }
+        /* Record end of idle period. */
+        smp_mb__before_atomic_inc();
+        atomic_inc(&rdtp->dynticks_idle);
+        smp_mb__after_atomic_inc();
+        WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks_idle) & 0x1));
+        /*
+         * If we are the timekeeping CPU, we are permitted to be non-idle
+         * during a system-idle state.  This must be the case, because
+         * the timekeeping CPU has to take scheduling-clock interrupts
+         * during the time that the system is transitioning to full
+         * system-idle state.  This means that the timekeeping CPU must
+         * invoke rcu_sysidle_force_exit() directly if it does anything
+         * more than take a scheduling-clock interrupt.
+         */
+        if (smp_processor_id() == tick_do_timer_cpu)
+                return;
+        /* Update system-idle state: We are clearly no longer fully idle! */
+        rcu_sysidle_force_exit();
+}
+/*
+ * Check to see if the current CPU is idle.  Note that usermode execution
+ * does not count as idle.  The caller must have disabled interrupts.
+ */
+static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
+                                  unsigned long *maxj)
+{
+        int cur;
+        unsigned long j;
+        struct rcu_dynticks *rdtp = rdp->dynticks;
+        /*
+         * If some other CPU has already reported non-idle, if this is
+         * not the flavor of RCU that tracks sysidle state, or if this
+         * is an offline or the timekeeping CPU, nothing to do.
+         */
+        if (!*isidle || rdp->rsp != rcu_sysidle_state ||
+            cpu_is_offline(rdp->cpu) || rdp->cpu == tick_do_timer_cpu)
+                return;
+        if (rcu_gp_in_progress(rdp->rsp))
+                WARN_ON_ONCE(smp_processor_id() != tick_do_timer_cpu);
+        /* Pick up current idle and NMI-nesting counter and check. */
+        cur = atomic_read(&rdtp->dynticks_idle);
+        if (cur & 0x1) {
+                *isidle = false; /* We are not idle! */
+                return;
+        }
+        smp_mb(); /* Read counters before timestamps. */
+        /* Pick up timestamps. */
+        j = ACCESS_ONCE(rdtp->dynticks_idle_jiffies);
+        /* If this CPU entered idle more recently, update maxj timestamp. */
+        if (ULONG_CMP_LT(*maxj, j))
+                *maxj = j;
+}
+/*
+ * Is this the flavor of RCU that is handling full-system idle?
+ */
+static bool is_sysidle_rcu_state(struct rcu_state *rsp)
+{
+        return rsp == rcu_sysidle_state;
+}
+/*
+ * Bind the grace-period kthread for the sysidle flavor of RCU to the
+ * timekeeping CPU.
+ */
+static void rcu_bind_gp_kthread(void)
+{
+        int cpu = ACCESS_ONCE(tick_do_timer_cpu);
+        if (cpu < 0 || cpu >= nr_cpu_ids)
+                return;
+        if (raw_smp_processor_id() != cpu)
+                set_cpus_allowed_ptr(current, cpumask_of(cpu));
+}
+/*
+ * Return a delay in jiffies based on the number of CPUs, rcu_node
+ * leaf fanout, and jiffies tick rate.  The idea is to allow larger
+ * systems more time to transition to full-idle state in order to
+ * avoid the cache thrashing that otherwise occur on the state variable.
+ * Really small systems (less than a couple of tens of CPUs) should
+ * instead use a single global atomically incremented counter, and later
+ * versions of this will automatically reconfigure themselves accordingly.
+ */
+static unsigned long rcu_sysidle_delay(void)
+{
+        if (nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL)
+                return 0;
+        return DIV_ROUND_UP(nr_cpu_ids * HZ, rcu_fanout_leaf * 1000);
+}
+/*
+ * Advance the full-system-idle state.  This is invoked when all of
+ * the non-timekeeping CPUs are idle.
+ */
+static void rcu_sysidle(unsigned long j)
+{
+        /* Check the current state. */
+        switch (ACCESS_ONCE(full_sysidle_state)) {
+        case RCU_SYSIDLE_NOT:
+                /* First time all are idle, so note a short idle period. */
+                ACCESS_ONCE(full_sysidle_state) = RCU_SYSIDLE_SHORT;
+                break;
+        case RCU_SYSIDLE_SHORT:
+                /*
+                 * Idle for a bit, time to advance to next state?
+                 * cmpxchg failure means race with non-idle, let them win.
+                 */
+                if (ULONG_CMP_GE(jiffies, j + rcu_sysidle_delay()))
+                        (void)cmpxchg(&full_sysidle_state,
+                                      RCU_SYSIDLE_SHORT, RCU_SYSIDLE_LONG);
+                break;
+        case RCU_SYSIDLE_LONG:
+                /*
+                 * Do an additional check pass before advancing to full.
+                 * cmpxchg failure means race with non-idle, let them win.
+                 */
+                if (ULONG_CMP_GE(jiffies, j + rcu_sysidle_delay()))
+                        (void)cmpxchg(&full_sysidle_state,
+                                      RCU_SYSIDLE_LONG, RCU_SYSIDLE_FULL);
+                break;
+        default:
+                break;
+        }
+}
+/*
+ * Found a non-idle non-timekeeping CPU, so kick the system-idle state
+ * back to the beginning.
+ */
+static void rcu_sysidle_cancel(void)
+{
+        smp_mb();
+        ACCESS_ONCE(full_sysidle_state) = RCU_SYSIDLE_NOT;
+}
+/*
+ * Update the sysidle state based on the results of a force-quiescent-state
+ * scan of the CPUs' dyntick-idle state.
+ */
+static void rcu_sysidle_report(struct rcu_state *rsp, int isidle,
+                               unsigned long maxj, bool gpkt)
+{
+        if (rsp != rcu_sysidle_state)
+                return;  /* Wrong flavor, ignore. */
+        if (gpkt && nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL)
+                return;  /* Running state machine from timekeeping CPU. */
+        if (isidle)
+                rcu_sysidle(maxj);    /* More idle! */
+        else
+                rcu_sysidle_cancel(); /* Idle is over. */
+}
+/*
+ * Wrapper for rcu_sysidle_report() when called from the grace-period
+ * kthread's context.
+ */
+static void rcu_sysidle_report_gp(struct rcu_state *rsp, int isidle,
+                                  unsigned long maxj)
+{
+        rcu_sysidle_report(rsp, isidle, maxj, true);
+}
+/* Callback and function for forcing an RCU grace period. */
+struct rcu_sysidle_head {
+        struct rcu_head rh;
+        int inuse;
+};
+static void rcu_sysidle_cb(struct rcu_head *rhp)
+{
+        struct rcu_sysidle_head *rshp;
+        /*
+         * The following memory barrier is needed to replace the
+         * memory barriers that would normally be in the memory
+         * allocator.
+         */
+        smp_mb();  /* grace period precedes setting inuse. */
+        rshp = container_of(rhp, struct rcu_sysidle_head, rh);
+        ACCESS_ONCE(rshp->inuse) = 0;
+}
+/*
+ * Check to see if the system is fully idle, other than the timekeeping CPU.
+ * The caller must have disabled interrupts.
+ */
+bool rcu_sys_is_idle(void)
+{
+        static struct rcu_sysidle_head rsh;
+        int rss = ACCESS_ONCE(full_sysidle_state);
+        if (WARN_ON_ONCE(smp_processor_id() != tick_do_timer_cpu))
+                return false;
+        /* Handle small-system case by doing a full scan of CPUs. */
+        if (nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL) {
+                int oldrss = rss - 1;
+                /*
+                 * One pass to advance to each state up to _FULL.
+                 * Give up if any pass fails to advance the state.
+                 */
+                while (rss < RCU_SYSIDLE_FULL && oldrss < rss) {
+                        int cpu;
+                        bool isidle = true;
+                        unsigned long maxj = jiffies - ULONG_MAX / 4;
+                        struct rcu_data *rdp;
+                        /* Scan all the CPUs looking for nonidle CPUs. */
+                        for_each_possible_cpu(cpu) {
+                                rdp = per_cpu_ptr(rcu_sysidle_state->rda, cpu);
+                                rcu_sysidle_check_cpu(rdp, &isidle, &maxj);
+                                if (!isidle)
+                                        break;
+                        }
+                        rcu_sysidle_report(rcu_sysidle_state,
+                                           isidle, maxj, false);
+                        oldrss = rss;
+                        rss = ACCESS_ONCE(full_sysidle_state);
+                }
+        }
+        /* If this is the first observation of an idle period, record it. */
+        if (rss == RCU_SYSIDLE_FULL) {
+                rss = cmpxchg(&full_sysidle_state,
+                              RCU_SYSIDLE_FULL, RCU_SYSIDLE_FULL_NOTED);
+                return rss == RCU_SYSIDLE_FULL;
+        }
+        smp_mb(); /* ensure rss load happens before later caller actions. */
+        /* If already fully idle, tell the caller (in case of races). */
+        if (rss == RCU_SYSIDLE_FULL_NOTED)
+                return true;
+        /*
+         * If we aren't there yet, and a grace period is not in flight,
+         * initiate a grace period.  Either way, tell the caller that
+         * we are not there yet.  We use an xchg() rather than an assignment
+         * to make up for the memory barriers that would otherwise be
+         * provided by the memory allocator.
+         */
+        if (nr_cpu_ids > CONFIG_NO_HZ_FULL_SYSIDLE_SMALL &&
+            !rcu_gp_in_progress(rcu_sysidle_state) &&
+            !rsh.inuse && xchg(&rsh.inuse, 1) == 0)
+                call_rcu(&rsh.rh, rcu_sysidle_cb);
+        return false;
+}
+/*
+ * Initialize dynticks sysidle state for CPUs coming online.
+ */
+static void rcu_sysidle_init_percpu_data(struct rcu_dynticks *rdtp)
+{
+        rdtp->dynticks_idle_nesting = DYNTICK_TASK_NEST_VALUE;
+}
+#else /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
+static void rcu_sysidle_enter(struct rcu_dynticks *rdtp, int irq)
+{
+}
+static void rcu_sysidle_exit(struct rcu_dynticks *rdtp, int irq)
+{
+}
+static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
+                                  unsigned long *maxj)
+{
+}
+static bool is_sysidle_rcu_state(struct rcu_state *rsp)
+{
+        return false;
+}
+static void rcu_bind_gp_kthread(void)
+{
+}
+static void rcu_sysidle_report_gp(struct rcu_state *rsp, int isidle,
+                                  unsigned long maxj)
+{
+}
+static void rcu_sysidle_init_percpu_data(struct rcu_dynticks *rdtp)
+{
+}
+#endif /* #else #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
diff --git a/kernel/relay.c b/kernel/relay.c
index b91488ba2e5a..5001c9887db1 100644
--- a/kernel/relay.c
+++ b/kernel/relay.c
@@ -516,7 +516,7 @@ static void setup_callbacks(struct rchan *chan,
 *
 *      Returns the success/failure of the operation. (%NOTIFY_OK, %NOTIFY_BAD)
 */
-static int __cpuinit relay_hotcpu_callback(struct notifier_block *nb,
+static int relay_hotcpu_callback(struct notifier_block *nb,
                                unsigned long action,
                                void *hcpu)
 {
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 0d8eb4525e76..725aa067ad63 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -933,6 +933,8 @@ static int effective_prio(struct task_struct *p)
 /**
 * task_curr - is this task currently executing on a CPU?
 * @p: the task in question.
+ *
+ * Return: 1 if the task is currently executing. 0 otherwise.
 */
 inline int task_curr(const struct task_struct *p)
 {
@@ -1482,7 +1484,7 @@ static void ttwu_queue(struct task_struct *p, int cpu)
 * the simpler "current->state = TASK_RUNNING" to mark yourself
 * runnable without the overhead of this.
 *
- * Returns %true if @p was woken up, %false if it was already running
+ * Return: %true if @p was woken up, %false if it was already running.
 * or @state didn't match @p's state.
 */
 static int
@@ -1491,7 +1493,13 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
        unsigned long flags;
        int cpu, success = 0;
-        smp_wmb();
+        /*
+         * If we are going to wake up a thread waiting for CONDITION we
+         * need to ensure that CONDITION=1 done by the caller can not be
+         * reordered with p->state check below. This pairs with mb() in
+         * set_current_state() the waiting thread does.
+         */
+        smp_mb__before_spinlock();
        raw_spin_lock_irqsave(&p->pi_lock, flags);
        if (!(p->state & state))
                goto out;
@@ -1577,8 +1585,9 @@ out:
 * @p: The process to be woken up.
 *
 * Attempt to wake up the nominated process and move it to the set of runnable
- * processes.  Returns 1 if the process was woken up, 0 if it was already
+ * processes.
- * running.
+ *
+ * Return: 1 if the process was woken up, 0 if it was already running.
 *
 * It may be assumed that this function implies a write memory barrier before
 * changing the task state if and only if any tasks are woken up.
@@ -2191,6 +2200,8 @@ void scheduler_tick(void)
 * This makes sure that uptime, CFS vruntime, load
 * balancing, etc... continue to move forward, even
 * with a very low granularity.
+ *
+ * Return: Maximum deferment in nanoseconds.
 */
 u64 scheduler_tick_max_deferment(void)
 {
@@ -2394,6 +2405,12 @@ need_resched:
        if (sched_feat(HRTICK))
                hrtick_clear(rq);
+        /*
+         * Make sure that signal_pending_state()->signal_pending() below
+         * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE)
+         * done by the caller to avoid the race with signal_wake_up().
+         */
+        smp_mb__before_spinlock();
        raw_spin_lock_irq(&rq->lock);
        switch_count = &prev->nivcsw;
@@ -2510,13 +2527,11 @@ void __sched schedule_preempt_disabled(void)
 */
 asmlinkage void __sched notrace preempt_schedule(void)
 {
-        struct thread_info *ti = current_thread_info();
        /*
         * If there is a non-zero preempt_count or interrupts are disabled,
         * we do not want to preempt the current task. Just return..
         */
-        if (likely(ti->preempt_count || irqs_disabled()))
+        if (likely(!preemptible()))
                return;
        do {
@@ -2660,7 +2675,7 @@ void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode,
        if (unlikely(!q))
                return;
-        if (unlikely(!nr_exclusive))
+        if (unlikely(nr_exclusive != 1))
                wake_flags = 0;
        spin_lock_irqsave(&q->lock, flags);
@@ -2796,8 +2811,8 @@ EXPORT_SYMBOL(wait_for_completion);
 * specified timeout to expire. The timeout is in jiffies. It is not
 * interruptible.
 *
- * The return value is 0 if timed out, and positive (at least 1, or number of
+ * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
- * jiffies left till timeout) if completed.
+ * till timeout) if completed.
 */
 unsigned long __sched
 wait_for_completion_timeout(struct completion *x, unsigned long timeout)
@@ -2829,8 +2844,8 @@ EXPORT_SYMBOL(wait_for_completion_io);
 * specified timeout to expire. The timeout is in jiffies. It is not
 * interruptible. The caller is accounted as waiting for IO.
 *
- * The return value is 0 if timed out, and positive (at least 1, or number of
+ * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
- * jiffies left till timeout) if completed.
+ * till timeout) if completed.
 */
 unsigned long __sched
 wait_for_completion_io_timeout(struct completion *x, unsigned long timeout)
@@ -2846,7 +2861,7 @@ EXPORT_SYMBOL(wait_for_completion_io_timeout);
 * This waits for completion of a specific task to be signaled. It is
 * interruptible.
 *
- * The return value is -ERESTARTSYS if interrupted, 0 if completed.
+ * Return: -ERESTARTSYS if interrupted, 0 if completed.
 */
 int __sched wait_for_completion_interruptible(struct completion *x)
 {
@@ -2865,8 +2880,8 @@ EXPORT_SYMBOL(wait_for_completion_interruptible);
 * This waits for either a completion of a specific task to be signaled or for a
 * specified timeout to expire. It is interruptible. The timeout is in jiffies.
 *
- * The return value is -ERESTARTSYS if interrupted, 0 if timed out,
+ * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
- * positive (at least 1, or number of jiffies left till timeout) if completed.
+ * or number of jiffies left till timeout) if completed.
 */
 long __sched
 wait_for_completion_interruptible_timeout(struct completion *x,
@@ -2883,7 +2898,7 @@ EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
 * This waits to be signaled for completion of a specific task. It can be
 * interrupted by a kill signal.
 *
- * The return value is -ERESTARTSYS if interrupted, 0 if completed.
+ * Return: -ERESTARTSYS if interrupted, 0 if completed.
 */
 int __sched wait_for_completion_killable(struct completion *x)
 {
@@ -2903,8 +2918,8 @@ EXPORT_SYMBOL(wait_for_completion_killable);
 * signaled or for a specified timeout to expire. It can be
 * interrupted by a kill signal. The timeout is in jiffies.
 *
- * The return value is -ERESTARTSYS if interrupted, 0 if timed out,
+ * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
- * positive (at least 1, or number of jiffies left till timeout) if completed.
+ * or number of jiffies left till timeout) if completed.
 */
 long __sched
 wait_for_completion_killable_timeout(struct completion *x,
@@ -2918,7 +2933,7 @@ EXPORT_SYMBOL(wait_for_completion_killable_timeout);
 *      try_wait_for_completion - try to decrement a completion without blocking
 *      @x:     completion structure
 *
- *      Returns: 0 if a decrement cannot be done without blocking
+ *      Return: 0 if a decrement cannot be done without blocking
 *               1 if a decrement succeeded.
 *
 *      If a completion is being used as a counting completion,
@@ -2945,7 +2960,7 @@ EXPORT_SYMBOL(try_wait_for_completion);
 *      completion_done - Test to see if a completion has any waiters
 *      @x:     completion structure
 *
- *      Returns: 0 if there are waiters (wait_for_completion() in progress)
+ *      Return: 0 if there are waiters (wait_for_completion() in progress)
 *               1 if there are no waiters.
 *
 */
@@ -3182,7 +3197,7 @@ SYSCALL_DEFINE1(nice, int, increment)
 * task_prio - return the priority value of a given task.
 * @p: the task in question.
 *
- * This is the priority value as seen by users in /proc.
+ * Return: The priority value as seen by users in /proc.
 * RT tasks are offset by -200. Normal tasks are centered
 * around 0, value goes from -16 to +15.
 */
@@ -3194,6 +3209,8 @@ int task_prio(const struct task_struct *p)
 /**
 * task_nice - return the nice value of a given task.
 * @p: the task in question.
+ *
+ * Return: The nice value [ -20 ... 0 ... 19 ].
 */
 int task_nice(const struct task_struct *p)
 {
@@ -3204,6 +3221,8 @@ EXPORT_SYMBOL(task_nice);
 /**
 * idle_cpu - is a given cpu idle currently?
 * @cpu: the processor in question.
+ *
+ * Return: 1 if the CPU is currently idle. 0 otherwise.
 */
 int idle_cpu(int cpu)
 {
@@ -3226,6 +3245,8 @@ int idle_cpu(int cpu)
 /**
 * idle_task - return the idle task for a given cpu.
 * @cpu: the processor in question.
+ *
+ * Return: The idle task for the cpu @cpu.
 */
 struct task_struct *idle_task(int cpu)
 {
@@ -3235,6 +3256,8 @@ struct task_struct *idle_task(int cpu)
 /**
 * find_process_by_pid - find a process with a matching PID value.
 * @pid: the pid in question.
+ *
+ * The task of @pid, if found. %NULL otherwise.
 */
 static struct task_struct *find_process_by_pid(pid_t pid)
 {
@@ -3432,6 +3455,8 @@ recheck:
 * @policy: new policy.
 * @param: structure containing the new RT priority.
 *
+ * Return: 0 on success. An error code otherwise.
+ *
 * NOTE that the task may be already dead.
 */
 int sched_setscheduler(struct task_struct *p, int policy,
@@ -3451,6 +3476,8 @@ EXPORT_SYMBOL_GPL(sched_setscheduler);
 * current context has permission.  For example, this is needed in
 * stop_machine(): we create temporary high priority worker threads,
 * but our caller might not have that capability.
+ *
+ * Return: 0 on success. An error code otherwise.
 */
 int sched_setscheduler_nocheck(struct task_struct *p, int policy,
                               const struct sched_param *param)
@@ -3485,6 +3512,8 @@ do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
 * @pid: the pid in question.
 * @policy: new policy.
 * @param: structure containing the new RT priority.
+ *
+ * Return: 0 on success. An error code otherwise.
 */
 SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy,
                struct sched_param __user *, param)
@@ -3500,6 +3529,8 @@ SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy,
 * sys_sched_setparam - set/change the RT priority of a thread
 * @pid: the pid in question.
 * @param: structure containing the new RT priority.
+ *
+ * Return: 0 on success. An error code otherwise.
 */
 SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param)
 {
@@ -3509,6 +3540,9 @@ SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param)
 /**
 * sys_sched_getscheduler - get the policy (scheduling class) of a thread
 * @pid: the pid in question.
+ *
+ * Return: On success, the policy of the thread. Otherwise, a negative error
+ * code.
 */
 SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
 {
@@ -3535,6 +3569,9 @@ SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
 * sys_sched_getparam - get the RT priority of a thread
 * @pid: the pid in question.
 * @param: structure containing the RT priority.
+ *
+ * Return: On success, 0 and the RT priority is in @param. Otherwise, an error
+ * code.
 */
 SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
 {
@@ -3659,6 +3696,8 @@ static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
 * @pid: pid of the process
 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
 * @user_mask_ptr: user-space pointer to the new cpu mask
+ *
+ * Return: 0 on success. An error code otherwise.
 */
 SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len,
                unsigned long __user *, user_mask_ptr)
@@ -3710,6 +3749,8 @@ out_unlock:
 * @pid: pid of the process
 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
 * @user_mask_ptr: user-space pointer to hold the current cpu mask
+ *
+ * Return: 0 on success. An error code otherwise.
 */
 SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len,
                unsigned long __user *, user_mask_ptr)
@@ -3744,6 +3785,8 @@ SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len,
 *
 * This function yields the current CPU to other tasks. If there are no
 * other threads running on this CPU then this function will return.
+ *
+ * Return: 0.
 */
 SYSCALL_DEFINE0(sched_yield)
 {
@@ -3869,7 +3912,7 @@ EXPORT_SYMBOL(yield);
 * It's the caller's job to ensure that the target task struct
 * can't go away on us before we can do any checks.
 *
- * Returns:
+ * Return:
 *      true (>0) if we indeed boosted the target task.
 *      false (0) if we failed to boost the target.
 *      -ESRCH if there's no task to yield to.
@@ -3972,8 +4015,9 @@ long __sched io_schedule_timeout(long timeout)
 * sys_sched_get_priority_max - return maximum RT priority.
 * @policy: scheduling class.
 *
- * this syscall returns the maximum rt_priority that can be used
+ * Return: On success, this syscall returns the maximum
- * by a given scheduling class.
+ * rt_priority that can be used by a given scheduling class.
+ * On failure, a negative error code is returned.
 */
 SYSCALL_DEFINE1(sched_get_priority_max, int, policy)
 {
@@ -3997,8 +4041,9 @@ SYSCALL_DEFINE1(sched_get_priority_max, int, policy)
 * sys_sched_get_priority_min - return minimum RT priority.
 * @policy: scheduling class.
 *
- * this syscall returns the minimum rt_priority that can be used
+ * Return: On success, this syscall returns the minimum
- * by a given scheduling class.
+ * rt_priority that can be used by a given scheduling class.
+ * On failure, a negative error code is returned.
 */
 SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
 {
@@ -4024,6 +4069,9 @@ SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
 *
 * this syscall writes the default timeslice value of a given process
 * into the user-space timespec buffer. A value of '0' means infinity.
+ *
+ * Return: On success, 0 and the timeslice is in @interval. Otherwise,
+ * an error code.
 */
 SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
                struct timespec __user *, interval)
@@ -4133,7 +4181,7 @@ void show_state_filter(unsigned long state_filter)
                debug_show_all_locks();
 }
-void __cpuinit init_idle_bootup_task(struct task_struct *idle)
+void init_idle_bootup_task(struct task_struct *idle)
 {
        idle->sched_class = &idle_sched_class;
 }
@@ -4146,7 +4194,7 @@ void __cpuinit init_idle_bootup_task(struct task_struct *idle)
 * NOTE: this function does not set the idle thread's NEED_RESCHED
 * flag, to make booting more robust.
 */
-void __cpuinit init_idle(struct task_struct *idle, int cpu)
+void init_idle(struct task_struct *idle, int cpu)
 {
        struct rq *rq = cpu_rq(cpu);
        unsigned long flags;
@@ -4630,7 +4678,7 @@ static void set_rq_offline(struct rq *rq)
 * migration_call - callback that gets triggered when a CPU is added.
 * Here we can start up the necessary migration thread for the new CPU.
 */
-static int __cpuinit
+static int
 migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
 {
        int cpu = (long)hcpu;
@@ -4684,12 +4732,12 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
 * happens before everything else.  This has to be lower priority than
 * the notifier in the perf_event subsystem, though.
 */
-static struct notifier_block __cpuinitdata migration_notifier = {
+static struct notifier_block migration_notifier = {
        .notifier_call = migration_call,
        .priority = CPU_PRI_MIGRATION,
 };
-static int __cpuinit sched_cpu_active(struct notifier_block *nfb,
+static int sched_cpu_active(struct notifier_block *nfb,
                                      unsigned long action, void *hcpu)
 {
        switch (action & ~CPU_TASKS_FROZEN) {
@@ -4702,7 +4750,7 @@ static int __cpuinit sched_cpu_active(struct notifier_block *nfb,
        }
 }
-static int __cpuinit sched_cpu_inactive(struct notifier_block *nfb,
+static int sched_cpu_inactive(struct notifier_block *nfb,
                                        unsigned long action, void *hcpu)
 {
        switch (action & ~CPU_TASKS_FROZEN) {
@@ -4914,7 +4962,8 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
                                SD_BALANCE_FORK |
                                SD_BALANCE_EXEC |
                                SD_SHARE_CPUPOWER |
-                                SD_SHARE_PKG_RESOURCES);
+                                SD_SHARE_PKG_RESOURCES |
+                                SD_PREFER_SIBLING);
                if (nr_node_ids == 1)
                        pflags &= ~SD_SERIALIZE;
        }
@@ -5083,18 +5132,23 @@ static void destroy_sched_domains(struct sched_domain *sd, int cpu)
 * two cpus are in the same cache domain, see cpus_share_cache().
 */
 DEFINE_PER_CPU(struct sched_domain *, sd_llc);
+DEFINE_PER_CPU(int, sd_llc_size);
 DEFINE_PER_CPU(int, sd_llc_id);
 static void update_top_cache_domain(int cpu)
 {
        struct sched_domain *sd;
        int id = cpu;
+        int size = 1;
        sd = highest_flag_domain(cpu, SD_SHARE_PKG_RESOURCES);
-        if (sd)
+        if (sd) {
                id = cpumask_first(sched_domain_span(sd));
+                size = cpumask_weight(sched_domain_span(sd));
+        }
        rcu_assign_pointer(per_cpu(sd_llc, cpu), sd);
+        per_cpu(sd_llc_size, cpu) = size;
        per_cpu(sd_llc_id, cpu) = id;
 }
@@ -5118,6 +5172,13 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
                        tmp->parent = parent->parent;
                        if (parent->parent)
                                parent->parent->child = tmp;
+                        /*
+                         * Transfer SD_PREFER_SIBLING down in case of a
+                         * degenerate parent; the spans match for this
+                         * so the property transfers.
+                         */
+                        if (parent->flags & SD_PREFER_SIBLING)
+                                tmp->flags |= SD_PREFER_SIBLING;
                        destroy_sched_domain(parent, cpu);
                } else
                        tmp = tmp->parent;
@@ -6184,8 +6245,9 @@ match1:
                ;
        }
+        n = ndoms_cur;
        if (doms_new == NULL) {
-                ndoms_cur = 0;
+                n = 0;
                doms_new = &fallback_doms;
                cpumask_andnot(doms_new[0], cpu_active_mask, cpu_isolated_map);
                WARN_ON_ONCE(dattr_new);
@@ -6193,7 +6255,7 @@ match1:
        /* Build new domains */
        for (i = 0; i < ndoms_new; i++) {
-                for (j = 0; j < ndoms_cur && !new_topology; j++) {
+                for (j = 0; j < n && !new_topology; j++) {
                        if (cpumask_equal(doms_new[i], doms_cur[j])
                            && dattrs_equal(dattr_new, i, dattr_cur, j))
                                goto match2;
@@ -6632,6 +6694,8 @@ void normalize_rt_tasks(void)
 * @cpu: the processor in question.
 *
 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
+ *
+ * Return: The current task for @cpu.
 */
 struct task_struct *curr_task(int cpu)
 {
@@ -6763,7 +6827,7 @@ void sched_move_task(struct task_struct *tsk)
        if (unlikely(running))
                tsk->sched_class->put_prev_task(rq, tsk);
-        tg = container_of(task_subsys_state_check(tsk, cpu_cgroup_subsys_id,
+        tg = container_of(task_css_check(tsk, cpu_cgroup_subsys_id,
                                lockdep_is_held(&tsk->sighand->siglock)),
                          struct task_group, css);
        tg = autogroup_task_group(tsk, tg);
@@ -7085,23 +7149,22 @@ int sched_rt_handler(struct ctl_table *table, int write,
 #ifdef CONFIG_CGROUP_SCHED
-/* return corresponding task_group object of a cgroup */
+static inline struct task_group *css_tg(struct cgroup_subsys_state *css)
-static inline struct task_group *cgroup_tg(struct cgroup *cgrp)
 {
-        return container_of(cgroup_subsys_state(cgrp, cpu_cgroup_subsys_id),
+        return css ? container_of(css, struct task_group, css) : NULL;
-                            struct task_group, css);
 }
-static struct cgroup_subsys_state *cpu_cgroup_css_alloc(struct cgroup *cgrp)
+static struct cgroup_subsys_state *
+cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
 {
-        struct task_group *tg, *parent;
+        struct task_group *parent = css_tg(parent_css);
+        struct task_group *tg;
-        if (!cgrp->parent) {
+        if (!parent) {
                /* This is early initialization for the top cgroup */
                return &root_task_group.css;
        }
-        parent = cgroup_tg(cgrp->parent);
        tg = sched_create_group(parent);
        if (IS_ERR(tg))
                return ERR_PTR(-ENOMEM);
@@ -7109,41 +7172,38 @@ static struct cgroup_subsys_state *cpu_cgroup_css_alloc(struct cgroup *cgrp)
        return &tg->css;
 }
-static int cpu_cgroup_css_online(struct cgroup *cgrp)
+static int cpu_cgroup_css_online(struct cgroup_subsys_state *css)
 {
-        struct task_group *tg = cgroup_tg(cgrp);
+        struct task_group *tg = css_tg(css);
-        struct task_group *parent;
+        struct task_group *parent = css_tg(css_parent(css));
-        if (!cgrp->parent)
+        if (parent)
-                return 0;
+                sched_online_group(tg, parent);
-        parent = cgroup_tg(cgrp->parent);
-        sched_online_group(tg, parent);
        return 0;
 }
-static void cpu_cgroup_css_free(struct cgroup *cgrp)
+static void cpu_cgroup_css_free(struct cgroup_subsys_state *css)
 {
-        struct task_group *tg = cgroup_tg(cgrp);
+        struct task_group *tg = css_tg(css);
        sched_destroy_group(tg);
 }
-static void cpu_cgroup_css_offline(struct cgroup *cgrp)
+static void cpu_cgroup_css_offline(struct cgroup_subsys_state *css)
 {
-        struct task_group *tg = cgroup_tg(cgrp);
+        struct task_group *tg = css_tg(css);
        sched_offline_group(tg);
 }
-static int cpu_cgroup_can_attach(struct cgroup *cgrp,
+static int cpu_cgroup_can_attach(struct cgroup_subsys_state *css,
                                 struct cgroup_taskset *tset)
 {
        struct task_struct *task;
-        cgroup_taskset_for_each(task, cgrp, tset) {
+        cgroup_taskset_for_each(task, css, tset) {
 #ifdef CONFIG_RT_GROUP_SCHED
-                if (!sched_rt_can_attach(cgroup_tg(cgrp), task))
+                if (!sched_rt_can_attach(css_tg(css), task))
                        return -EINVAL;
 #else
                /* We don't support RT-tasks being in separate groups */
@@ -7154,18 +7214,18 @@ static int cpu_cgroup_can_attach(struct cgroup *cgrp,
        return 0;
 }
-static void cpu_cgroup_attach(struct cgroup *cgrp,
+static void cpu_cgroup_attach(struct cgroup_subsys_state *css,
                              struct cgroup_taskset *tset)
 {
        struct task_struct *task;
-        cgroup_taskset_for_each(task, cgrp, tset)
+        cgroup_taskset_for_each(task, css, tset)
                sched_move_task(task);
 }
-static void
+static void cpu_cgroup_exit(struct cgroup_subsys_state *css,
-cpu_cgroup_exit(struct cgroup *cgrp, struct cgroup *old_cgrp,
+                            struct cgroup_subsys_state *old_css,
-                struct task_struct *task)
+                            struct task_struct *task)
 {
        /*
         * cgroup_exit() is called in the copy_process() failure path.
@@ -7179,15 +7239,16 @@ cpu_cgroup_exit(struct cgroup *cgrp, struct cgroup *old_cgrp,
 }
 #ifdef CONFIG_FAIR_GROUP_SCHED
-static int cpu_shares_write_u64(struct cgroup *cgrp, struct cftype *cftype,
+static int cpu_shares_write_u64(struct cgroup_subsys_state *css,
-                                u64 shareval)
+                                struct cftype *cftype, u64 shareval)
 {
-        return sched_group_set_shares(cgroup_tg(cgrp), scale_load(shareval));
+        return sched_group_set_shares(css_tg(css), scale_load(shareval));
 }
-static u64 cpu_shares_read_u64(struct cgroup *cgrp, struct cftype *cft)
+static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css,
+                               struct cftype *cft)
 {
-        struct task_group *tg = cgroup_tg(cgrp);
+        struct task_group *tg = css_tg(css);
        return (u64) scale_load_down(tg->shares);
 }
@@ -7309,26 +7370,28 @@ long tg_get_cfs_period(struct task_group *tg)
        return cfs_period_us;
 }
-static s64 cpu_cfs_quota_read_s64(struct cgroup *cgrp, struct cftype *cft)
+static s64 cpu_cfs_quota_read_s64(struct cgroup_subsys_state *css,
+                                  struct cftype *cft)
 {
-        return tg_get_cfs_quota(cgroup_tg(cgrp));
+        return tg_get_cfs_quota(css_tg(css));
 }
-static int cpu_cfs_quota_write_s64(struct cgroup *cgrp, struct cftype *cftype,
+static int cpu_cfs_quota_write_s64(struct cgroup_subsys_state *css,
-                                s64 cfs_quota_us)
+                                   struct cftype *cftype, s64 cfs_quota_us)
 {
-        return tg_set_cfs_quota(cgroup_tg(cgrp), cfs_quota_us);
+        return tg_set_cfs_quota(css_tg(css), cfs_quota_us);
 }
-static u64 cpu_cfs_period_read_u64(struct cgroup *cgrp, struct cftype *cft)
+static u64 cpu_cfs_period_read_u64(struct cgroup_subsys_state *css,
+                                   struct cftype *cft)
 {
-        return tg_get_cfs_period(cgroup_tg(cgrp));
+        return tg_get_cfs_period(css_tg(css));
 }
-static int cpu_cfs_period_write_u64(struct cgroup *cgrp, struct cftype *cftype,
+static int cpu_cfs_period_write_u64(struct cgroup_subsys_state *css,
-                                u64 cfs_period_us)
+                                    struct cftype *cftype, u64 cfs_period_us)
 {
-        return tg_set_cfs_period(cgroup_tg(cgrp), cfs_period_us);
+        return tg_set_cfs_period(css_tg(css), cfs_period_us);
 }
 struct cfs_schedulable_data {
@@ -7409,10 +7472,10 @@ static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota)
        return ret;
 }
-static int cpu_stats_show(struct cgroup *cgrp, struct cftype *cft,
+static int cpu_stats_show(struct cgroup_subsys_state *css, struct cftype *cft,
                struct cgroup_map_cb *cb)
 {
-        struct task_group *tg = cgroup_tg(cgrp);
+        struct task_group *tg = css_tg(css);
        struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
        cb->fill(cb, "nr_periods", cfs_b->nr_periods);
@@ -7425,26 +7488,28 @@ static int cpu_stats_show(struct cgroup *cgrp, struct cftype *cft,
 #endif /* CONFIG_FAIR_GROUP_SCHED */
 #ifdef CONFIG_RT_GROUP_SCHED
-static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft,
+static int cpu_rt_runtime_write(struct cgroup_subsys_state *css,
-                                s64 val)
+                                struct cftype *cft, s64 val)
 {
-        return sched_group_set_rt_runtime(cgroup_tg(cgrp), val);
+        return sched_group_set_rt_runtime(css_tg(css), val);
 }
-static s64 cpu_rt_runtime_read(struct cgroup *cgrp, struct cftype *cft)
+static s64 cpu_rt_runtime_read(struct cgroup_subsys_state *css,
+                               struct cftype *cft)
 {
-        return sched_group_rt_runtime(cgroup_tg(cgrp));
+        return sched_group_rt_runtime(css_tg(css));
 }
-static int cpu_rt_period_write_uint(struct cgroup *cgrp, struct cftype *cftype,
+static int cpu_rt_period_write_uint(struct cgroup_subsys_state *css,
-                u64 rt_period_us)
+                                    struct cftype *cftype, u64 rt_period_us)
 {
-        return sched_group_set_rt_period(cgroup_tg(cgrp), rt_period_us);
+        return sched_group_set_rt_period(css_tg(css), rt_period_us);
 }
-static u64 cpu_rt_period_read_uint(struct cgroup *cgrp, struct cftype *cft)
+static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css,
+                                   struct cftype *cft)
 {
-        return sched_group_rt_period(cgroup_tg(cgrp));
+        return sched_group_rt_period(css_tg(css));
 }
 #endif /* CONFIG_RT_GROUP_SCHED */
diff --git a/kernel/sched/cpuacct.c b/kernel/sched/cpuacct.c
index dbb7e2cd95eb..f64722ff0299 100644
--- a/kernel/sched/cpuacct.c
+++ b/kernel/sched/cpuacct.c
@@ -33,30 +33,20 @@ struct cpuacct {
        struct kernel_cpustat __percpu *cpustat;
 };
-/* return cpu accounting group corresponding to this container */
+static inline struct cpuacct *css_ca(struct cgroup_subsys_state *css)
-static inline struct cpuacct *cgroup_ca(struct cgroup *cgrp)
 {
-        return container_of(cgroup_subsys_state(cgrp, cpuacct_subsys_id),
+        return css ? container_of(css, struct cpuacct, css) : NULL;
-                            struct cpuacct, css);
 }
 /* return cpu accounting group to which this task belongs */
 static inline struct cpuacct *task_ca(struct task_struct *tsk)
 {
-        return container_of(task_subsys_state(tsk, cpuacct_subsys_id),
+        return css_ca(task_css(tsk, cpuacct_subsys_id));
-                            struct cpuacct, css);
-}
-static inline struct cpuacct *__parent_ca(struct cpuacct *ca)
-{
-        return cgroup_ca(ca->css.cgroup->parent);
 }
 static inline struct cpuacct *parent_ca(struct cpuacct *ca)
 {
-        if (!ca->css.cgroup->parent)
+        return css_ca(css_parent(&ca->css));
-                return NULL;
-        return cgroup_ca(ca->css.cgroup->parent);
 }
 static DEFINE_PER_CPU(u64, root_cpuacct_cpuusage);
@@ -66,11 +56,12 @@ static struct cpuacct root_cpuacct = {
 };
 /* create a new cpu accounting group */
-static struct cgroup_subsys_state *cpuacct_css_alloc(struct cgroup *cgrp)
+static struct cgroup_subsys_state *
+cpuacct_css_alloc(struct cgroup_subsys_state *parent_css)
 {
        struct cpuacct *ca;
-        if (!cgrp->parent)
+        if (!parent_css)
                return &root_cpuacct.css;
        ca = kzalloc(sizeof(*ca), GFP_KERNEL);
@@ -96,9 +87,9 @@ out:
 }
 /* destroy an existing cpu accounting group */
-static void cpuacct_css_free(struct cgroup *cgrp)
+static void cpuacct_css_free(struct cgroup_subsys_state *css)
 {
-        struct cpuacct *ca = cgroup_ca(cgrp);
+        struct cpuacct *ca = css_ca(css);
        free_percpu(ca->cpustat);
        free_percpu(ca->cpuusage);
@@ -141,9 +132,9 @@ static void cpuacct_cpuusage_write(struct cpuacct *ca, int cpu, u64 val)
 }
 /* return total cpu usage (in nanoseconds) of a group */
-static u64 cpuusage_read(struct cgroup *cgrp, struct cftype *cft)
+static u64 cpuusage_read(struct cgroup_subsys_state *css, struct cftype *cft)
 {
-        struct cpuacct *ca = cgroup_ca(cgrp);
+        struct cpuacct *ca = css_ca(css);
        u64 totalcpuusage = 0;
        int i;
@@ -153,10 +144,10 @@ static u64 cpuusage_read(struct cgroup *cgrp, struct cftype *cft)
        return totalcpuusage;
 }
-static int cpuusage_write(struct cgroup *cgrp, struct cftype *cftype,
+static int cpuusage_write(struct cgroup_subsys_state *css, struct cftype *cft,
-                                                                u64 reset)
+                          u64 reset)
 {
-        struct cpuacct *ca = cgroup_ca(cgrp);
+        struct cpuacct *ca = css_ca(css);
        int err = 0;
        int i;
@@ -172,10 +163,10 @@ out:
        return err;
 }
-static int cpuacct_percpu_seq_read(struct cgroup *cgroup, struct cftype *cft,
+static int cpuacct_percpu_seq_read(struct cgroup_subsys_state *css,
-                                   struct seq_file *m)
+                                   struct cftype *cft, struct seq_file *m)
 {
-        struct cpuacct *ca = cgroup_ca(cgroup);
+        struct cpuacct *ca = css_ca(css);
        u64 percpu;
        int i;
@@ -192,10 +183,10 @@ static const char * const cpuacct_stat_desc[] = {
        [CPUACCT_STAT_SYSTEM] = "system",
 };
-static int cpuacct_stats_show(struct cgroup *cgrp, struct cftype *cft,
+static int cpuacct_stats_show(struct cgroup_subsys_state *css,
-                              struct cgroup_map_cb *cb)
+                              struct cftype *cft, struct cgroup_map_cb *cb)
 {
-        struct cpuacct *ca = cgroup_ca(cgrp);
+        struct cpuacct *ca = css_ca(css);
        int cpu;
        s64 val = 0;
@@ -281,7 +272,7 @@ void cpuacct_account_field(struct task_struct *p, int index, u64 val)
        while (ca != &root_cpuacct) {
                kcpustat = this_cpu_ptr(ca->cpustat);
                kcpustat->cpustat[index] += val;
-                ca = __parent_ca(ca);
+                ca = parent_ca(ca);
        }
        rcu_read_unlock();
 }
diff --git a/kernel/sched/cpupri.c b/kernel/sched/cpupri.c
index 1095e878a46f..8b836b376d91 100644
--- a/kernel/sched/cpupri.c
+++ b/kernel/sched/cpupri.c
@@ -62,7 +62,7 @@ static int convert_prio(int prio)
 * any discrepancies created by racing against the uncertainty of the current
 * priority configuration.
 *
- * Returns: (int)bool - CPUs were found
+ * Return: (int)bool - CPUs were found
 */
 int cpupri_find(struct cpupri *cp, struct task_struct *p,
                struct cpumask *lowest_mask)
@@ -203,7 +203,7 @@ void cpupri_set(struct cpupri *cp, int cpu, int newpri)
 * cpupri_init - initialize the cpupri structure
 * @cp: The cpupri context
 *
- * Returns: -ENOMEM if memory fails.
+ * Return: -ENOMEM on memory allocation failure.
 */
 int cpupri_init(struct cpupri *cp)
 {
diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c
index a7959e05a9d5..ace34f95e200 100644
--- a/kernel/sched/cputime.c
+++ b/kernel/sched/cputime.c
@@ -121,7 +121,7 @@ static inline void task_group_account_field(struct task_struct *p, int index,
         * is the only cgroup, then nothing else should be necessary.
         *
         */
-        __get_cpu_var(kernel_cpustat).cpustat[index] += tmp;
+        __this_cpu_add(kernel_cpustat.cpustat[index], tmp);
        cpuacct_account_field(p, index, tmp);
 }
@@ -378,11 +378,8 @@ static inline void irqtime_account_process_tick(struct task_struct *p, int user_
 #ifdef CONFIG_VIRT_CPU_ACCOUNTING
 #ifndef __ARCH_HAS_VTIME_TASK_SWITCH
-void vtime_task_switch(struct task_struct *prev)
+void vtime_common_task_switch(struct task_struct *prev)
 {
-        if (!vtime_accounting_enabled())
-                return;
        if (is_idle_task(prev))
                vtime_account_idle(prev);
        else
@@ -404,11 +401,8 @@ void vtime_task_switch(struct task_struct *prev)
 * vtime_account().
 */
 #ifndef __ARCH_HAS_VTIME_ACCOUNT
-void vtime_account_irq_enter(struct task_struct *tsk)
+void vtime_common_account_irq_enter(struct task_struct *tsk)
 {
-        if (!vtime_accounting_enabled())
-                return;
        if (!in_interrupt()) {
                /*
                 * If we interrupted user, context_tracking_in_user()
@@ -428,7 +422,7 @@ void vtime_account_irq_enter(struct task_struct *tsk)
        }
        vtime_account_system(tsk);
 }
-EXPORT_SYMBOL_GPL(vtime_account_irq_enter);
+EXPORT_SYMBOL_GPL(vtime_common_account_irq_enter);
 #endif /* __ARCH_HAS_VTIME_ACCOUNT */
 #endif /* CONFIG_VIRT_CPU_ACCOUNTING */
@@ -559,12 +553,6 @@ static void cputime_adjust(struct task_cputime *curr,
 {
        cputime_t rtime, stime, utime, total;
-        if (vtime_accounting_enabled()) {
-                *ut = curr->utime;
-                *st = curr->stime;
-                return;
-        }
        stime = curr->stime;
        total = stime + curr->utime;
@@ -664,23 +652,17 @@ static void __vtime_account_system(struct task_struct *tsk)
 void vtime_account_system(struct task_struct *tsk)
 {
-        if (!vtime_accounting_enabled())
-                return;
        write_seqlock(&tsk->vtime_seqlock);
        __vtime_account_system(tsk);
        write_sequnlock(&tsk->vtime_seqlock);
 }
-void vtime_account_irq_exit(struct task_struct *tsk)
+void vtime_gen_account_irq_exit(struct task_struct *tsk)
 {
-        if (!vtime_accounting_enabled())
-                return;
        write_seqlock(&tsk->vtime_seqlock);
+        __vtime_account_system(tsk);
        if (context_tracking_in_user())
                tsk->vtime_snap_whence = VTIME_USER;
-        __vtime_account_system(tsk);
        write_sequnlock(&tsk->vtime_seqlock);
 }
@@ -688,12 +670,8 @@ void vtime_account_user(struct task_struct *tsk)
 {
        cputime_t delta_cpu;
-        if (!vtime_accounting_enabled())
-                return;
-        delta_cpu = get_vtime_delta(tsk);
        write_seqlock(&tsk->vtime_seqlock);
+        delta_cpu = get_vtime_delta(tsk);
        tsk->vtime_snap_whence = VTIME_SYS;
        account_user_time(tsk, delta_cpu, cputime_to_scaled(delta_cpu));
        write_sequnlock(&tsk->vtime_seqlock);
@@ -701,22 +679,27 @@ void vtime_account_user(struct task_struct *tsk)
 void vtime_user_enter(struct task_struct *tsk)
 {
-        if (!vtime_accounting_enabled())
-                return;
        write_seqlock(&tsk->vtime_seqlock);
-        tsk->vtime_snap_whence = VTIME_USER;
        __vtime_account_system(tsk);
+        tsk->vtime_snap_whence = VTIME_USER;
        write_sequnlock(&tsk->vtime_seqlock);
 }
 void vtime_guest_enter(struct task_struct *tsk)
 {
+        /*
+         * The flags must be updated under the lock with
+         * the vtime_snap flush and update.
+         * That enforces a right ordering and update sequence
+         * synchronization against the reader (task_gtime())
+         * that can thus safely catch up with a tickless delta.
+         */
        write_seqlock(&tsk->vtime_seqlock);
        __vtime_account_system(tsk);
        current->flags |= PF_VCPU;
        write_sequnlock(&tsk->vtime_seqlock);
 }
+EXPORT_SYMBOL_GPL(vtime_guest_enter);
 void vtime_guest_exit(struct task_struct *tsk)
 {
@@ -725,6 +708,7 @@ void vtime_guest_exit(struct task_struct *tsk)
        current->flags &= ~PF_VCPU;
        write_sequnlock(&tsk->vtime_seqlock);
 }
+EXPORT_SYMBOL_GPL(vtime_guest_exit);
 void vtime_account_idle(struct task_struct *tsk)
 {
@@ -733,11 +717,6 @@ void vtime_account_idle(struct task_struct *tsk)
        account_idle_time(delta_cpu);
 }
-bool vtime_accounting_enabled(void)
-{
-        return context_tracking_active();
-}
 void arch_vtime_task_switch(struct task_struct *prev)
 {
        write_seqlock(&prev->vtime_seqlock);
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index f77f9c527449..7f0a5e6cdae0 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -851,7 +851,7 @@ void task_numa_fault(int node, int pages, bool migrated)
 {
        struct task_struct *p = current;
-        if (!sched_feat_numa(NUMA))
+        if (!numabalancing_enabled)
                return;
        /* FIXME: Allocate task-specific structure for placement policy here */
@@ -2032,6 +2032,7 @@ entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued)
         */
        update_entity_load_avg(curr, 1);
        update_cfs_rq_blocked_load(cfs_rq, 1);
+        update_cfs_shares(cfs_rq);
 #ifdef CONFIG_SCHED_HRTICK
        /*
@@ -3017,6 +3018,23 @@ static unsigned long cpu_avg_load_per_task(int cpu)
        return 0;
 }
+static void record_wakee(struct task_struct *p)
+{
+        /*
+         * Rough decay (wiping) for cost saving, don't worry
+         * about the boundary, really active task won't care
+         * about the loss.
+         */
+        if (jiffies > current->wakee_flip_decay_ts + HZ) {
+                current->wakee_flips = 0;
+                current->wakee_flip_decay_ts = jiffies;
+        }
+        if (current->last_wakee != p) {
+                current->last_wakee = p;
+                current->wakee_flips++;
+        }
+}
 static void task_waking_fair(struct task_struct *p)
 {
@@ -3037,6 +3055,7 @@ static void task_waking_fair(struct task_struct *p)
 #endif
        se->vruntime -= min_vruntime;
+        record_wakee(p);
 }
 #ifdef CONFIG_FAIR_GROUP_SCHED
@@ -3155,6 +3174,28 @@ static inline unsigned long effective_load(struct task_group *tg, int cpu,
 #endif
+static int wake_wide(struct task_struct *p)
+{
+        int factor = this_cpu_read(sd_llc_size);
+        /*
+         * Yeah, it's the switching-frequency, could means many wakee or
+         * rapidly switch, use factor here will just help to automatically
+         * adjust the loose-degree, so bigger node will lead to more pull.
+         */
+        if (p->wakee_flips > factor) {
+                /*
+                 * wakee is somewhat hot, it needs certain amount of cpu
+                 * resource, so if waker is far more hot, prefer to leave
+                 * it alone.
+                 */
+                if (current->wakee_flips > (factor * p->wakee_flips))
+                        return 1;
+        }
+        return 0;
+}
 static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync)
 {
        s64 this_load, load;
@@ -3164,6 +3205,13 @@ static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync)
        unsigned long weight;
        int balanced;
+        /*
+         * If we wake multiple tasks be careful to not bounce
+         * ourselves around too much.
+         */
+        if (wake_wide(p))
+                return 0;
        idx       = sd->wake_idx;
        this_cpu  = smp_processor_id();
        prev_cpu  = task_cpu(p);
@@ -4171,47 +4219,48 @@ static void update_blocked_averages(int cpu)
 }
 /*
- * Compute the cpu's hierarchical load factor for each task group.
+ * Compute the hierarchical load factor for cfs_rq and all its ascendants.
 * This needs to be done in a top-down fashion because the load of a child
 * group is a fraction of its parents load.
 */
-static int tg_load_down(struct task_group *tg, void *data)
+static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq)
-{
-        unsigned long load;
-        long cpu = (long)data;
-        if (!tg->parent) {
-                load = cpu_rq(cpu)->avg.load_avg_contrib;
-        } else {
-                load = tg->parent->cfs_rq[cpu]->h_load;
-                load = div64_ul(load * tg->se[cpu]->avg.load_avg_contrib,
-                                tg->parent->cfs_rq[cpu]->runnable_load_avg + 1);
-        }
-        tg->cfs_rq[cpu]->h_load = load;
-        return 0;
-}
-static void update_h_load(long cpu)
 {
-        struct rq *rq = cpu_rq(cpu);
+        struct rq *rq = rq_of(cfs_rq);
+        struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)];
        unsigned long now = jiffies;
+        unsigned long load;
-        if (rq->h_load_throttle == now)
+        if (cfs_rq->last_h_load_update == now)
                return;
-        rq->h_load_throttle = now;
+        cfs_rq->h_load_next = NULL;
+        for_each_sched_entity(se) {
+                cfs_rq = cfs_rq_of(se);
+                cfs_rq->h_load_next = se;
+                if (cfs_rq->last_h_load_update == now)
+                        break;
+        }
-        rcu_read_lock();
+        if (!se) {
-        walk_tg_tree(tg_load_down, tg_nop, (void *)cpu);
+                cfs_rq->h_load = rq->avg.load_avg_contrib;
-        rcu_read_unlock();
+                cfs_rq->last_h_load_update = now;
+        }
+        while ((se = cfs_rq->h_load_next) != NULL) {
+                load = cfs_rq->h_load;
+                load = div64_ul(load * se->avg.load_avg_contrib,
+                                cfs_rq->runnable_load_avg + 1);
+                cfs_rq = group_cfs_rq(se);
+                cfs_rq->h_load = load;
+                cfs_rq->last_h_load_update = now;
+        }
 }
 static unsigned long task_h_load(struct task_struct *p)
 {
        struct cfs_rq *cfs_rq = task_cfs_rq(p);
+        update_cfs_rq_h_load(cfs_rq);
        return div64_ul(p->se.avg.load_avg_contrib * cfs_rq->h_load,
                        cfs_rq->runnable_load_avg + 1);
 }
@@ -4220,10 +4269,6 @@ static inline void update_blocked_averages(int cpu)
 {
 }
-static inline void update_h_load(long cpu)
-{
-}
 static unsigned long task_h_load(struct task_struct *p)
 {
        return p->se.avg.load_avg_contrib;
@@ -4232,54 +4277,62 @@ static unsigned long task_h_load(struct task_struct *p)
 /********** Helpers for find_busiest_group ************************/
 /*
- * sd_lb_stats - Structure to store the statistics of a sched_domain
- *              during load balancing.
- */
-struct sd_lb_stats {
-        struct sched_group *busiest; /* Busiest group in this sd */
-        struct sched_group *this;  /* Local group in this sd */
-        unsigned long total_load;  /* Total load of all groups in sd */
-        unsigned long total_pwr;   /*   Total power of all groups in sd */
-        unsigned long avg_load;    /* Average load across all groups in sd */
-        /** Statistics of this group */
-        unsigned long this_load;
-        unsigned long this_load_per_task;
-        unsigned long this_nr_running;
-        unsigned long this_has_capacity;
-        unsigned int  this_idle_cpus;
-        /* Statistics of the busiest group */
-        unsigned int  busiest_idle_cpus;
-        unsigned long max_load;
-        unsigned long busiest_load_per_task;
-        unsigned long busiest_nr_running;
-        unsigned long busiest_group_capacity;
-        unsigned long busiest_has_capacity;
-        unsigned int  busiest_group_weight;
-        int group_imb; /* Is there imbalance in this sd */
-};
-/*
 * sg_lb_stats - stats of a sched_group required for load_balancing
 */
 struct sg_lb_stats {
        unsigned long avg_load; /*Avg load across the CPUs of the group */
        unsigned long group_load; /* Total load over the CPUs of the group */
-        unsigned long sum_nr_running; /* Nr tasks running in the group */
        unsigned long sum_weighted_load; /* Weighted load of group's tasks */
-        unsigned long group_capacity;
+        unsigned long load_per_task;
-        unsigned long idle_cpus;
+        unsigned long group_power;
-        unsigned long group_weight;
+        unsigned int sum_nr_running; /* Nr tasks running in the group */
+        unsigned int group_capacity;
+        unsigned int idle_cpus;
+        unsigned int group_weight;
        int group_imb; /* Is there an imbalance in the group ? */
        int group_has_capacity; /* Is there extra capacity in the group? */
 };
+/*
+ * sd_lb_stats - Structure to store the statistics of a sched_domain
+ *               during load balancing.
+ */
+struct sd_lb_stats {
+        struct sched_group *busiest;    /* Busiest group in this sd */
+        struct sched_group *local;      /* Local group in this sd */
+        unsigned long total_load;       /* Total load of all groups in sd */
+        unsigned long total_pwr;        /* Total power of all groups in sd */
+        unsigned long avg_load; /* Average load across all groups in sd */
+        struct sg_lb_stats busiest_stat;/* Statistics of the busiest group */
+        struct sg_lb_stats local_stat;  /* Statistics of the local group */
+};
+static inline void init_sd_lb_stats(struct sd_lb_stats *sds)
+{
+        /*
+         * Skimp on the clearing to avoid duplicate work. We can avoid clearing
+         * local_stat because update_sg_lb_stats() does a full clear/assignment.
+         * We must however clear busiest_stat::avg_load because
+         * update_sd_pick_busiest() reads this before assignment.
+         */
+        *sds = (struct sd_lb_stats){
+                .busiest = NULL,
+                .local = NULL,
+                .total_load = 0UL,
+                .total_pwr = 0UL,
+                .busiest_stat = {
+                        .avg_load = 0UL,
+                },
+        };
+}
 /**
 * get_sd_load_idx - Obtain the load index for a given sched domain.
 * @sd: The sched_domain whose load_idx is to be obtained.
 * @idle: The Idle status of the CPU for whose sd load_icx is obtained.
+ *
+ * Return: The load index.
 */
 static inline int get_sd_load_idx(struct sched_domain *sd,
                                        enum cpu_idle_type idle)
@@ -4457,33 +4510,99 @@ fix_small_capacity(struct sched_domain *sd, struct sched_group *group)
        return 0;
 }
+/*
+ * Group imbalance indicates (and tries to solve) the problem where balancing
+ * groups is inadequate due to tsk_cpus_allowed() constraints.
+ *
+ * Imagine a situation of two groups of 4 cpus each and 4 tasks each with a
+ * cpumask covering 1 cpu of the first group and 3 cpus of the second group.
+ * Something like:
+ *
+ *      { 0 1 2 3 } { 4 5 6 7 }
+ *              *     * * *
+ *
+ * If we were to balance group-wise we'd place two tasks in the first group and
+ * two tasks in the second group. Clearly this is undesired as it will overload
+ * cpu 3 and leave one of the cpus in the second group unused.
+ *
+ * The current solution to this issue is detecting the skew in the first group
+ * by noticing it has a cpu that is overloaded while the remaining cpus are
+ * idle -- or rather, there's a distinct imbalance in the cpus; see
+ * sg_imbalanced().
+ *
+ * When this is so detected; this group becomes a candidate for busiest; see
+ * update_sd_pick_busiest(). And calculcate_imbalance() and
+ * find_busiest_group() avoid some of the usual balance conditional to allow it
+ * to create an effective group imbalance.
+ *
+ * This is a somewhat tricky proposition since the next run might not find the
+ * group imbalance and decide the groups need to be balanced again. A most
+ * subtle and fragile situation.
+ */
+struct sg_imb_stats {
+        unsigned long max_nr_running, min_nr_running;
+        unsigned long max_cpu_load, min_cpu_load;
+};
+static inline void init_sg_imb_stats(struct sg_imb_stats *sgi)
+{
+        sgi->max_cpu_load = sgi->max_nr_running = 0UL;
+        sgi->min_cpu_load = sgi->min_nr_running = ~0UL;
+}
+static inline void
+update_sg_imb_stats(struct sg_imb_stats *sgi,
+                    unsigned long load, unsigned long nr_running)
+{
+        if (load > sgi->max_cpu_load)
+                sgi->max_cpu_load = load;
+        if (sgi->min_cpu_load > load)
+                sgi->min_cpu_load = load;
+        if (nr_running > sgi->max_nr_running)
+                sgi->max_nr_running = nr_running;
+        if (sgi->min_nr_running > nr_running)
+                sgi->min_nr_running = nr_running;
+}
+static inline int
+sg_imbalanced(struct sg_lb_stats *sgs, struct sg_imb_stats *sgi)
+{
+        /*
+         * Consider the group unbalanced when the imbalance is larger
+         * than the average weight of a task.
+         *
+         * APZ: with cgroup the avg task weight can vary wildly and
+         *      might not be a suitable number - should we keep a
+         *      normalized nr_running number somewhere that negates
+         *      the hierarchy?
+         */
+        if ((sgi->max_cpu_load - sgi->min_cpu_load) >= sgs->load_per_task &&
+            (sgi->max_nr_running - sgi->min_nr_running) > 1)
+                return 1;
+        return 0;
+}
 /**
 * update_sg_lb_stats - Update sched_group's statistics for load balancing.
 * @env: The load balancing environment.
 * @group: sched_group whose statistics are to be updated.
 * @load_idx: Load index of sched_domain of this_cpu for load calc.
 * @local_group: Does group contain this_cpu.
- * @balance: Should we balance.
 * @sgs: variable to hold the statistics for this group.
 */
 static inline void update_sg_lb_stats(struct lb_env *env,
                        struct sched_group *group, int load_idx,
-                        int local_group, int *balance, struct sg_lb_stats *sgs)
+                        int local_group, struct sg_lb_stats *sgs)
 {
-        unsigned long nr_running, max_nr_running, min_nr_running;
+        struct sg_imb_stats sgi;
-        unsigned long load, max_cpu_load, min_cpu_load;
+        unsigned long nr_running;
-        unsigned int balance_cpu = -1, first_idle_cpu = 0;
+        unsigned long load;
-        unsigned long avg_load_per_task = 0;
        int i;
-        if (local_group)
+        init_sg_imb_stats(&sgi);
-                balance_cpu = group_balance_cpu(group);
-        /* Tally up the load of all CPUs in the group */
-        max_cpu_load = 0;
-        min_cpu_load = ~0UL;
-        max_nr_running = 0;
-        min_nr_running = ~0UL;
        for_each_cpu_and(i, sched_group_cpus(group), env->cpus) {
                struct rq *rq = cpu_rq(i);
@@ -4492,24 +4611,10 @@ static inline void update_sg_lb_stats(struct lb_env *env,
                /* Bias balancing toward cpus of our domain */
                if (local_group) {
-                        if (idle_cpu(i) && !first_idle_cpu &&
-                                        cpumask_test_cpu(i, sched_group_mask(group))) {
-                                first_idle_cpu = 1;
-                                balance_cpu = i;
-                        }
                        load = target_load(i, load_idx);
                } else {
                        load = source_load(i, load_idx);
-                        if (load > max_cpu_load)
+                        update_sg_imb_stats(&sgi, load, nr_running);
-                                max_cpu_load = load;
-                        if (min_cpu_load > load)
-                                min_cpu_load = load;
-                        if (nr_running > max_nr_running)
-                                max_nr_running = nr_running;
-                        if (min_nr_running > nr_running)
-                                min_nr_running = nr_running;
                }
                sgs->group_load += load;
@@ -4519,46 +4624,25 @@ static inline void update_sg_lb_stats(struct lb_env *env,
                        sgs->idle_cpus++;
        }
-        /*
+        if (local_group && (env->idle != CPU_NEWLY_IDLE ||
-         * First idle cpu or the first cpu(busiest) in this sched group
+                        time_after_eq(jiffies, group->sgp->next_update)))
-         * is eligible for doing load balancing at this and above
+                update_group_power(env->sd, env->dst_cpu);
-         * domains. In the newly idle case, we will allow all the cpu's
-         * to do the newly idle load balance.
-         */
-        if (local_group) {
-                if (env->idle != CPU_NEWLY_IDLE) {
-                        if (balance_cpu != env->dst_cpu) {
-                                *balance = 0;
-                                return;
-                        }
-                        update_group_power(env->sd, env->dst_cpu);
-                } else if (time_after_eq(jiffies, group->sgp->next_update))
-                        update_group_power(env->sd, env->dst_cpu);
-        }
        /* Adjust by relative CPU power of the group */
-        sgs->avg_load = (sgs->group_load*SCHED_POWER_SCALE) / group->sgp->power;
+        sgs->group_power = group->sgp->power;
+        sgs->avg_load = (sgs->group_load*SCHED_POWER_SCALE) / sgs->group_power;
-        /*
-         * Consider the group unbalanced when the imbalance is larger
-         * than the average weight of a task.
-         *
-         * APZ: with cgroup the avg task weight can vary wildly and
-         *      might not be a suitable number - should we keep a
-         *      normalized nr_running number somewhere that negates
-         *      the hierarchy?
-         */
        if (sgs->sum_nr_running)
-                avg_load_per_task = sgs->sum_weighted_load / sgs->sum_nr_running;
+                sgs->load_per_task = sgs->sum_weighted_load / sgs->sum_nr_running;
+        sgs->group_imb = sg_imbalanced(sgs, &sgi);
-        if ((max_cpu_load - min_cpu_load) >= avg_load_per_task &&
+        sgs->group_capacity =
-            (max_nr_running - min_nr_running) > 1)
+                DIV_ROUND_CLOSEST(sgs->group_power, SCHED_POWER_SCALE);
-                sgs->group_imb = 1;
-        sgs->group_capacity = DIV_ROUND_CLOSEST(group->sgp->power,
-                                                SCHED_POWER_SCALE);
        if (!sgs->group_capacity)
                sgs->group_capacity = fix_small_capacity(env->sd, group);
        sgs->group_weight = group->group_weight;
        if (sgs->group_capacity > sgs->sum_nr_running)
@@ -4574,13 +4658,16 @@ static inline void update_sg_lb_stats(struct lb_env *env,
 *
 * Determine if @sg is a busier group than the previously selected
 * busiest group.
+ *
+ * Return: %true if @sg is a busier group than the previously selected
+ * busiest group. %false otherwise.
 */
 static bool update_sd_pick_busiest(struct lb_env *env,
                                   struct sd_lb_stats *sds,
                                   struct sched_group *sg,
                                   struct sg_lb_stats *sgs)
 {
-        if (sgs->avg_load <= sds->max_load)
+        if (sgs->avg_load <= sds->busiest_stat.avg_load)
                return false;
        if (sgs->sum_nr_running > sgs->group_capacity)
@@ -4613,11 +4700,11 @@ static bool update_sd_pick_busiest(struct lb_env *env,
 * @sds: variable to hold the statistics for this sched_domain.
 */
 static inline void update_sd_lb_stats(struct lb_env *env,
-                                        int *balance, struct sd_lb_stats *sds)
+                                        struct sd_lb_stats *sds)
 {
        struct sched_domain *child = env->sd->child;
        struct sched_group *sg = env->sd->groups;
-        struct sg_lb_stats sgs;
+        struct sg_lb_stats tmp_sgs;
        int load_idx, prefer_sibling = 0;
        if (child && child->flags & SD_PREFER_SIBLING)
@@ -4626,17 +4713,17 @@ static inline void update_sd_lb_stats(struct lb_env *env,
        load_idx = get_sd_load_idx(env->sd, env->idle);
        do {
+                struct sg_lb_stats *sgs = &tmp_sgs;
                int local_group;
                local_group = cpumask_test_cpu(env->dst_cpu, sched_group_cpus(sg));
-                memset(&sgs, 0, sizeof(sgs));
+                if (local_group) {
-                update_sg_lb_stats(env, sg, load_idx, local_group, balance, &sgs);
+                        sds->local = sg;
+                        sgs = &sds->local_stat;
-                if (local_group && !(*balance))
+                }
-                        return;
-                sds->total_load += sgs.group_load;
+                memset(sgs, 0, sizeof(*sgs));
-                sds->total_pwr += sg->sgp->power;
+                update_sg_lb_stats(env, sg, load_idx, local_group, sgs);
                /*
                 * In case the child domain prefers tasks go to siblings
@@ -4648,26 +4735,17 @@ static inline void update_sd_lb_stats(struct lb_env *env,
                 * heaviest group when it is already under-utilized (possible
                 * with a large weight task outweighs the tasks on the system).
                 */
-                if (prefer_sibling && !local_group && sds->this_has_capacity)
+                if (prefer_sibling && !local_group &&
-                        sgs.group_capacity = min(sgs.group_capacity, 1UL);
+                                sds->local && sds->local_stat.group_has_capacity)
+                        sgs->group_capacity = min(sgs->group_capacity, 1U);
-                if (local_group) {
+                /* Now, start updating sd_lb_stats */
-                        sds->this_load = sgs.avg_load;
+                sds->total_load += sgs->group_load;
-                        sds->this = sg;
+                sds->total_pwr += sgs->group_power;
-                        sds->this_nr_running = sgs.sum_nr_running;
-                        sds->this_load_per_task = sgs.sum_weighted_load;
+                if (!local_group && update_sd_pick_busiest(env, sds, sg, sgs)) {
-                        sds->this_has_capacity = sgs.group_has_capacity;
-                        sds->this_idle_cpus = sgs.idle_cpus;
-                } else if (update_sd_pick_busiest(env, sds, sg, &sgs)) {
-                        sds->max_load = sgs.avg_load;
                        sds->busiest = sg;
-                        sds->busiest_nr_running = sgs.sum_nr_running;
+                        sds->busiest_stat = *sgs;
-                        sds->busiest_idle_cpus = sgs.idle_cpus;
-                        sds->busiest_group_capacity = sgs.group_capacity;
-                        sds->busiest_load_per_task = sgs.sum_weighted_load;
-                        sds->busiest_has_capacity = sgs.group_has_capacity;
-                        sds->busiest_group_weight = sgs.group_weight;
-                        sds->group_imb = sgs.group_imb;
                }
                sg = sg->next;
@@ -4691,7 +4769,7 @@ static inline void update_sd_lb_stats(struct lb_env *env,
 * assuming lower CPU number will be equivalent to lower a SMT thread
 * number.
 *
- * Returns 1 when packing is required and a task should be moved to
+ * Return: 1 when packing is required and a task should be moved to
 * this CPU.  The amount of the imbalance is returned in *imbalance.
 *
 * @env: The load balancing environment.
@@ -4712,7 +4790,8 @@ static int check_asym_packing(struct lb_env *env, struct sd_lb_stats *sds)
                return 0;
        env->imbalance = DIV_ROUND_CLOSEST(
-                sds->max_load * sds->busiest->sgp->power, SCHED_POWER_SCALE);
+                sds->busiest_stat.avg_load * sds->busiest_stat.group_power,
+                SCHED_POWER_SCALE);
        return 1;
 }
@@ -4730,24 +4809,23 @@ void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds)
        unsigned long tmp, pwr_now = 0, pwr_move = 0;
        unsigned int imbn = 2;
        unsigned long scaled_busy_load_per_task;
+        struct sg_lb_stats *local, *busiest;
-        if (sds->this_nr_running) {
+        local = &sds->local_stat;
-                sds->this_load_per_task /= sds->this_nr_running;
+        busiest = &sds->busiest_stat;
-                if (sds->busiest_load_per_task >
-                                sds->this_load_per_task)
+        if (!local->sum_nr_running)
-                        imbn = 1;
+                local->load_per_task = cpu_avg_load_per_task(env->dst_cpu);
-        } else {
+        else if (busiest->load_per_task > local->load_per_task)
-                sds->this_load_per_task =
+                imbn = 1;
-                        cpu_avg_load_per_task(env->dst_cpu);
-        }
-        scaled_busy_load_per_task = sds->busiest_load_per_task
+        scaled_busy_load_per_task =
-                                         * SCHED_POWER_SCALE;
+                (busiest->load_per_task * SCHED_POWER_SCALE) /
-        scaled_busy_load_per_task /= sds->busiest->sgp->power;
+                busiest->group_power;
-        if (sds->max_load - sds->this_load + scaled_busy_load_per_task >=
+        if (busiest->avg_load - local->avg_load + scaled_busy_load_per_task >=
-                        (scaled_busy_load_per_task * imbn)) {
+            (scaled_busy_load_per_task * imbn)) {
-                env->imbalance = sds->busiest_load_per_task;
+                env->imbalance = busiest->load_per_task;
                return;
        }
@@ -4757,34 +4835,37 @@ void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds)
         * moving them.
         */
-        pwr_now += sds->busiest->sgp->power *
+        pwr_now += busiest->group_power *
-                        min(sds->busiest_load_per_task, sds->max_load);
+                        min(busiest->load_per_task, busiest->avg_load);
-        pwr_now += sds->this->sgp->power *
+        pwr_now += local->group_power *
-                        min(sds->this_load_per_task, sds->this_load);
+                        min(local->load_per_task, local->avg_load);
        pwr_now /= SCHED_POWER_SCALE;
        /* Amount of load we'd subtract */
-        tmp = (sds->busiest_load_per_task * SCHED_POWER_SCALE) /
+        tmp = (busiest->load_per_task * SCHED_POWER_SCALE) /
-                sds->busiest->sgp->power;
+                busiest->group_power;
-        if (sds->max_load > tmp)
+        if (busiest->avg_load > tmp) {
-                pwr_move += sds->busiest->sgp->power *
+                pwr_move += busiest->group_power *
-                        min(sds->busiest_load_per_task, sds->max_load - tmp);
+                            min(busiest->load_per_task,
+                                busiest->avg_load - tmp);
+        }
        /* Amount of load we'd add */
-        if (sds->max_load * sds->busiest->sgp->power <
+        if (busiest->avg_load * busiest->group_power <
-                sds->busiest_load_per_task * SCHED_POWER_SCALE)
+            busiest->load_per_task * SCHED_POWER_SCALE) {
-                tmp = (sds->max_load * sds->busiest->sgp->power) /
+                tmp = (busiest->avg_load * busiest->group_power) /
-                        sds->this->sgp->power;
+                      local->group_power;
-        else
+        } else {
-                tmp = (sds->busiest_load_per_task * SCHED_POWER_SCALE) /
+                tmp = (busiest->load_per_task * SCHED_POWER_SCALE) /
-                        sds->this->sgp->power;
+                      local->group_power;
-        pwr_move += sds->this->sgp->power *
+        }
-                        min(sds->this_load_per_task, sds->this_load + tmp);
+        pwr_move += local->group_power *
+                    min(local->load_per_task, local->avg_load + tmp);
        pwr_move /= SCHED_POWER_SCALE;
        /* Move if we gain throughput */
        if (pwr_move > pwr_now)
-                env->imbalance = sds->busiest_load_per_task;
+                env->imbalance = busiest->load_per_task;
 }
 /**
@@ -4796,11 +4877,18 @@ void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds)
 static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds)
 {
        unsigned long max_pull, load_above_capacity = ~0UL;
+        struct sg_lb_stats *local, *busiest;
-        sds->busiest_load_per_task /= sds->busiest_nr_running;
+        local = &sds->local_stat;
-        if (sds->group_imb) {
+        busiest = &sds->busiest_stat;
-                sds->busiest_load_per_task =
-                        min(sds->busiest_load_per_task, sds->avg_load);
+        if (busiest->group_imb) {
+                /*
+                 * In the group_imb case we cannot rely on group-wide averages
+                 * to ensure cpu-load equilibrium, look at wider averages. XXX
+                 */
+                busiest->load_per_task =
+                        min(busiest->load_per_task, sds->avg_load);
        }
        /*
@@ -4808,21 +4896,22 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s
         * max load less than avg load(as we skip the groups at or below
         * its cpu_power, while calculating max_load..)
         */
-        if (sds->max_load < sds->avg_load) {
+        if (busiest->avg_load < sds->avg_load) {
                env->imbalance = 0;
                return fix_small_imbalance(env, sds);
        }
-        if (!sds->group_imb) {
+        if (!busiest->group_imb) {
                /*
                 * Don't want to pull so many tasks that a group would go idle.
+                 * Except of course for the group_imb case, since then we might
+                 * have to drop below capacity to reach cpu-load equilibrium.
                 */
-                load_above_capacity = (sds->busiest_nr_running -
+                load_above_capacity =
-                                                sds->busiest_group_capacity);
+                        (busiest->sum_nr_running - busiest->group_capacity);
                load_above_capacity *= (SCHED_LOAD_SCALE * SCHED_POWER_SCALE);
+                load_above_capacity /= busiest->group_power;
-                load_above_capacity /= sds->busiest->sgp->power;
        }
        /*
@@ -4832,15 +4921,14 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s
         * we also don't want to reduce the group load below the group capacity
         * (so that we can implement power-savings policies etc). Thus we look
         * for the minimum possible imbalance.
-         * Be careful of negative numbers as they'll appear as very large values
-         * with unsigned longs.
         */
-        max_pull = min(sds->max_load - sds->avg_load, load_above_capacity);
+        max_pull = min(busiest->avg_load - sds->avg_load, load_above_capacity);
        /* How much load to actually move to equalise the imbalance */
-        env->imbalance = min(max_pull * sds->busiest->sgp->power,
+        env->imbalance = min(
-                (sds->avg_load - sds->this_load) * sds->this->sgp->power)
+                max_pull * busiest->group_power,
-                        / SCHED_POWER_SCALE;
+                (sds->avg_load - local->avg_load) * local->group_power
+        ) / SCHED_POWER_SCALE;
        /*
         * if *imbalance is less than the average load per runnable task
@@ -4848,9 +4936,8 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s
         * a think about bumping its value to force at least one task to be
         * moved
         */
-        if (env->imbalance < sds->busiest_load_per_task)
+        if (env->imbalance < busiest->load_per_task)
                return fix_small_imbalance(env, sds);
 }
 /******* find_busiest_group() helpers end here *********************/
@@ -4866,69 +4953,62 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s
 * to restore balance.
 *
 * @env: The load balancing environment.
- * @balance: Pointer to a variable indicating if this_cpu
- *      is the appropriate cpu to perform load balancing at this_level.
 *
- * Returns:     - the busiest group if imbalance exists.
+ * Return:      - The busiest group if imbalance exists.
 *              - If no imbalance and user has opted for power-savings balance,
 *                 return the least loaded group whose CPUs can be
 *                 put to idle by rebalancing its tasks onto our group.
 */
-static struct sched_group *
+static struct sched_group *find_busiest_group(struct lb_env *env)
-find_busiest_group(struct lb_env *env, int *balance)
 {
+        struct sg_lb_stats *local, *busiest;
        struct sd_lb_stats sds;
-        memset(&sds, 0, sizeof(sds));
+        init_sd_lb_stats(&sds);
        /*
         * Compute the various statistics relavent for load balancing at
         * this level.
         */
-        update_sd_lb_stats(env, balance, &sds);
+        update_sd_lb_stats(env, &sds);
+        local = &sds.local_stat;
-        /*
+        busiest = &sds.busiest_stat;
-         * this_cpu is not the appropriate cpu to perform load balancing at
-         * this level.
-         */
-        if (!(*balance))
-                goto ret;
        if ((env->idle == CPU_IDLE || env->idle == CPU_NEWLY_IDLE) &&
            check_asym_packing(env, &sds))
                return sds.busiest;
        /* There is no busy sibling group to pull tasks from */
-        if (!sds.busiest || sds.busiest_nr_running == 0)
+        if (!sds.busiest || busiest->sum_nr_running == 0)
                goto out_balanced;
        sds.avg_load = (SCHED_POWER_SCALE * sds.total_load) / sds.total_pwr;
        /*
         * If the busiest group is imbalanced the below checks don't
-         * work because they assumes all things are equal, which typically
+         * work because they assume all things are equal, which typically
         * isn't true due to cpus_allowed constraints and the like.
         */
-        if (sds.group_imb)
+        if (busiest->group_imb)
                goto force_balance;
        /* SD_BALANCE_NEWIDLE trumps SMP nice when underutilized */
-        if (env->idle == CPU_NEWLY_IDLE && sds.this_has_capacity &&
+        if (env->idle == CPU_NEWLY_IDLE && local->group_has_capacity &&
-                        !sds.busiest_has_capacity)
+            !busiest->group_has_capacity)
                goto force_balance;
        /*
         * If the local group is more busy than the selected busiest group
         * don't try and pull any tasks.
         */
-        if (sds.this_load >= sds.max_load)
+        if (local->avg_load >= busiest->avg_load)
                goto out_balanced;
        /*
         * Don't pull any tasks if this group is already above the domain
         * average load.
         */
-        if (sds.this_load >= sds.avg_load)
+        if (local->avg_load >= sds.avg_load)
                goto out_balanced;
        if (env->idle == CPU_IDLE) {
@@ -4938,15 +5018,16 @@ find_busiest_group(struct lb_env *env, int *balance)
                 * there is no imbalance between this and busiest group
                 * wrt to idle cpu's, it is balanced.
                 */
-                if ((sds.this_idle_cpus <= sds.busiest_idle_cpus + 1) &&
+                if ((local->idle_cpus < busiest->idle_cpus) &&
-                    sds.busiest_nr_running <= sds.busiest_group_weight)
+                    busiest->sum_nr_running <= busiest->group_weight)
                        goto out_balanced;
        } else {
                /*
                 * In the CPU_NEWLY_IDLE, CPU_NOT_IDLE cases, use
                 * imbalance_pct to be conservative.
                 */
-                if (100 * sds.max_load <= env->sd->imbalance_pct * sds.this_load)
+                if (100 * busiest->avg_load <=
+                                env->sd->imbalance_pct * local->avg_load)
                        goto out_balanced;
        }
@@ -4956,7 +5037,6 @@ force_balance:
        return sds.busiest;
 out_balanced:
-ret:
        env->imbalance = 0;
        return NULL;
 }
@@ -4968,10 +5048,10 @@ static struct rq *find_busiest_queue(struct lb_env *env,
                                     struct sched_group *group)
 {
        struct rq *busiest = NULL, *rq;
-        unsigned long max_load = 0;
+        unsigned long busiest_load = 0, busiest_power = 1;
        int i;
-        for_each_cpu(i, sched_group_cpus(group)) {
+        for_each_cpu_and(i, sched_group_cpus(group), env->cpus) {
                unsigned long power = power_of(i);
                unsigned long capacity = DIV_ROUND_CLOSEST(power,
                                                           SCHED_POWER_SCALE);
@@ -4980,9 +5060,6 @@ static struct rq *find_busiest_queue(struct lb_env *env,
                if (!capacity)
                        capacity = fix_small_capacity(env->sd, group);
-                if (!cpumask_test_cpu(i, env->cpus))
-                        continue;
                rq = cpu_rq(i);
                wl = weighted_cpuload(i);
@@ -4998,11 +5075,15 @@ static struct rq *find_busiest_queue(struct lb_env *env,
                 * the weighted_cpuload() scaled with the cpu power, so that
                 * the load can be moved away from the cpu that is potentially
                 * running at a lower capacity.
+                 *
+                 * Thus we're looking for max(wl_i / power_i), crosswise
+                 * multiplication to rid ourselves of the division works out
+                 * to: wl_i * power_j > wl_j * power_i;  where j is our
+                 * previous maximum.
                 */
-                wl = (wl * SCHED_POWER_SCALE) / power;
+                if (wl * busiest_power > busiest_load * power) {
+                        busiest_load = wl;
-                if (wl > max_load) {
+                        busiest_power = power;
-                        max_load = wl;
                        busiest = rq;
                }
        }
@@ -5039,13 +5120,47 @@ static int need_active_balance(struct lb_env *env)
 static int active_load_balance_cpu_stop(void *data);
+static int should_we_balance(struct lb_env *env)
+{
+        struct sched_group *sg = env->sd->groups;
+        struct cpumask *sg_cpus, *sg_mask;
+        int cpu, balance_cpu = -1;
+        /*
+         * In the newly idle case, we will allow all the cpu's
+         * to do the newly idle load balance.
+         */
+        if (env->idle == CPU_NEWLY_IDLE)
+                return 1;
+        sg_cpus = sched_group_cpus(sg);
+        sg_mask = sched_group_mask(sg);
+        /* Try to find first idle cpu */
+        for_each_cpu_and(cpu, sg_cpus, env->cpus) {
+                if (!cpumask_test_cpu(cpu, sg_mask) || !idle_cpu(cpu))
+                        continue;
+                balance_cpu = cpu;
+                break;
+        }
+        if (balance_cpu == -1)
+                balance_cpu = group_balance_cpu(sg);
+        /*
+         * First idle cpu or the first cpu(busiest) in this sched group
+         * is eligible for doing load balancing at this and above domains.
+         */
+        return balance_cpu != env->dst_cpu;
+}
 /*
 * Check this_cpu to ensure it is balanced within domain. Attempt to move
 * tasks if there is an imbalance.
 */
 static int load_balance(int this_cpu, struct rq *this_rq,
                        struct sched_domain *sd, enum cpu_idle_type idle,
-                        int *balance)
+                        int *continue_balancing)
 {
        int ld_moved, cur_ld_moved, active_balance = 0;
        struct sched_group *group;
@@ -5075,11 +5190,12 @@ static int load_balance(int this_cpu, struct rq *this_rq,
        schedstat_inc(sd, lb_count[idle]);
 redo:
-        group = find_busiest_group(&env, balance);
+        if (!should_we_balance(&env)) {
+                *continue_balancing = 0;
-        if (*balance == 0)
                goto out_balanced;
+        }
+        group = find_busiest_group(&env);
        if (!group) {
                schedstat_inc(sd, lb_nobusyg[idle]);
                goto out_balanced;
@@ -5108,7 +5224,6 @@ redo:
                env.src_rq    = busiest;
                env.loop_max  = min(sysctl_sched_nr_migrate, busiest->nr_running);
-                update_h_load(env.src_cpu);
 more_balance:
                local_irq_save(flags);
                double_rq_lock(env.dst_rq, busiest);
@@ -5292,7 +5407,7 @@ void idle_balance(int this_cpu, struct rq *this_rq)
        rcu_read_lock();
        for_each_domain(this_cpu, sd) {
                unsigned long interval;
-                int balance = 1;
+                int continue_balancing = 1;
                if (!(sd->flags & SD_LOAD_BALANCE))
                        continue;
@@ -5300,7 +5415,8 @@ void idle_balance(int this_cpu, struct rq *this_rq)
                if (sd->flags & SD_BALANCE_NEWIDLE) {
                        /* If we've pulled tasks over stop searching: */
                        pulled_task = load_balance(this_cpu, this_rq,
-                                                   sd, CPU_NEWLY_IDLE, &balance);
+                                                   sd, CPU_NEWLY_IDLE,
+                                                   &continue_balancing);
                }
                interval = msecs_to_jiffies(sd->balance_interval);
@@ -5506,7 +5622,7 @@ void nohz_balance_enter_idle(int cpu)
        set_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu));
 }
-static int __cpuinit sched_ilb_notifier(struct notifier_block *nfb,
+static int sched_ilb_notifier(struct notifier_block *nfb,
                                        unsigned long action, void *hcpu)
 {
        switch (action & ~CPU_TASKS_FROZEN) {
@@ -5538,7 +5654,7 @@ void update_max_interval(void)
 */
 static void rebalance_domains(int cpu, enum cpu_idle_type idle)
 {
-        int balance = 1;
+        int continue_balancing = 1;
        struct rq *rq = cpu_rq(cpu);
        unsigned long interval;
        struct sched_domain *sd;
@@ -5570,7 +5686,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
                }
                if (time_after_eq(jiffies, sd->last_balance + interval)) {
-                        if (load_balance(cpu, rq, sd, idle, &balance)) {
+                        if (load_balance(cpu, rq, sd, idle, &continue_balancing)) {
                                /*
                                 * The LBF_SOME_PINNED logic could have changed
                                 * env->dst_cpu, so we can't know our idle
@@ -5593,7 +5709,7 @@ out:
                 * CPU in our sched group which is doing load balancing more
                 * actively.
                 */
-                if (!balance)
+                if (!continue_balancing)
                        break;
        }
        rcu_read_unlock();
@@ -5786,7 +5902,7 @@ static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued)
                entity_tick(cfs_rq, se, queued);
        }
-        if (sched_feat_numa(NUMA))
+        if (numabalancing_enabled)
                task_tick_numa(rq, curr);
        update_rq_runnable_avg(rq, 1);
@@ -5889,11 +6005,9 @@ static void switched_from_fair(struct rq *rq, struct task_struct *p)
        * and ensure we don't carry in an old decay_count if we
        * switch back.
        */
-        if (p->se.avg.decay_count) {
+        if (se->avg.decay_count) {
-                struct cfs_rq *cfs_rq = cfs_rq_of(&p->se);
+                __synchronize_entity_decay(se);
-                __synchronize_entity_decay(&p->se);
+                subtract_blocked_load_contrib(cfs_rq, se->avg.load_avg_contrib);
-                subtract_blocked_load_contrib(cfs_rq,
-                                p->se.avg.load_avg_contrib);
        }
 #endif
 }
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index ef0a7b2439dd..b3c5653e1dca 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -285,7 +285,6 @@ struct cfs_rq {
        /* Required to track per-cpu representation of a task_group */
        u32 tg_runnable_contrib;
        unsigned long tg_load_contrib;
-#endif /* CONFIG_FAIR_GROUP_SCHED */
        /*
         *   h_load = weight * f(tg)
@@ -294,6 +293,9 @@ struct cfs_rq {
         * this group.
         */
        unsigned long h_load;
+        u64 last_h_load_update;
+        struct sched_entity *h_load_next;
+#endif /* CONFIG_FAIR_GROUP_SCHED */
 #endif /* CONFIG_SMP */
 #ifdef CONFIG_FAIR_GROUP_SCHED
@@ -429,9 +431,6 @@ struct rq {
 #ifdef CONFIG_FAIR_GROUP_SCHED
        /* list of leaf cfs_rq on this cpu: */
        struct list_head leaf_cfs_rq_list;
-#ifdef CONFIG_SMP
-        unsigned long h_load_throttle;
-#endif /* CONFIG_SMP */
 #endif /* CONFIG_FAIR_GROUP_SCHED */
 #ifdef CONFIG_RT_GROUP_SCHED
@@ -595,6 +594,7 @@ static inline struct sched_domain *highest_flag_domain(int cpu, int flag)
 }
 DECLARE_PER_CPU(struct sched_domain *, sd_llc);
+DECLARE_PER_CPU(int, sd_llc_size);
 DECLARE_PER_CPU(int, sd_llc_id);
 struct sched_group_power {
@@ -665,9 +665,9 @@ extern int group_balance_cpu(struct sched_group *sg);
 /*
 * Return the group to which this tasks belongs.
 *
- * We cannot use task_subsys_state() and friends because the cgroup
+ * We cannot use task_css() and friends because the cgroup subsystem
- * subsystem changes that value before the cgroup_subsys::attach() method
+ * changes that value before the cgroup_subsys::attach() method is called,
- * is called, therefore we cannot pin it and might observe the wrong value.
+ * therefore we cannot pin it and might observe the wrong value.
 *
 * The same is true for autogroup's p->signal->autogroup->tg, the autogroup
 * core changes this before calling sched_move_task().
diff --git a/kernel/smp.c b/kernel/smp.c
index 4dba0f7b72ad..449b707fc20d 100644
--- a/kernel/smp.c
+++ b/kernel/smp.c
@@ -73,7 +73,7 @@ hotplug_cfd(struct notifier_block *nfb, unsigned long action, void *hcpu)
        return NOTIFY_OK;
 }
-static struct notifier_block __cpuinitdata hotplug_cfd_notifier = {
+static struct notifier_block hotplug_cfd_notifier = {
        .notifier_call          = hotplug_cfd,
 };
@@ -186,25 +186,13 @@ void generic_smp_call_function_single_interrupt(void)
        while (!list_empty(&list)) {
                struct call_single_data *csd;
-                unsigned int csd_flags;
                csd = list_entry(list.next, struct call_single_data, list);
                list_del(&csd->list);
-                /*
-                 * 'csd' can be invalid after this call if flags == 0
-                 * (when called through generic_exec_single()),
-                 * so save them away before making the call:
-                 */
-                csd_flags = csd->flags;
                csd->func(csd->info);
-                /*
+                csd_unlock(csd);
-                 * Unlocked CSDs are valid through generic_exec_single():
-                 */
-                if (csd_flags & CSD_FLAG_LOCK)
-                        csd_unlock(csd);
        }
 }
@@ -278,8 +266,6 @@ EXPORT_SYMBOL(smp_call_function_single);
 * @wait: If true, wait until function has completed.
 *
 * Returns 0 on success, else a negative status code (if no cpus were online).
- * Note that @wait will be implicitly turned on in case of allocation failures,
- * since we fall back to on-stack allocation.
 *
 * Selection preference:
 *      1) current cpu if in @mask
diff --git a/kernel/smpboot.c b/kernel/smpboot.c
index 02fc5c933673..eb89e1807408 100644
--- a/kernel/smpboot.c
+++ b/kernel/smpboot.c
@@ -24,7 +24,7 @@
 */
 static DEFINE_PER_CPU(struct task_struct *, idle_threads);
-struct task_struct * __cpuinit idle_thread_get(unsigned int cpu)
+struct task_struct *idle_thread_get(unsigned int cpu)
 {
        struct task_struct *tsk = per_cpu(idle_threads, cpu);
diff --git a/kernel/softirq.c b/kernel/softirq.c
index ca25e6e704a2..be3d3514c325 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -699,7 +699,7 @@ void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 }
 EXPORT_SYMBOL(send_remote_softirq);
-static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
+static int remote_softirq_cpu_notify(struct notifier_block *self,
                                               unsigned long action, void *hcpu)
 {
        /*
@@ -728,7 +728,7 @@ static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
        return NOTIFY_OK;
 }
-static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
+static struct notifier_block remote_softirq_cpu_notifier = {
        .notifier_call  = remote_softirq_cpu_notify,
 };
@@ -830,7 +830,7 @@ static void takeover_tasklets(unsigned int cpu)
 }
 #endif /* CONFIG_HOTPLUG_CPU */
-static int __cpuinit cpu_callback(struct notifier_block *nfb,
+static int cpu_callback(struct notifier_block *nfb,
                                  unsigned long action,
                                  void *hcpu)
 {
@@ -845,7 +845,7 @@ static int __cpuinit cpu_callback(struct notifier_block *nfb,
        return NOTIFY_OK;
 }
-static struct notifier_block __cpuinitdata cpu_nfb = {
+static struct notifier_block cpu_nfb = {
        .notifier_call = cpu_callback
 };
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index ac09d98490aa..07f6fc468e17 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -2346,7 +2346,11 @@ static int do_proc_dointvec_ms_jiffies_conv(bool *negp, unsigned long *lvalp,
                                            int write, void *data)
 {
        if (write) {
-                *valp = msecs_to_jiffies(*negp ? -*lvalp : *lvalp);
+                unsigned long jif = msecs_to_jiffies(*negp ? -*lvalp : *lvalp);
+                if (jif > INT_MAX)
+                        return 1;
+                *valp = (int)jif;
        } else {
                int val = *valp;
                unsigned long lval;
diff --git a/kernel/time/Kconfig b/kernel/time/Kconfig
index 70f27e89012b..2b62fe86f9ec 100644
--- a/kernel/time/Kconfig
+++ b/kernel/time/Kconfig
@@ -105,7 +105,6 @@ config NO_HZ_FULL
        select RCU_USER_QS
        select RCU_NOCB_CPU
        select VIRT_CPU_ACCOUNTING_GEN
-        select CONTEXT_TRACKING_FORCE
        select IRQ_WORK
        help
         Adaptively try to shutdown the tick whenever possible, even when
@@ -134,6 +133,56 @@ config NO_HZ_FULL_ALL
         Note the boot CPU will still be kept outside the range to
         handle the timekeeping duty.
+config NO_HZ_FULL_SYSIDLE
+        bool "Detect full-system idle state for full dynticks system"
+        depends on NO_HZ_FULL
+        default n
+        help
+         At least one CPU must keep the scheduling-clock tick running for
+         timekeeping purposes whenever there is a non-idle CPU, where
+         "non-idle" also includes dynticks CPUs as long as they are
+         running non-idle tasks.  Because the underlying adaptive-tick
+         support cannot distinguish between all CPUs being idle and
+         all CPUs each running a single task in dynticks mode, the
+         underlying support simply ensures that there is always a CPU
+         handling the scheduling-clock tick, whether or not all CPUs
+         are idle.  This Kconfig option enables scalable detection of
+         the all-CPUs-idle state, thus allowing the scheduling-clock
+         tick to be disabled when all CPUs are idle.  Note that scalable
+         detection of the all-CPUs-idle state means that larger systems
+         will be slower to declare the all-CPUs-idle state.
+         Say Y if you would like to help debug all-CPUs-idle detection.
+         Say N if you are unsure.
+config NO_HZ_FULL_SYSIDLE_SMALL
+        int "Number of CPUs above which large-system approach is used"
+        depends on NO_HZ_FULL_SYSIDLE
+        range 1 NR_CPUS
+        default 8
+        help
+         The full-system idle detection mechanism takes a lazy approach
+         on large systems, as is required to attain decent scalability.
+         However, on smaller systems, scalability is not anywhere near as
+         large a concern as is energy efficiency.  The sysidle subsystem
+         therefore uses a fast but non-scalable algorithm for small
+         systems and a lazier but scalable algorithm for large systems.
+         This Kconfig parameter defines the number of CPUs in the largest
+         system that will be considered to be "small".
+         The default value will be fine in most cases.  Battery-powered
+         systems that (1) enable NO_HZ_FULL_SYSIDLE, (2) have larger
+         numbers of CPUs, and (3) are suffering from battery-lifetime
+         problems due to long sysidle latencies might wish to experiment
+         with larger values for this Kconfig parameter.  On the other
+         hand, they might be even better served by disabling NO_HZ_FULL
+         entirely, given that NO_HZ_FULL is intended for HPC and
+         real-time workloads that at present do not tend to be run on
+         battery-powered systems.
+         Take the default if you are unsure.
 config NO_HZ
        bool "Old Idle dynticks config"
        depends on !ARCH_USES_GETTIMEOFFSET && GENERIC_CLOCKEVENTS
diff --git a/kernel/time/sched_clock.c b/kernel/time/sched_clock.c
index a326f27d7f09..0b479a6a22bb 100644
--- a/kernel/time/sched_clock.c
+++ b/kernel/time/sched_clock.c
@@ -121,7 +121,7 @@ void __init setup_sched_clock(u32 (*read)(void), int bits, unsigned long rate)
        BUG_ON(bits > 32);
        WARN_ON(!irqs_disabled());
        read_sched_clock = read;
-        sched_clock_mask = (1 << bits) - 1;
+        sched_clock_mask = (1ULL << bits) - 1;
        cd.rate = rate;
        /* calculate the mult/shift to convert counter ticks to ns. */
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 69601726a745..3612fc77f834 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -23,6 +23,7 @@
 #include <linux/irq_work.h>
 #include <linux/posix-timers.h>
 #include <linux/perf_event.h>
+#include <linux/context_tracking.h>
 #include <asm/irq_regs.h>
@@ -148,8 +149,8 @@ static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs)
 }
 #ifdef CONFIG_NO_HZ_FULL
-static cpumask_var_t nohz_full_mask;
+cpumask_var_t tick_nohz_full_mask;
-bool have_nohz_full_mask;
+bool tick_nohz_full_running;
 static bool can_stop_full_tick(void)
 {
@@ -182,7 +183,8 @@ static bool can_stop_full_tick(void)
                 * Don't allow the user to think they can get
                 * full NO_HZ with this machine.
                 */
-                WARN_ONCE(1, "NO_HZ FULL will not work with unstable sched clock");
+                WARN_ONCE(tick_nohz_full_running,
+                          "NO_HZ FULL will not work with unstable sched clock");
                return false;
        }
 #endif
@@ -196,7 +198,7 @@ static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now);
 * Re-evaluate the need for the tick on the current CPU
 * and restart it if necessary.
 */
-void tick_nohz_full_check(void)
+void __tick_nohz_full_check(void)
 {
        struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
@@ -210,7 +212,7 @@ void tick_nohz_full_check(void)
 static void nohz_full_kick_work_func(struct irq_work *work)
 {
-        tick_nohz_full_check();
+        __tick_nohz_full_check();
 }
 static DEFINE_PER_CPU(struct irq_work, nohz_full_kick_work) = {
@@ -229,7 +231,7 @@ void tick_nohz_full_kick(void)
 static void nohz_full_kick_ipi(void *info)
 {
-        tick_nohz_full_check();
+        __tick_nohz_full_check();
 }
 /*
@@ -238,12 +240,13 @@ static void nohz_full_kick_ipi(void *info)
 */
 void tick_nohz_full_kick_all(void)
 {
-        if (!have_nohz_full_mask)
+        if (!tick_nohz_full_running)
                return;
        preempt_disable();
-        smp_call_function_many(nohz_full_mask,
+        smp_call_function_many(tick_nohz_full_mask,
                               nohz_full_kick_ipi, NULL, false);
+        tick_nohz_full_kick();
        preempt_enable();
 }
@@ -252,7 +255,7 @@ void tick_nohz_full_kick_all(void)
 * It might need the tick due to per task/process properties:
 * perf events, posix cpu timers, ...
 */
-void tick_nohz_task_switch(struct task_struct *tsk)
+void __tick_nohz_task_switch(struct task_struct *tsk)
 {
        unsigned long flags;
@@ -268,37 +271,29 @@ out:
        local_irq_restore(flags);
 }
-int tick_nohz_full_cpu(int cpu)
-{
-        if (!have_nohz_full_mask)
-                return 0;
-        return cpumask_test_cpu(cpu, nohz_full_mask);
-}
 /* Parse the boot-time nohz CPU list from the kernel parameters. */
 static int __init tick_nohz_full_setup(char *str)
 {
        int cpu;
-        alloc_bootmem_cpumask_var(&nohz_full_mask);
+        alloc_bootmem_cpumask_var(&tick_nohz_full_mask);
-        if (cpulist_parse(str, nohz_full_mask) < 0) {
+        if (cpulist_parse(str, tick_nohz_full_mask) < 0) {
                pr_warning("NOHZ: Incorrect nohz_full cpumask\n");
                return 1;
        }
        cpu = smp_processor_id();
-        if (cpumask_test_cpu(cpu, nohz_full_mask)) {
+        if (cpumask_test_cpu(cpu, tick_nohz_full_mask)) {
                pr_warning("NO_HZ: Clearing %d from nohz_full range for timekeeping\n", cpu);
-                cpumask_clear_cpu(cpu, nohz_full_mask);
+                cpumask_clear_cpu(cpu, tick_nohz_full_mask);
        }
-        have_nohz_full_mask = true;
+        tick_nohz_full_running = true;
        return 1;
 }
 __setup("nohz_full=", tick_nohz_full_setup);
-static int __cpuinit tick_nohz_cpu_down_callback(struct notifier_block *nfb,
+static int tick_nohz_cpu_down_callback(struct notifier_block *nfb,
                                                 unsigned long action,
                                                 void *hcpu)
 {
@@ -310,7 +305,7 @@ static int __cpuinit tick_nohz_cpu_down_callback(struct notifier_block *nfb,
                 * If we handle the timekeeping duty for full dynticks CPUs,
                 * we can't safely shutdown that CPU.
                 */
-                if (have_nohz_full_mask && tick_do_timer_cpu == cpu)
+                if (tick_nohz_full_running && tick_do_timer_cpu == cpu)
                        return NOTIFY_BAD;
                break;
        }
@@ -329,14 +324,14 @@ static int tick_nohz_init_all(void)
        int err = -1;
 #ifdef CONFIG_NO_HZ_FULL_ALL
-        if (!alloc_cpumask_var(&nohz_full_mask, GFP_KERNEL)) {
+        if (!alloc_cpumask_var(&tick_nohz_full_mask, GFP_KERNEL)) {
                pr_err("NO_HZ: Can't allocate full dynticks cpumask\n");
                return err;
        }
        err = 0;
-        cpumask_setall(nohz_full_mask);
+        cpumask_setall(tick_nohz_full_mask);
-        cpumask_clear_cpu(smp_processor_id(), nohz_full_mask);
+        cpumask_clear_cpu(smp_processor_id(), tick_nohz_full_mask);
-        have_nohz_full_mask = true;
+        tick_nohz_full_running = true;
 #endif
        return err;
 }
@@ -345,17 +340,18 @@ void __init tick_nohz_init(void)
 {
        int cpu;
-        if (!have_nohz_full_mask) {
+        if (!tick_nohz_full_running) {
                if (tick_nohz_init_all() < 0)
                        return;
        }
+        for_each_cpu(cpu, tick_nohz_full_mask)
+                context_tracking_cpu_set(cpu);
        cpu_notifier(tick_nohz_cpu_down_callback, 0);
-        cpulist_scnprintf(nohz_full_buf, sizeof(nohz_full_buf), nohz_full_mask);
+        cpulist_scnprintf(nohz_full_buf, sizeof(nohz_full_buf), tick_nohz_full_mask);
        pr_info("NO_HZ: Full dynticks CPUs: %s.\n", nohz_full_buf);
 }
-#else
-#define have_nohz_full_mask (0)
 #endif
 /*
@@ -733,7 +729,7 @@ static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)
                return false;
        }
-        if (have_nohz_full_mask) {
+        if (tick_nohz_full_enabled()) {
                /*
                 * Keep the tick alive to guarantee timekeeping progression
                 * if there are full dynticks CPUs around
@@ -827,13 +823,10 @@ void tick_nohz_irq_exit(void)
 {
        struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
-        if (ts->inidle) {
+        if (ts->inidle)
-                /* Cancel the timer because CPU already waken up from the C-states*/
-                menu_hrtimer_cancel();
                __tick_nohz_idle_enter(ts);
-        } else {
+        else
                tick_nohz_full_stop_tick(ts);
-        }
 }
 /**
@@ -931,8 +924,6 @@ void tick_nohz_idle_exit(void)
        ts->inidle = 0;
-        /* Cancel the timer because CPU already waken up from the C-states*/
-        menu_hrtimer_cancel();
        if (ts->idle_active || ts->tick_stopped)
                now = ktime_get();
diff --git a/kernel/time/timer_list.c b/kernel/time/timer_list.c
index 3bdf28323012..61ed862cdd37 100644
--- a/kernel/time/timer_list.c
+++ b/kernel/time/timer_list.c
@@ -265,10 +265,9 @@ static inline void timer_list_header(struct seq_file *m, u64 now)
 static int timer_list_show(struct seq_file *m, void *v)
 {
        struct timer_list_iter *iter = v;
-        u64 now = ktime_to_ns(ktime_get());
        if (iter->cpu == -1 && !iter->second_pass)
-                timer_list_header(m, now);
+                timer_list_header(m, iter->now);
        else if (!iter->second_pass)
                print_cpu(m, iter->cpu, iter->now);
 #ifdef CONFIG_GENERIC_CLOCKEVENTS
@@ -298,33 +297,41 @@ void sysrq_timer_list_show(void)
        return;
 }
-static void *timer_list_start(struct seq_file *file, loff_t *offset)
+static void *move_iter(struct timer_list_iter *iter, loff_t offset)
 {
-        struct timer_list_iter *iter = file->private;
+        for (; offset; offset--) {
+                iter->cpu = cpumask_next(iter->cpu, cpu_online_mask);
-        if (!*offset) {
+                if (iter->cpu >= nr_cpu_ids) {
-                iter->cpu = -1;
-                iter->now = ktime_to_ns(ktime_get());
-        } else if (iter->cpu >= nr_cpu_ids) {
 #ifdef CONFIG_GENERIC_CLOCKEVENTS
-                if (!iter->second_pass) {
+                        if (!iter->second_pass) {
-                        iter->cpu = -1;
+                                iter->cpu = -1;
-                        iter->second_pass = true;
+                                iter->second_pass = true;
-                } else
+                        } else
-                        return NULL;
+                                return NULL;
 #else
-                return NULL;
+                        return NULL;
 #endif
+                }
        }
        return iter;
 }
+static void *timer_list_start(struct seq_file *file, loff_t *offset)
+{
+        struct timer_list_iter *iter = file->private;
+        if (!*offset)
+                iter->now = ktime_to_ns(ktime_get());
+        iter->cpu = -1;
+        iter->second_pass = false;
+        return move_iter(iter, *offset);
+}
 static void *timer_list_next(struct seq_file *file, void *v, loff_t *offset)
 {
        struct timer_list_iter *iter = file->private;
-        iter->cpu = cpumask_next(iter->cpu, cpu_online_mask);
        ++*offset;
-        return timer_list_start(file, offset);
+        return move_iter(iter, 1);
 }
 static void timer_list_stop(struct seq_file *seq, void *v)
diff --git a/kernel/timer.c b/kernel/timer.c
index 15bc1b41021d..4296d13db3d1 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -1505,11 +1505,11 @@ signed long __sched schedule_timeout_uninterruptible(signed long timeout)
 }
 EXPORT_SYMBOL(schedule_timeout_uninterruptible);
-static int __cpuinit init_timers_cpu(int cpu)
+static int init_timers_cpu(int cpu)
 {
        int j;
        struct tvec_base *base;
-        static char __cpuinitdata tvec_base_done[NR_CPUS];
+        static char tvec_base_done[NR_CPUS];
        if (!tvec_base_done[cpu]) {
                static char boot_done;
@@ -1577,7 +1577,7 @@ static void migrate_timer_list(struct tvec_base *new_base, struct list_head *hea
        }
 }
-static void __cpuinit migrate_timers(int cpu)
+static void migrate_timers(int cpu)
 {
        struct tvec_base *old_base;
        struct tvec_base *new_base;
@@ -1610,7 +1610,7 @@ static void __cpuinit migrate_timers(int cpu)
 }
 #endif /* CONFIG_HOTPLUG_CPU */
-static int __cpuinit timer_cpu_notify(struct notifier_block *self,
+static int timer_cpu_notify(struct notifier_block *self,
                                unsigned long action, void *hcpu)
 {
        long cpu = (long)hcpu;
@@ -1635,7 +1635,7 @@ static int __cpuinit timer_cpu_notify(struct notifier_block *self,
        return NOTIFY_OK;
 }
-static struct notifier_block __cpuinitdata timers_nb = {
+static struct notifier_block timers_nb = {
        .notifier_call  = timer_cpu_notify,
 };
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 67708f46baae..a6d098c6df3f 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -1441,12 +1441,22 @@ ftrace_hash_move(struct ftrace_ops *ops, int enable,
 * the hashes are freed with call_rcu_sched().
 */
 static int
-ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip)
+ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
 {
        struct ftrace_hash *filter_hash;
        struct ftrace_hash *notrace_hash;
        int ret;
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
+        /*
+         * There's a small race when adding ops that the ftrace handler
+         * that wants regs, may be called without them. We can not
+         * allow that handler to be called if regs is NULL.
+         */
+        if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS))
+                return 0;
+#endif
        filter_hash = rcu_dereference_raw_notrace(ops->filter_hash);
        notrace_hash = rcu_dereference_raw_notrace(ops->notrace_hash);
@@ -2159,12 +2169,57 @@ static cycle_t		ftrace_update_time;
 static unsigned long    ftrace_update_cnt;
 unsigned long           ftrace_update_tot_cnt;
-static int ops_traces_mod(struct ftrace_ops *ops)
+static inline int ops_traces_mod(struct ftrace_ops *ops)
 {
-        struct ftrace_hash *hash;
+        /*
+         * Filter_hash being empty will default to trace module.
+         * But notrace hash requires a test of individual module functions.
+         */
+        return ftrace_hash_empty(ops->filter_hash) &&
+                ftrace_hash_empty(ops->notrace_hash);
+}
+/*
+ * Check if the current ops references the record.
+ *
+ * If the ops traces all functions, then it was already accounted for.
+ * If the ops does not trace the current record function, skip it.
+ * If the ops ignores the function via notrace filter, skip it.
+ */
+static inline bool
+ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec)
+{
+        /* If ops isn't enabled, ignore it */
+        if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
+                return 0;
+        /* If ops traces all mods, we already accounted for it */
+        if (ops_traces_mod(ops))
+                return 0;
+        /* The function must be in the filter */
+        if (!ftrace_hash_empty(ops->filter_hash) &&
+            !ftrace_lookup_ip(ops->filter_hash, rec->ip))
+                return 0;
+        /* If in notrace hash, we ignore it too */
+        if (ftrace_lookup_ip(ops->notrace_hash, rec->ip))
+                return 0;
-        hash = ops->filter_hash;
+        return 1;
-        return ftrace_hash_empty(hash);
+}
+static int referenced_filters(struct dyn_ftrace *rec)
+{
+        struct ftrace_ops *ops;
+        int cnt = 0;
+        for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) {
+                if (ops_references_rec(ops, rec))
+                    cnt++;
+        }
+        return cnt;
 }
 static int ftrace_update_code(struct module *mod)
@@ -2173,6 +2228,7 @@ static int ftrace_update_code(struct module *mod)
        struct dyn_ftrace *p;
        cycle_t start, stop;
        unsigned long ref = 0;
+        bool test = false;
        int i;
        /*
@@ -2186,9 +2242,12 @@ static int ftrace_update_code(struct module *mod)
                for (ops = ftrace_ops_list;
                     ops != &ftrace_list_end; ops = ops->next) {
-                        if (ops->flags & FTRACE_OPS_FL_ENABLED &&
+                        if (ops->flags & FTRACE_OPS_FL_ENABLED) {
-                            ops_traces_mod(ops))
+                                if (ops_traces_mod(ops))
-                                ref++;
+                                        ref++;
+                                else
+                                        test = true;
+                        }
                }
        }
@@ -2198,12 +2257,16 @@ static int ftrace_update_code(struct module *mod)
        for (pg = ftrace_new_pgs; pg; pg = pg->next) {
                for (i = 0; i < pg->index; i++) {
+                        int cnt = ref;
                        /* If something went wrong, bail without enabling anything */
                        if (unlikely(ftrace_disabled))
                                return -1;
                        p = &pg->records[i];
-                        p->flags = ref;
+                        if (test)
+                                cnt += referenced_filters(p);
+                        p->flags = cnt;
                        /*
                         * Do the initial record conversion from mcount jump
@@ -2223,7 +2286,7 @@ static int ftrace_update_code(struct module *mod)
                         * conversion puts the module to the correct state, thus
                         * passing the ftrace_make_call check.
                         */
-                        if (ftrace_start_up && ref) {
+                        if (ftrace_start_up && cnt) {
                                int failed = __ftrace_replace_code(p, 1);
                                if (failed)
                                        ftrace_bug(failed, p->ip);
@@ -3374,6 +3437,12 @@ ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove)
        return add_hash_entry(hash, ip);
 }
+static void ftrace_ops_update_code(struct ftrace_ops *ops)
+{
+        if (ops->flags & FTRACE_OPS_FL_ENABLED && ftrace_enabled)
+                ftrace_run_update_code(FTRACE_UPDATE_CALLS);
+}
 static int
 ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len,
                unsigned long ip, int remove, int reset, int enable)
@@ -3416,9 +3485,8 @@ ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len,
        mutex_lock(&ftrace_lock);
        ret = ftrace_hash_move(ops, enable, orig_hash, hash);
-        if (!ret && ops->flags & FTRACE_OPS_FL_ENABLED
+        if (!ret)
-            && ftrace_enabled)
+                ftrace_ops_update_code(ops);
-                ftrace_run_update_code(FTRACE_UPDATE_CALLS);
        mutex_unlock(&ftrace_lock);
@@ -3645,9 +3713,8 @@ int ftrace_regex_release(struct inode *inode, struct file *file)
                mutex_lock(&ftrace_lock);
                ret = ftrace_hash_move(iter->ops, filter_hash,
                                       orig_hash, iter->hash);
-                if (!ret && (iter->ops->flags & FTRACE_OPS_FL_ENABLED)
+                if (!ret)
-                    && ftrace_enabled)
+                        ftrace_ops_update_code(iter->ops);
-                        ftrace_run_update_code(FTRACE_UPDATE_CALLS);
                mutex_unlock(&ftrace_lock);
        }
@@ -4218,7 +4285,7 @@ static inline void ftrace_startup_enable(int command) { }
 # define ftrace_shutdown_sysctl()       do { } while (0)
 static inline int
-ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip)
+ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
 {
        return 1;
 }
@@ -4241,7 +4308,7 @@ ftrace_ops_control_func(unsigned long ip, unsigned long parent_ip,
        do_for_each_ftrace_op(op, ftrace_control_list) {
                if (!(op->flags & FTRACE_OPS_FL_STUB) &&
                    !ftrace_function_local_disabled(op) &&
-                    ftrace_ops_test(op, ip))
+                    ftrace_ops_test(op, ip, regs))
                        op->func(ip, parent_ip, op, regs);
        } while_for_each_ftrace_op(op);
        trace_recursion_clear(TRACE_CONTROL_BIT);
@@ -4274,7 +4341,7 @@ __ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
         */
        preempt_disable_notrace();
        do_for_each_ftrace_op(op, ftrace_ops_list) {
-                if (ftrace_ops_test(op, ip))
+                if (ftrace_ops_test(op, ip, regs))
                        op->func(ip, parent_ip, op, regs);
        } while_for_each_ftrace_op(op);
        preempt_enable_notrace();
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index e444ff88f0a4..cc2f66f68dc5 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -36,11 +36,11 @@ int ring_buffer_print_entry_header(struct trace_seq *s)
 {
        int ret;
-        ret = trace_seq_printf(s, "# compressed entry header\n");
+        ret = trace_seq_puts(s, "# compressed entry header\n");
-        ret = trace_seq_printf(s, "\ttype_len    :    5 bits\n");
+        ret = trace_seq_puts(s, "\ttype_len    :    5 bits\n");
-        ret = trace_seq_printf(s, "\ttime_delta  :   27 bits\n");
+        ret = trace_seq_puts(s, "\ttime_delta  :   27 bits\n");
-        ret = trace_seq_printf(s, "\tarray       :   32 bits\n");
+        ret = trace_seq_puts(s, "\tarray       :   32 bits\n");
-        ret = trace_seq_printf(s, "\n");
+        ret = trace_seq_putc(s, '\n');
        ret = trace_seq_printf(s, "\tpadding     : type == %d\n",
                               RINGBUF_TYPE_PADDING);
        ret = trace_seq_printf(s, "\ttime_extend : type == %d\n",
@@ -1066,7 +1066,7 @@ static int rb_check_list(struct ring_buffer_per_cpu *cpu_buffer,
 }
 /**
- * check_pages - integrity check of buffer pages
+ * rb_check_pages - integrity check of buffer pages
 * @cpu_buffer: CPU buffer with pages to test
 *
 * As a safety measure we check to make sure the data pages have not
@@ -1258,7 +1258,7 @@ static int rb_cpu_notify(struct notifier_block *self,
 #endif
 /**
- * ring_buffer_alloc - allocate a new ring_buffer
+ * __ring_buffer_alloc - allocate a new ring_buffer
 * @size: the size in bytes per cpu that is needed.
 * @flags: attributes to set for the ring buffer.
 *
@@ -1607,6 +1607,7 @@ static void update_pages_handler(struct work_struct *work)
 * ring_buffer_resize - resize the ring buffer
 * @buffer: the buffer to resize.
 * @size: the new size.
+ * @cpu_id: the cpu buffer to resize
 *
 * Minimum size is 2 * BUF_PAGE_SIZE.
 *
@@ -3956,11 +3957,11 @@ EXPORT_SYMBOL_GPL(ring_buffer_consume);
 * expected.
 *
 * After a sequence of ring_buffer_read_prepare calls, the user is
- * expected to make at least one call to ring_buffer_prepare_sync.
+ * expected to make at least one call to ring_buffer_read_prepare_sync.
 * Afterwards, ring_buffer_read_start is invoked to get things going
 * for real.
 *
- * This overall must be paired with ring_buffer_finish.
+ * This overall must be paired with ring_buffer_read_finish.
 */
 struct ring_buffer_iter *
 ring_buffer_read_prepare(struct ring_buffer *buffer, int cpu)
@@ -4009,7 +4010,7 @@ EXPORT_SYMBOL_GPL(ring_buffer_read_prepare_sync);
 * an intervening ring_buffer_read_prepare_sync must have been
 * performed.
 *
- * Must be paired with ring_buffer_finish.
+ * Must be paired with ring_buffer_read_finish.
 */
 void
 ring_buffer_read_start(struct ring_buffer_iter *iter)
@@ -4031,7 +4032,7 @@ ring_buffer_read_start(struct ring_buffer_iter *iter)
 EXPORT_SYMBOL_GPL(ring_buffer_read_start);
 /**
- * ring_buffer_finish - finish reading the iterator of the buffer
+ * ring_buffer_read_finish - finish reading the iterator of the buffer
 * @iter: The iterator retrieved by ring_buffer_start
 *
 * This re-enables the recording to the buffer, and frees the
@@ -4346,6 +4347,7 @@ EXPORT_SYMBOL_GPL(ring_buffer_swap_cpu);
 /**
 * ring_buffer_alloc_read_page - allocate a page to read from buffer
 * @buffer: the buffer to allocate for.
+ * @cpu: the cpu buffer to allocate.
 *
 * This function is used in conjunction with ring_buffer_read_page.
 * When reading a full page from the ring buffer, these functions
@@ -4403,7 +4405,7 @@ EXPORT_SYMBOL_GPL(ring_buffer_free_read_page);
 * to swap with a page in the ring buffer.
 *
 * for example:
- *      rpage = ring_buffer_alloc_read_page(buffer);
+ *      rpage = ring_buffer_alloc_read_page(buffer, cpu);
 *      if (!rpage)
 *              return error;
 *      ret = ring_buffer_read_page(buffer, &rpage, len, cpu, 0);
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 0cd500bffd9b..496f94d57698 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -243,20 +243,25 @@ int filter_current_check_discard(struct ring_buffer *buffer,
 }
 EXPORT_SYMBOL_GPL(filter_current_check_discard);
-cycle_t ftrace_now(int cpu)
+cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
 {
        u64 ts;
        /* Early boot up does not have a buffer yet */
-        if (!global_trace.trace_buffer.buffer)
+        if (!buf->buffer)
                return trace_clock_local();
-        ts = ring_buffer_time_stamp(global_trace.trace_buffer.buffer, cpu);
+        ts = ring_buffer_time_stamp(buf->buffer, cpu);
-        ring_buffer_normalize_time_stamp(global_trace.trace_buffer.buffer, cpu, &ts);
+        ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
        return ts;
 }
+cycle_t ftrace_now(int cpu)
+{
+        return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
+}
 /**
 * tracing_is_enabled - Show if global_trace has been disabled
 *
@@ -1211,7 +1216,7 @@ void tracing_reset_online_cpus(struct trace_buffer *buf)
        /* Make sure all commits have finished */
        synchronize_sched();
-        buf->time_start = ftrace_now(buf->cpu);
+        buf->time_start = buffer_ftrace_now(buf, buf->cpu);
        for_each_online_cpu(cpu)
                ring_buffer_reset_cpu(buffer, cpu);
@@ -1219,23 +1224,17 @@ void tracing_reset_online_cpus(struct trace_buffer *buf)
        ring_buffer_record_enable(buffer);
 }
-void tracing_reset_current(int cpu)
+/* Must have trace_types_lock held */
-{
-        tracing_reset(&global_trace.trace_buffer, cpu);
-}
 void tracing_reset_all_online_cpus(void)
 {
        struct trace_array *tr;
-        mutex_lock(&trace_types_lock);
        list_for_each_entry(tr, &ftrace_trace_arrays, list) {
                tracing_reset_online_cpus(&tr->trace_buffer);
 #ifdef CONFIG_TRACER_MAX_TRACE
                tracing_reset_online_cpus(&tr->max_buffer);
 #endif
        }
-        mutex_unlock(&trace_types_lock);
 }
 #define SAVED_CMDLINES 128
@@ -2843,6 +2842,17 @@ static int s_show(struct seq_file *m, void *v)
        return 0;
 }
+/*
+ * Should be used after trace_array_get(), trace_types_lock
+ * ensures that i_cdev was already initialized.
+ */
+static inline int tracing_get_cpu(struct inode *inode)
+{
+        if (inode->i_cdev) /* See trace_create_cpu_file() */
+                return (long)inode->i_cdev - 1;
+        return RING_BUFFER_ALL_CPUS;
+}
 static const struct seq_operations tracer_seq_ops = {
        .start          = s_start,
        .next           = s_next,
@@ -2851,9 +2861,9 @@ static const struct seq_operations tracer_seq_ops = {
 };
 static struct trace_iterator *
-__tracing_open(struct trace_array *tr, struct trace_cpu *tc,
+__tracing_open(struct inode *inode, struct file *file, bool snapshot)
-               struct inode *inode, struct file *file, bool snapshot)
 {
+        struct trace_array *tr = inode->i_private;
        struct trace_iterator *iter;
        int cpu;
@@ -2894,8 +2904,8 @@ __tracing_open(struct trace_array *tr, struct trace_cpu *tc,
                iter->trace_buffer = &tr->trace_buffer;
        iter->snapshot = snapshot;
        iter->pos = -1;
+        iter->cpu_file = tracing_get_cpu(inode);
        mutex_init(&iter->mutex);
-        iter->cpu_file = tc->cpu;
        /* Notify the tracer early; before we stop tracing. */
        if (iter->trace && iter->trace->open)
@@ -2971,45 +2981,22 @@ static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
        filp->private_data = inode->i_private;
        return 0;
-        
-}
-static int tracing_open_generic_tc(struct inode *inode, struct file *filp)
-{
-        struct trace_cpu *tc = inode->i_private;
-        struct trace_array *tr = tc->tr;
-        if (tracing_disabled)
-                return -ENODEV;
-        if (trace_array_get(tr) < 0)
-                return -ENODEV;
-        filp->private_data = inode->i_private;
-        return 0;
-        
 }
 static int tracing_release(struct inode *inode, struct file *file)
 {
+        struct trace_array *tr = inode->i_private;
        struct seq_file *m = file->private_data;
        struct trace_iterator *iter;
-        struct trace_array *tr;
        int cpu;
-        /* Writes do not use seq_file, need to grab tr from inode */
        if (!(file->f_mode & FMODE_READ)) {
-                struct trace_cpu *tc = inode->i_private;
+                trace_array_put(tr);
-                trace_array_put(tc->tr);
                return 0;
        }
+        /* Writes do not use seq_file */
        iter = m->private;
-        tr = iter->tr;
-        trace_array_put(tr);
        mutex_lock(&trace_types_lock);
        for_each_tracing_cpu(cpu) {
@@ -3023,6 +3010,9 @@ static int tracing_release(struct inode *inode, struct file *file)
        if (!iter->snapshot)
                /* reenable tracing if it was previously enabled */
                tracing_start_tr(tr);
+        __trace_array_put(tr);
        mutex_unlock(&trace_types_lock);
        mutex_destroy(&iter->mutex);
@@ -3042,15 +3032,6 @@ static int tracing_release_generic_tr(struct inode *inode, struct file *file)
        return 0;
 }
-static int tracing_release_generic_tc(struct inode *inode, struct file *file)
-{
-        struct trace_cpu *tc = inode->i_private;
-        struct trace_array *tr = tc->tr;
-        trace_array_put(tr);
-        return 0;
-}
 static int tracing_single_release_tr(struct inode *inode, struct file *file)
 {
        struct trace_array *tr = inode->i_private;
@@ -3062,8 +3043,7 @@ static int tracing_single_release_tr(struct inode *inode, struct file *file)
 static int tracing_open(struct inode *inode, struct file *file)
 {
-        struct trace_cpu *tc = inode->i_private;
+        struct trace_array *tr = inode->i_private;
-        struct trace_array *tr = tc->tr;
        struct trace_iterator *iter;
        int ret = 0;
@@ -3071,16 +3051,17 @@ static int tracing_open(struct inode *inode, struct file *file)
                return -ENODEV;
        /* If this file was open for write, then erase contents */
-        if ((file->f_mode & FMODE_WRITE) &&
+        if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
-            (file->f_flags & O_TRUNC)) {
+                int cpu = tracing_get_cpu(inode);
-                if (tc->cpu == RING_BUFFER_ALL_CPUS)
+                if (cpu == RING_BUFFER_ALL_CPUS)
                        tracing_reset_online_cpus(&tr->trace_buffer);
                else
-                        tracing_reset(&tr->trace_buffer, tc->cpu);
+                        tracing_reset(&tr->trace_buffer, cpu);
        }
        if (file->f_mode & FMODE_READ) {
-                iter = __tracing_open(tr, tc, inode, file, false);
+                iter = __tracing_open(inode, file, false);
                if (IS_ERR(iter))
                        ret = PTR_ERR(iter);
                else if (trace_flags & TRACE_ITER_LATENCY_FMT)
@@ -3447,6 +3428,7 @@ tracing_trace_options_write(struct file *filp, const char __user *ubuf,
 static int tracing_trace_options_open(struct inode *inode, struct file *file)
 {
        struct trace_array *tr = inode->i_private;
+        int ret;
        if (tracing_disabled)
                return -ENODEV;
@@ -3454,7 +3436,11 @@ static int tracing_trace_options_open(struct inode *inode, struct file *file)
        if (trace_array_get(tr) < 0)
                return -ENODEV;
-        return single_open(file, tracing_trace_options_show, inode->i_private);
+        ret = single_open(file, tracing_trace_options_show, inode->i_private);
+        if (ret < 0)
+                trace_array_put(tr);
+        return ret;
 }
 static const struct file_operations tracing_iter_fops = {
@@ -3537,14 +3523,14 @@ static const char readme_msg[] =
        "\n  snapshot\t\t- Like 'trace' but shows the content of the static snapshot buffer\n"
        "\t\t\t  Read the contents for more information\n"
 #endif
-#ifdef CONFIG_STACKTRACE
+#ifdef CONFIG_STACK_TRACER
        "  stack_trace\t\t- Shows the max stack trace when active\n"
        "  stack_max_size\t- Shows current max stack size that was traced\n"
        "\t\t\t  Write into this file to reset the max size (trigger a new trace)\n"
 #ifdef CONFIG_DYNAMIC_FTRACE
        "  stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace traces\n"
 #endif
-#endif /* CONFIG_STACKTRACE */
+#endif /* CONFIG_STACK_TRACER */
 ;
 static ssize_t
@@ -3941,8 +3927,7 @@ tracing_max_lat_write(struct file *filp, const char __user *ubuf,
 static int tracing_open_pipe(struct inode *inode, struct file *filp)
 {
-        struct trace_cpu *tc = inode->i_private;
+        struct trace_array *tr = inode->i_private;
-        struct trace_array *tr = tc->tr;
        struct trace_iterator *iter;
        int ret = 0;
@@ -3958,6 +3943,7 @@ static int tracing_open_pipe(struct inode *inode, struct file *filp)
        iter = kzalloc(sizeof(*iter), GFP_KERNEL);
        if (!iter) {
                ret = -ENOMEM;
+                __trace_array_put(tr);
                goto out;
        }
@@ -3987,9 +3973,9 @@ static int tracing_open_pipe(struct inode *inode, struct file *filp)
        if (trace_clocks[tr->clock_id].in_ns)
                iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
-        iter->cpu_file = tc->cpu;
+        iter->tr = tr;
-        iter->tr = tc->tr;
+        iter->trace_buffer = &tr->trace_buffer;
-        iter->trace_buffer = &tc->tr->trace_buffer;
+        iter->cpu_file = tracing_get_cpu(inode);
        mutex_init(&iter->mutex);
        filp->private_data = iter;
@@ -4012,8 +3998,7 @@ fail:
 static int tracing_release_pipe(struct inode *inode, struct file *file)
 {
        struct trace_iterator *iter = file->private_data;
-        struct trace_cpu *tc = inode->i_private;
+        struct trace_array *tr = inode->i_private;
-        struct trace_array *tr = tc->tr;
        mutex_lock(&trace_types_lock);
@@ -4166,6 +4151,7 @@ waitagain:
        memset(&iter->seq, 0,
               sizeof(struct trace_iterator) -
               offsetof(struct trace_iterator, seq));
+        cpumask_clear(iter->started);
        iter->pos = -1;
        trace_event_read_lock();
@@ -4366,15 +4352,16 @@ static ssize_t
 tracing_entries_read(struct file *filp, char __user *ubuf,
                     size_t cnt, loff_t *ppos)
 {
-        struct trace_cpu *tc = filp->private_data;
+        struct inode *inode = file_inode(filp);
-        struct trace_array *tr = tc->tr;
+        struct trace_array *tr = inode->i_private;
+        int cpu = tracing_get_cpu(inode);
        char buf[64];
        int r = 0;
        ssize_t ret;
        mutex_lock(&trace_types_lock);
-        if (tc->cpu == RING_BUFFER_ALL_CPUS) {
+        if (cpu == RING_BUFFER_ALL_CPUS) {
                int cpu, buf_size_same;
                unsigned long size;
@@ -4401,7 +4388,7 @@ tracing_entries_read(struct file *filp, char __user *ubuf,
                } else
                        r = sprintf(buf, "X\n");
        } else
-                r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, tc->cpu)->entries >> 10);
+                r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
        mutex_unlock(&trace_types_lock);
@@ -4413,7 +4400,8 @@ static ssize_t
 tracing_entries_write(struct file *filp, const char __user *ubuf,
                      size_t cnt, loff_t *ppos)
 {
-        struct trace_cpu *tc = filp->private_data;
+        struct inode *inode = file_inode(filp);
+        struct trace_array *tr = inode->i_private;
        unsigned long val;
        int ret;
@@ -4427,8 +4415,7 @@ tracing_entries_write(struct file *filp, const char __user *ubuf,
        /* value is in KB */
        val <<= 10;
+        ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
-        ret = tracing_resize_ring_buffer(tc->tr, val, tc->cpu);
        if (ret < 0)
                return ret;
@@ -4482,7 +4469,7 @@ tracing_free_buffer_release(struct inode *inode, struct file *filp)
        /* disable tracing ? */
        if (trace_flags & TRACE_ITER_STOP_ON_FREE)
-                tracing_off();
+                tracer_tracing_off(tr);
        /* resize the ring buffer to 0 */
        tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
@@ -4647,12 +4634,12 @@ static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
         * New clock may not be consistent with the previous clock.
         * Reset the buffer so that it doesn't have incomparable timestamps.
         */
-        tracing_reset_online_cpus(&global_trace.trace_buffer);
+        tracing_reset_online_cpus(&tr->trace_buffer);
 #ifdef CONFIG_TRACER_MAX_TRACE
        if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
                ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
-        tracing_reset_online_cpus(&global_trace.max_buffer);
+        tracing_reset_online_cpus(&tr->max_buffer);
 #endif
        mutex_unlock(&trace_types_lock);
@@ -4689,8 +4676,7 @@ struct ftrace_buffer_info {
 #ifdef CONFIG_TRACER_SNAPSHOT
 static int tracing_snapshot_open(struct inode *inode, struct file *file)
 {
-        struct trace_cpu *tc = inode->i_private;
+        struct trace_array *tr = inode->i_private;
-        struct trace_array *tr = tc->tr;
        struct trace_iterator *iter;
        struct seq_file *m;
        int ret = 0;
@@ -4699,26 +4685,29 @@ static int tracing_snapshot_open(struct inode *inode, struct file *file)
                return -ENODEV;
        if (file->f_mode & FMODE_READ) {
-                iter = __tracing_open(tr, tc, inode, file, true);
+                iter = __tracing_open(inode, file, true);
                if (IS_ERR(iter))
                        ret = PTR_ERR(iter);
        } else {
                /* Writes still need the seq_file to hold the private data */
+                ret = -ENOMEM;
                m = kzalloc(sizeof(*m), GFP_KERNEL);
                if (!m)
-                        return -ENOMEM;
+                        goto out;
                iter = kzalloc(sizeof(*iter), GFP_KERNEL);
                if (!iter) {
                        kfree(m);
-                        return -ENOMEM;
+                        goto out;
                }
+                ret = 0;
                iter->tr = tr;
-                iter->trace_buffer = &tc->tr->max_buffer;
+                iter->trace_buffer = &tr->max_buffer;
-                iter->cpu_file = tc->cpu;
+                iter->cpu_file = tracing_get_cpu(inode);
                m->private = iter;
                file->private_data = m;
        }
+out:
        if (ret < 0)
                trace_array_put(tr);
@@ -4873,11 +4862,11 @@ static const struct file_operations tracing_pipe_fops = {
 };
 static const struct file_operations tracing_entries_fops = {
-        .open           = tracing_open_generic_tc,
+        .open           = tracing_open_generic_tr,
        .read           = tracing_entries_read,
        .write          = tracing_entries_write,
        .llseek         = generic_file_llseek,
-        .release        = tracing_release_generic_tc,
+        .release        = tracing_release_generic_tr,
 };
 static const struct file_operations tracing_total_entries_fops = {
@@ -4929,8 +4918,7 @@ static const struct file_operations snapshot_raw_fops = {
 static int tracing_buffers_open(struct inode *inode, struct file *filp)
 {
-        struct trace_cpu *tc = inode->i_private;
+        struct trace_array *tr = inode->i_private;
-        struct trace_array *tr = tc->tr;
        struct ftrace_buffer_info *info;
        int ret;
@@ -4948,10 +4936,8 @@ static int tracing_buffers_open(struct inode *inode, struct file *filp)
        mutex_lock(&trace_types_lock);
-        tr->ref++;
        info->iter.tr           = tr;
-        info->iter.cpu_file     = tc->cpu;
+        info->iter.cpu_file     = tracing_get_cpu(inode);
        info->iter.trace        = tr->current_trace;
        info->iter.trace_buffer = &tr->trace_buffer;
        info->spare             = NULL;
@@ -5268,14 +5254,14 @@ static ssize_t
 tracing_stats_read(struct file *filp, char __user *ubuf,
                   size_t count, loff_t *ppos)
 {
-        struct trace_cpu *tc = filp->private_data;
+        struct inode *inode = file_inode(filp);
-        struct trace_array *tr = tc->tr;
+        struct trace_array *tr = inode->i_private;
        struct trace_buffer *trace_buf = &tr->trace_buffer;
+        int cpu = tracing_get_cpu(inode);
        struct trace_seq *s;
        unsigned long cnt;
        unsigned long long t;
        unsigned long usec_rem;
-        int cpu = tc->cpu;
        s = kmalloc(sizeof(*s), GFP_KERNEL);
        if (!s)
@@ -5328,9 +5314,10 @@ tracing_stats_read(struct file *filp, char __user *ubuf,
 }
 static const struct file_operations tracing_stats_fops = {
-        .open           = tracing_open_generic,
+        .open           = tracing_open_generic_tr,
        .read           = tracing_stats_read,
        .llseek         = generic_file_llseek,
+        .release        = tracing_release_generic_tr,
 };
 #ifdef CONFIG_DYNAMIC_FTRACE
@@ -5519,10 +5506,20 @@ static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
        return tr->percpu_dir;
 }
+static struct dentry *
+trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
+                      void *data, long cpu, const struct file_operations *fops)
+{
+        struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
+        if (ret) /* See tracing_get_cpu() */
+                ret->d_inode->i_cdev = (void *)(cpu + 1);
+        return ret;
+}
 static void
 tracing_init_debugfs_percpu(struct trace_array *tr, long cpu)
 {
-        struct trace_array_cpu *data = per_cpu_ptr(tr->trace_buffer.data, cpu);
        struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
        struct dentry *d_cpu;
        char cpu_dir[30]; /* 30 characters should be more than enough */
@@ -5538,28 +5535,28 @@ tracing_init_debugfs_percpu(struct trace_array *tr, long cpu)
        }
        /* per cpu trace_pipe */
-        trace_create_file("trace_pipe", 0444, d_cpu,
+        trace_create_cpu_file("trace_pipe", 0444, d_cpu,
-                        (void *)&data->trace_cpu, &tracing_pipe_fops);
+                                tr, cpu, &tracing_pipe_fops);
        /* per cpu trace */
-        trace_create_file("trace", 0644, d_cpu,
+        trace_create_cpu_file("trace", 0644, d_cpu,
-                        (void *)&data->trace_cpu, &tracing_fops);
+                                tr, cpu, &tracing_fops);
-        trace_create_file("trace_pipe_raw", 0444, d_cpu,
+        trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
-                        (void *)&data->trace_cpu, &tracing_buffers_fops);
+                                tr, cpu, &tracing_buffers_fops);
-        trace_create_file("stats", 0444, d_cpu,
+        trace_create_cpu_file("stats", 0444, d_cpu,
-                        (void *)&data->trace_cpu, &tracing_stats_fops);
+                                tr, cpu, &tracing_stats_fops);
-        trace_create_file("buffer_size_kb", 0444, d_cpu,
+        trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
-                        (void *)&data->trace_cpu, &tracing_entries_fops);
+                                tr, cpu, &tracing_entries_fops);
 #ifdef CONFIG_TRACER_SNAPSHOT
-        trace_create_file("snapshot", 0644, d_cpu,
+        trace_create_cpu_file("snapshot", 0644, d_cpu,
-                          (void *)&data->trace_cpu, &snapshot_fops);
+                                tr, cpu, &snapshot_fops);
-        trace_create_file("snapshot_raw", 0444, d_cpu,
+        trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
-                        (void *)&data->trace_cpu, &snapshot_raw_fops);
+                                tr, cpu, &snapshot_raw_fops);
 #endif
 }
@@ -5868,17 +5865,6 @@ struct dentry *trace_instance_dir;
 static void
 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer);
-static void init_trace_buffers(struct trace_array *tr, struct trace_buffer *buf)
-{
-        int cpu;
-        for_each_tracing_cpu(cpu) {
-                memset(per_cpu_ptr(buf->data, cpu), 0, sizeof(struct trace_array_cpu));
-                per_cpu_ptr(buf->data, cpu)->trace_cpu.cpu = cpu;
-                per_cpu_ptr(buf->data, cpu)->trace_cpu.tr = tr;
-        }
-}
 static int
 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
 {
@@ -5896,8 +5882,6 @@ allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size
                return -ENOMEM;
        }
-        init_trace_buffers(tr, buf);
        /* Allocate the first page for all buffers */
        set_buffer_entries(&tr->trace_buffer,
                           ring_buffer_size(tr->trace_buffer.buffer, 0));
@@ -5964,17 +5948,15 @@ static int new_instance_create(const char *name)
        if (allocate_trace_buffers(tr, trace_buf_size) < 0)
                goto out_free_tr;
-        /* Holder for file callbacks */
-        tr->trace_cpu.cpu = RING_BUFFER_ALL_CPUS;
-        tr->trace_cpu.tr = tr;
        tr->dir = debugfs_create_dir(name, trace_instance_dir);
        if (!tr->dir)
                goto out_free_tr;
        ret = event_trace_add_tracer(tr->dir, tr);
-        if (ret)
+        if (ret) {
+                debugfs_remove_recursive(tr->dir);
                goto out_free_tr;
+        }
        init_tracer_debugfs(tr, tr->dir);
@@ -6120,13 +6102,13 @@ init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer)
                          tr, &tracing_iter_fops);
        trace_create_file("trace", 0644, d_tracer,
-                        (void *)&tr->trace_cpu, &tracing_fops);
+                          tr, &tracing_fops);
        trace_create_file("trace_pipe", 0444, d_tracer,
-                        (void *)&tr->trace_cpu, &tracing_pipe_fops);
+                          tr, &tracing_pipe_fops);
        trace_create_file("buffer_size_kb", 0644, d_tracer,
-                        (void *)&tr->trace_cpu, &tracing_entries_fops);
+                          tr, &tracing_entries_fops);
        trace_create_file("buffer_total_size_kb", 0444, d_tracer,
                          tr, &tracing_total_entries_fops);
@@ -6141,11 +6123,11 @@ init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer)
                          &trace_clock_fops);
        trace_create_file("tracing_on", 0644, d_tracer,
-                            tr, &rb_simple_fops);
+                          tr, &rb_simple_fops);
 #ifdef CONFIG_TRACER_SNAPSHOT
        trace_create_file("snapshot", 0644, d_tracer,
-                          (void *)&tr->trace_cpu, &snapshot_fops);
+                          tr, &snapshot_fops);
 #endif
        for_each_tracing_cpu(cpu)
@@ -6439,10 +6421,6 @@ __init static int tracer_alloc_buffers(void)
        global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
-        /* Holder for file callbacks */
-        global_trace.trace_cpu.cpu = RING_BUFFER_ALL_CPUS;
-        global_trace.trace_cpu.tr = &global_trace;
        INIT_LIST_HEAD(&global_trace.systems);
        INIT_LIST_HEAD(&global_trace.events);
        list_add(&global_trace.list, &ftrace_trace_arrays);
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index 4a4f6e1828b6..fe39acd4c1aa 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -130,19 +130,12 @@ enum trace_flag_type {
 struct trace_array;
-struct trace_cpu {
-        struct trace_array      *tr;
-        struct dentry           *dir;
-        int                     cpu;
-};
 /*
 * The CPU trace array - it consists of thousands of trace entries
 * plus some other descriptor data: (for example which task started
 * the trace, etc.)
 */
 struct trace_array_cpu {
-        struct trace_cpu        trace_cpu;
        atomic_t                disabled;
        void                    *buffer_page;   /* ring buffer spare */
@@ -196,7 +189,6 @@ struct trace_array {
        bool                    allocated_snapshot;
 #endif
        int                     buffer_disabled;
-        struct trace_cpu        trace_cpu;      /* place holder */
 #ifdef CONFIG_FTRACE_SYSCALLS
        int                     sys_refcount_enter;
        int                     sys_refcount_exit;
@@ -214,7 +206,6 @@ struct trace_array {
        struct dentry           *event_dir;
        struct list_head        systems;
        struct list_head        events;
-        struct task_struct      *waiter;
        int                     ref;
 };
@@ -680,6 +671,15 @@ extern int trace_selftest_startup_sched_switch(struct tracer *trace,
                                               struct trace_array *tr);
 extern int trace_selftest_startup_branch(struct tracer *trace,
                                         struct trace_array *tr);
+/*
+ * Tracer data references selftest functions that only occur
+ * on boot up. These can be __init functions. Thus, when selftests
+ * are enabled, then the tracers need to reference __init functions.
+ */
+#define __tracer_data           __refdata
+#else
+/* Tracers are seldom changed. Optimize when selftests are disabled. */
+#define __tracer_data           __read_mostly
 #endif /* CONFIG_FTRACE_STARTUP_TEST */
 extern void *head_page(struct trace_array_cpu *data);
@@ -1022,6 +1022,9 @@ extern struct list_head ftrace_events;
 extern const char *__start___trace_bprintk_fmt[];
 extern const char *__stop___trace_bprintk_fmt[];
+extern const char *__start___tracepoint_str[];
+extern const char *__stop___tracepoint_str[];
 void trace_printk_init_buffers(void);
 void trace_printk_start_comm(void);
 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set);
diff --git a/kernel/trace/trace_event_perf.c b/kernel/trace/trace_event_perf.c
index 84b1e045faba..80c36bcf66e8 100644
--- a/kernel/trace/trace_event_perf.c
+++ b/kernel/trace/trace_event_perf.c
@@ -236,6 +236,10 @@ __kprobes void *perf_trace_buf_prepare(int size, unsigned short type,
        BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(unsigned long));
+        if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
+                        "perf buffer not large enough"))
+                return NULL;
        pc = preempt_count();
        *rctxp = perf_swevent_get_recursion_context();
@@ -266,6 +270,10 @@ perf_ftrace_function_call(unsigned long ip, unsigned long parent_ip,
        struct pt_regs regs;
        int rctx;
+        head = this_cpu_ptr(event_function.perf_events);
+        if (hlist_empty(head))
+                return;
 #define ENTRY_SIZE (ALIGN(sizeof(struct ftrace_entry) + sizeof(u32), \
                    sizeof(u64)) - sizeof(u32))
@@ -279,8 +287,6 @@ perf_ftrace_function_call(unsigned long ip, unsigned long parent_ip,
        entry->ip = ip;
        entry->parent_ip = parent_ip;
-        head = this_cpu_ptr(event_function.perf_events);
        perf_trace_buf_submit(entry, ENTRY_SIZE, rctx, 0,
                              1, &regs, head, NULL);
diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c
index 7d854290bf81..29a7ebcfb426 100644
--- a/kernel/trace/trace_events.c
+++ b/kernel/trace/trace_events.c
@@ -409,33 +409,42 @@ static void put_system(struct ftrace_subsystem_dir *dir)
        mutex_unlock(&event_mutex);
 }
-/*
+static void remove_subsystem(struct ftrace_subsystem_dir *dir)
- * Open and update trace_array ref count.
- * Must have the current trace_array passed to it.
- */
-static int tracing_open_generic_file(struct inode *inode, struct file *filp)
 {
-        struct ftrace_event_file *file = inode->i_private;
+        if (!dir)
-        struct trace_array *tr = file->tr;
+                return;
-        int ret;
-        if (trace_array_get(tr) < 0)
+        if (!--dir->nr_events) {
-                return -ENODEV;
+                debugfs_remove_recursive(dir->entry);
+                list_del(&dir->list);
+                __put_system_dir(dir);
+        }
+}
-        ret = tracing_open_generic(inode, filp);
+static void *event_file_data(struct file *filp)
-        if (ret < 0)
+{
-                trace_array_put(tr);
+        return ACCESS_ONCE(file_inode(filp)->i_private);
-        return ret;
 }
-static int tracing_release_generic_file(struct inode *inode, struct file *filp)
+static void remove_event_file_dir(struct ftrace_event_file *file)
 {
-        struct ftrace_event_file *file = inode->i_private;
+        struct dentry *dir = file->dir;
-        struct trace_array *tr = file->tr;
+        struct dentry *child;
-        trace_array_put(tr);
+        if (dir) {
+                spin_lock(&dir->d_lock);        /* probably unneeded */
+                list_for_each_entry(child, &dir->d_subdirs, d_u.d_child) {
+                        if (child->d_inode)     /* probably unneeded */
+                                child->d_inode->i_private = NULL;
+                }
+                spin_unlock(&dir->d_lock);
-        return 0;
+                debugfs_remove_recursive(dir);
+        }
+        list_del(&file->list);
+        remove_subsystem(file->system);
+        kmem_cache_free(file_cachep, file);
 }
 /*
@@ -679,15 +688,25 @@ static ssize_t
 event_enable_read(struct file *filp, char __user *ubuf, size_t cnt,
                  loff_t *ppos)
 {
-        struct ftrace_event_file *file = filp->private_data;
+        struct ftrace_event_file *file;
+        unsigned long flags;
        char buf[4] = "0";
-        if (file->flags & FTRACE_EVENT_FL_ENABLED &&
+        mutex_lock(&event_mutex);
-            !(file->flags & FTRACE_EVENT_FL_SOFT_DISABLED))
+        file = event_file_data(filp);
+        if (likely(file))
+                flags = file->flags;
+        mutex_unlock(&event_mutex);
+        if (!file)
+                return -ENODEV;
+        if (flags & FTRACE_EVENT_FL_ENABLED &&
+            !(flags & FTRACE_EVENT_FL_SOFT_DISABLED))
                strcpy(buf, "1");
-        if (file->flags & FTRACE_EVENT_FL_SOFT_DISABLED ||
+        if (flags & FTRACE_EVENT_FL_SOFT_DISABLED ||
-            file->flags & FTRACE_EVENT_FL_SOFT_MODE)
+            flags & FTRACE_EVENT_FL_SOFT_MODE)
                strcat(buf, "*");
        strcat(buf, "\n");
@@ -699,13 +718,10 @@ static ssize_t
 event_enable_write(struct file *filp, const char __user *ubuf, size_t cnt,
                   loff_t *ppos)
 {
-        struct ftrace_event_file *file = filp->private_data;
+        struct ftrace_event_file *file;
        unsigned long val;
        int ret;
-        if (!file)
-                return -EINVAL;
        ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
        if (ret)
                return ret;
@@ -717,8 +733,11 @@ event_enable_write(struct file *filp, const char __user *ubuf, size_t cnt,
        switch (val) {
        case 0:
        case 1:
+                ret = -ENODEV;
                mutex_lock(&event_mutex);
-                ret = ftrace_event_enable_disable(file, val);
+                file = event_file_data(filp);
+                if (likely(file))
+                        ret = ftrace_event_enable_disable(file, val);
                mutex_unlock(&event_mutex);
                break;
@@ -825,65 +844,39 @@ enum {
 static void *f_next(struct seq_file *m, void *v, loff_t *pos)
 {
-        struct ftrace_event_call *call = m->private;
+        struct ftrace_event_call *call = event_file_data(m->private);
-        struct ftrace_event_field *field;
        struct list_head *common_head = &ftrace_common_fields;
        struct list_head *head = trace_get_fields(call);
+        struct list_head *node = v;
        (*pos)++;
        switch ((unsigned long)v) {
        case FORMAT_HEADER:
-                if (unlikely(list_empty(common_head)))
+                node = common_head;
-                        return NULL;
+                break;
-                field = list_entry(common_head->prev,
-                                   struct ftrace_event_field, link);
-                return field;
        case FORMAT_FIELD_SEPERATOR:
-                if (unlikely(list_empty(head)))
+                node = head;
-                        return NULL;
+                break;
-                field = list_entry(head->prev, struct ftrace_event_field, link);
-                return field;
        case FORMAT_PRINTFMT:
                /* all done */
                return NULL;
        }
-        field = v;
+        node = node->prev;
-        if (field->link.prev == common_head)
+        if (node == common_head)
                return (void *)FORMAT_FIELD_SEPERATOR;
-        else if (field->link.prev == head)
+        else if (node == head)
                return (void *)FORMAT_PRINTFMT;
+        else
-        field = list_entry(field->link.prev, struct ftrace_event_field, link);
+                return node;
-        return field;
-}
-static void *f_start(struct seq_file *m, loff_t *pos)
-{
-        loff_t l = 0;
-        void *p;
-        /* Start by showing the header */
-        if (!*pos)
-                return (void *)FORMAT_HEADER;
-        p = (void *)FORMAT_HEADER;
-        do {
-                p = f_next(m, p, &l);
-        } while (p && l < *pos);
-        return p;
 }
 static int f_show(struct seq_file *m, void *v)
 {
-        struct ftrace_event_call *call = m->private;
+        struct ftrace_event_call *call = event_file_data(m->private);
        struct ftrace_event_field *field;
        const char *array_descriptor;
@@ -904,8 +897,7 @@ static int f_show(struct seq_file *m, void *v)
                return 0;
        }
-        field = v;
+        field = list_entry(v, struct ftrace_event_field, link);
        /*
         * Smartly shows the array type(except dynamic array).
         * Normal:
@@ -932,8 +924,25 @@ static int f_show(struct seq_file *m, void *v)
        return 0;
 }
+static void *f_start(struct seq_file *m, loff_t *pos)
+{
+        void *p = (void *)FORMAT_HEADER;
+        loff_t l = 0;
+        /* ->stop() is called even if ->start() fails */
+        mutex_lock(&event_mutex);
+        if (!event_file_data(m->private))
+                return ERR_PTR(-ENODEV);
+        while (l < *pos && p)
+                p = f_next(m, p, &l);
+        return p;
+}
 static void f_stop(struct seq_file *m, void *p)
 {
+        mutex_unlock(&event_mutex);
 }
 static const struct seq_operations trace_format_seq_ops = {
@@ -945,7 +954,6 @@ static const struct seq_operations trace_format_seq_ops = {
 static int trace_format_open(struct inode *inode, struct file *file)
 {
-        struct ftrace_event_call *call = inode->i_private;
        struct seq_file *m;
        int ret;
@@ -954,7 +962,7 @@ static int trace_format_open(struct inode *inode, struct file *file)
                return ret;
        m = file->private_data;
-        m->private = call;
+        m->private = file;
        return 0;
 }
@@ -962,45 +970,47 @@ static int trace_format_open(struct inode *inode, struct file *file)
 static ssize_t
 event_id_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
 {
-        struct ftrace_event_call *call = filp->private_data;
+        int id = (long)event_file_data(filp);
-        struct trace_seq *s;
+        char buf[32];
-        int r;
+        int len;
        if (*ppos)
                return 0;
-        s = kmalloc(sizeof(*s), GFP_KERNEL);
+        if (unlikely(!id))
-        if (!s)
+                return -ENODEV;
-                return -ENOMEM;
-        trace_seq_init(s);
+        len = sprintf(buf, "%d\n", id);
-        trace_seq_printf(s, "%d\n", call->event.type);
-        r = simple_read_from_buffer(ubuf, cnt, ppos,
+        return simple_read_from_buffer(ubuf, cnt, ppos, buf, len);
-                                    s->buffer, s->len);
-        kfree(s);
-        return r;
 }
 static ssize_t
 event_filter_read(struct file *filp, char __user *ubuf, size_t cnt,
                  loff_t *ppos)
 {
-        struct ftrace_event_call *call = filp->private_data;
+        struct ftrace_event_call *call;
        struct trace_seq *s;
-        int r;
+        int r = -ENODEV;
        if (*ppos)
                return 0;
        s = kmalloc(sizeof(*s), GFP_KERNEL);
        if (!s)
                return -ENOMEM;
        trace_seq_init(s);
-        print_event_filter(call, s);
+        mutex_lock(&event_mutex);
-        r = simple_read_from_buffer(ubuf, cnt, ppos, s->buffer, s->len);
+        call = event_file_data(filp);
+        if (call)
+                print_event_filter(call, s);
+        mutex_unlock(&event_mutex);
+        if (call)
+                r = simple_read_from_buffer(ubuf, cnt, ppos, s->buffer, s->len);
        kfree(s);
@@ -1011,9 +1021,9 @@ static ssize_t
 event_filter_write(struct file *filp, const char __user *ubuf, size_t cnt,
                   loff_t *ppos)
 {
-        struct ftrace_event_call *call = filp->private_data;
+        struct ftrace_event_call *call;
        char *buf;
-        int err;
+        int err = -ENODEV;
        if (cnt >= PAGE_SIZE)
                return -EINVAL;
@@ -1028,7 +1038,12 @@ event_filter_write(struct file *filp, const char __user *ubuf, size_t cnt,
        }
        buf[cnt] = '\0';
-        err = apply_event_filter(call, buf);
+        mutex_lock(&event_mutex);
+        call = event_file_data(filp);
+        if (call)
+                err = apply_event_filter(call, buf);
+        mutex_unlock(&event_mutex);
        free_page((unsigned long) buf);
        if (err < 0)
                return err;
@@ -1218,6 +1233,7 @@ show_header(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
 static int ftrace_event_avail_open(struct inode *inode, struct file *file);
 static int ftrace_event_set_open(struct inode *inode, struct file *file);
+static int ftrace_event_release(struct inode *inode, struct file *file);
 static const struct seq_operations show_event_seq_ops = {
        .start = t_start,
@@ -1245,14 +1261,13 @@ static const struct file_operations ftrace_set_event_fops = {
        .read = seq_read,
        .write = ftrace_event_write,
        .llseek = seq_lseek,
-        .release = seq_release,
+        .release = ftrace_event_release,
 };
 static const struct file_operations ftrace_enable_fops = {
-        .open = tracing_open_generic_file,
+        .open = tracing_open_generic,
        .read = event_enable_read,
        .write = event_enable_write,
-        .release = tracing_release_generic_file,
        .llseek = default_llseek,
 };
@@ -1264,7 +1279,6 @@ static const struct file_operations ftrace_event_format_fops = {
 };
 static const struct file_operations ftrace_event_id_fops = {
-        .open = tracing_open_generic,
        .read = event_id_read,
        .llseek = default_llseek,
 };
@@ -1323,6 +1337,15 @@ ftrace_event_open(struct inode *inode, struct file *file,
        return ret;
 }
+static int ftrace_event_release(struct inode *inode, struct file *file)
+{
+        struct trace_array *tr = inode->i_private;
+        trace_array_put(tr);
+        return seq_release(inode, file);
+}
 static int
 ftrace_event_avail_open(struct inode *inode, struct file *file)
 {
@@ -1336,12 +1359,19 @@ ftrace_event_set_open(struct inode *inode, struct file *file)
 {
        const struct seq_operations *seq_ops = &show_set_event_seq_ops;
        struct trace_array *tr = inode->i_private;
+        int ret;
+        if (trace_array_get(tr) < 0)
+                return -ENODEV;
        if ((file->f_mode & FMODE_WRITE) &&
            (file->f_flags & O_TRUNC))
                ftrace_clear_events(tr);
-        return ftrace_event_open(inode, file, seq_ops);
+        ret = ftrace_event_open(inode, file, seq_ops);
+        if (ret < 0)
+                trace_array_put(tr);
+        return ret;
 }
 static struct event_subsystem *
@@ -1496,8 +1526,8 @@ event_create_dir(struct dentry *parent,
 #ifdef CONFIG_PERF_EVENTS
        if (call->event.type && call->class->reg)
-                trace_create_file("id", 0444, file->dir, call,
+                trace_create_file("id", 0444, file->dir,
-                                  id);
+                                  (void *)(long)call->event.type, id);
 #endif
        /*
@@ -1522,33 +1552,16 @@ event_create_dir(struct dentry *parent,
        return 0;
 }
-static void remove_subsystem(struct ftrace_subsystem_dir *dir)
-{
-        if (!dir)
-                return;
-        if (!--dir->nr_events) {
-                debugfs_remove_recursive(dir->entry);
-                list_del(&dir->list);
-                __put_system_dir(dir);
-        }
-}
 static void remove_event_from_tracers(struct ftrace_event_call *call)
 {
        struct ftrace_event_file *file;
        struct trace_array *tr;
        do_for_each_event_file_safe(tr, file) {
                if (file->event_call != call)
                        continue;
-                list_del(&file->list);
+                remove_event_file_dir(file);
-                debugfs_remove_recursive(file->dir);
-                remove_subsystem(file->system);
-                kmem_cache_free(file_cachep, file);
                /*
                 * The do_for_each_event_file_safe() is
                 * a double loop. After finding the call for this
@@ -1700,16 +1713,53 @@ static void __trace_remove_event_call(struct ftrace_event_call *call)
        destroy_preds(call);
 }
+static int probe_remove_event_call(struct ftrace_event_call *call)
+{
+        struct trace_array *tr;
+        struct ftrace_event_file *file;
+#ifdef CONFIG_PERF_EVENTS
+        if (call->perf_refcount)
+                return -EBUSY;
+#endif
+        do_for_each_event_file(tr, file) {
+                if (file->event_call != call)
+                        continue;
+                /*
+                 * We can't rely on ftrace_event_enable_disable(enable => 0)
+                 * we are going to do, FTRACE_EVENT_FL_SOFT_MODE can suppress
+                 * TRACE_REG_UNREGISTER.
+                 */
+                if (file->flags & FTRACE_EVENT_FL_ENABLED)
+                        return -EBUSY;
+                /*
+                 * The do_for_each_event_file_safe() is
+                 * a double loop. After finding the call for this
+                 * trace_array, we use break to jump to the next
+                 * trace_array.
+                 */
+                break;
+        } while_for_each_event_file();
+        __trace_remove_event_call(call);
+        return 0;
+}
 /* Remove an event_call */
-void trace_remove_event_call(struct ftrace_event_call *call)
+int trace_remove_event_call(struct ftrace_event_call *call)
 {
+        int ret;
        mutex_lock(&trace_types_lock);
        mutex_lock(&event_mutex);
        down_write(&trace_event_sem);
-        __trace_remove_event_call(call);
+        ret = probe_remove_event_call(call);
        up_write(&trace_event_sem);
        mutex_unlock(&event_mutex);
        mutex_unlock(&trace_types_lock);
+        return ret;
 }
 #define for_each_event(event, start, end)                       \
@@ -2278,12 +2328,8 @@ __trace_remove_event_dirs(struct trace_array *tr)
 {
        struct ftrace_event_file *file, *next;
-        list_for_each_entry_safe(file, next, &tr->events, list) {
+        list_for_each_entry_safe(file, next, &tr->events, list)
-                list_del(&file->list);
+                remove_event_file_dir(file);
-                debugfs_remove_recursive(file->dir);
-                remove_subsystem(file->system);
-                kmem_cache_free(file_cachep, file);
-        }
 }
 static void
diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c
index 0d883dc057d6..97daa8cf958d 100644
--- a/kernel/trace/trace_events_filter.c
+++ b/kernel/trace/trace_events_filter.c
@@ -637,17 +637,15 @@ static void append_filter_err(struct filter_parse_state *ps,
        free_page((unsigned long) buf);
 }
+/* caller must hold event_mutex */
 void print_event_filter(struct ftrace_event_call *call, struct trace_seq *s)
 {
-        struct event_filter *filter;
+        struct event_filter *filter = call->filter;
-        mutex_lock(&event_mutex);
-        filter = call->filter;
        if (filter && filter->filter_string)
                trace_seq_printf(s, "%s\n", filter->filter_string);
        else
-                trace_seq_printf(s, "none\n");
+                trace_seq_puts(s, "none\n");
-        mutex_unlock(&event_mutex);
 }
 void print_subsystem_event_filter(struct event_subsystem *system,
@@ -660,7 +658,7 @@ void print_subsystem_event_filter(struct event_subsystem *system,
        if (filter && filter->filter_string)
                trace_seq_printf(s, "%s\n", filter->filter_string);
        else
-                trace_seq_printf(s, DEFAULT_SYS_FILTER_MESSAGE "\n");
+                trace_seq_puts(s, DEFAULT_SYS_FILTER_MESSAGE "\n");
        mutex_unlock(&event_mutex);
 }
@@ -1841,23 +1839,22 @@ static int create_system_filter(struct event_subsystem *system,
        return err;
 }
+/* caller must hold event_mutex */
 int apply_event_filter(struct ftrace_event_call *call, char *filter_string)
 {
        struct event_filter *filter;
-        int err = 0;
+        int err;
-        mutex_lock(&event_mutex);
        if (!strcmp(strstrip(filter_string), "0")) {
                filter_disable(call);
                filter = call->filter;
                if (!filter)
-                        goto out_unlock;
+                        return 0;
                RCU_INIT_POINTER(call->filter, NULL);
                /* Make sure the filter is not being used */
                synchronize_sched();
                __free_filter(filter);
-                goto out_unlock;
+                return 0;
        }
        err = create_filter(call, filter_string, true, &filter);
@@ -1884,8 +1881,6 @@ int apply_event_filter(struct ftrace_event_call *call, char *filter_string)
                        __free_filter(tmp);
                }
        }
-out_unlock:
-        mutex_unlock(&event_mutex);
        return err;
 }
diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c
index b863f93b30f3..38fe1483c508 100644
--- a/kernel/trace/trace_functions.c
+++ b/kernel/trace/trace_functions.c
@@ -199,7 +199,7 @@ static int func_set_flag(u32 old_flags, u32 bit, int set)
        return 0;
 }
-static struct tracer function_trace __read_mostly =
+static struct tracer function_trace __tracer_data =
 {
        .name           = "function",
        .init           = function_trace_init,
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c
index 8388bc99f2ee..b5c09242683d 100644
--- a/kernel/trace/trace_functions_graph.c
+++ b/kernel/trace/trace_functions_graph.c
@@ -446,7 +446,7 @@ print_graph_proc(struct trace_seq *s, pid_t pid)
        /* First spaces to align center */
        for (i = 0; i < spaces / 2; i++) {
-                ret = trace_seq_printf(s, " ");
+                ret = trace_seq_putc(s, ' ');
                if (!ret)
                        return TRACE_TYPE_PARTIAL_LINE;
        }
@@ -457,7 +457,7 @@ print_graph_proc(struct trace_seq *s, pid_t pid)
        /* Last spaces to align center */
        for (i = 0; i < spaces - (spaces / 2); i++) {
-                ret = trace_seq_printf(s, " ");
+                ret = trace_seq_putc(s, ' ');
                if (!ret)
                        return TRACE_TYPE_PARTIAL_LINE;
        }
@@ -503,7 +503,7 @@ verif_pid(struct trace_seq *s, pid_t pid, int cpu, struct fgraph_data *data)
 ------------------------------------------
 */
-        ret = trace_seq_printf(s,
+        ret = trace_seq_puts(s,
                " ------------------------------------------\n");
        if (!ret)
                return TRACE_TYPE_PARTIAL_LINE;
@@ -516,7 +516,7 @@ verif_pid(struct trace_seq *s, pid_t pid, int cpu, struct fgraph_data *data)
        if (ret == TRACE_TYPE_PARTIAL_LINE)
                return TRACE_TYPE_PARTIAL_LINE;
-        ret = trace_seq_printf(s, " => ");
+        ret = trace_seq_puts(s, " => ");
        if (!ret)
                return TRACE_TYPE_PARTIAL_LINE;
@@ -524,7 +524,7 @@ verif_pid(struct trace_seq *s, pid_t pid, int cpu, struct fgraph_data *data)
        if (ret == TRACE_TYPE_PARTIAL_LINE)
                return TRACE_TYPE_PARTIAL_LINE;
-        ret = trace_seq_printf(s,
+        ret = trace_seq_puts(s,
                "\n ------------------------------------------\n\n");
        if (!ret)
                return TRACE_TYPE_PARTIAL_LINE;
@@ -645,7 +645,7 @@ print_graph_irq(struct trace_iterator *iter, unsigned long addr,
                        ret = print_graph_proc(s, pid);
                        if (ret == TRACE_TYPE_PARTIAL_LINE)
                                return TRACE_TYPE_PARTIAL_LINE;
-                        ret = trace_seq_printf(s, " | ");
+                        ret = trace_seq_puts(s, " | ");
                        if (!ret)
                                return TRACE_TYPE_PARTIAL_LINE;
                }
@@ -657,9 +657,9 @@ print_graph_irq(struct trace_iterator *iter, unsigned long addr,
                return ret;
        if (type == TRACE_GRAPH_ENT)
-                ret = trace_seq_printf(s, "==========>");
+                ret = trace_seq_puts(s, "==========>");
        else
-                ret = trace_seq_printf(s, "<==========");
+                ret = trace_seq_puts(s, "<==========");
        if (!ret)
                return TRACE_TYPE_PARTIAL_LINE;
@@ -668,7 +668,7 @@ print_graph_irq(struct trace_iterator *iter, unsigned long addr,
        if (ret != TRACE_TYPE_HANDLED)
                return ret;
-        ret = trace_seq_printf(s, "\n");
+        ret = trace_seq_putc(s, '\n');
        if (!ret)
                return TRACE_TYPE_PARTIAL_LINE;
@@ -705,13 +705,13 @@ trace_print_graph_duration(unsigned long long duration, struct trace_seq *s)
                len += strlen(nsecs_str);
        }
-        ret = trace_seq_printf(s, " us ");
+        ret = trace_seq_puts(s, " us ");
        if (!ret)
                return TRACE_TYPE_PARTIAL_LINE;
        /* Print remaining spaces to fit the row's width */
        for (i = len; i < 7; i++) {
-                ret = trace_seq_printf(s, " ");
+                ret = trace_seq_putc(s, ' ');
                if (!ret)
                        return TRACE_TYPE_PARTIAL_LINE;
        }
@@ -731,13 +731,13 @@ print_graph_duration(unsigned long long duration, struct trace_seq *s,
        /* No real adata, just filling the column with spaces */
        switch (duration) {
        case DURATION_FILL_FULL:
-                ret = trace_seq_printf(s, "              |  ");
+                ret = trace_seq_puts(s, "              |  ");
                return ret ? TRACE_TYPE_HANDLED : TRACE_TYPE_PARTIAL_LINE;
        case DURATION_FILL_START:
-                ret = trace_seq_printf(s, "  ");
+                ret = trace_seq_puts(s, "  ");
                return ret ? TRACE_TYPE_HANDLED : TRACE_TYPE_PARTIAL_LINE;
        case DURATION_FILL_END:
-                ret = trace_seq_printf(s, " |");
+                ret = trace_seq_puts(s, " |");
                return ret ? TRACE_TYPE_HANDLED : TRACE_TYPE_PARTIAL_LINE;
        }
@@ -745,10 +745,10 @@ print_graph_duration(unsigned long long duration, struct trace_seq *s,
        if (flags & TRACE_GRAPH_PRINT_OVERHEAD) {
                /* Duration exceeded 100 msecs */
                if (duration > 100000ULL)
-                        ret = trace_seq_printf(s, "! ");
+                        ret = trace_seq_puts(s, "! ");
                /* Duration exceeded 10 msecs */
                else if (duration > 10000ULL)
-                        ret = trace_seq_printf(s, "+ ");
+                        ret = trace_seq_puts(s, "+ ");
        }
        /*
@@ -757,7 +757,7 @@ print_graph_duration(unsigned long long duration, struct trace_seq *s,
         * to fill out the space.
         */
        if (ret == -1)
-                ret = trace_seq_printf(s, "  ");
+                ret = trace_seq_puts(s, "  ");
        /* Catching here any failure happenned above */
        if (!ret)
@@ -767,7 +767,7 @@ print_graph_duration(unsigned long long duration, struct trace_seq *s,
        if (ret != TRACE_TYPE_HANDLED)
                return ret;
-        ret = trace_seq_printf(s, "|  ");
+        ret = trace_seq_puts(s, "|  ");
        if (!ret)
                return TRACE_TYPE_PARTIAL_LINE;
@@ -817,7 +817,7 @@ print_graph_entry_leaf(struct trace_iterator *iter,
        /* Function */
        for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) {
-                ret = trace_seq_printf(s, " ");
+                ret = trace_seq_putc(s, ' ');
                if (!ret)
                        return TRACE_TYPE_PARTIAL_LINE;
        }
@@ -858,7 +858,7 @@ print_graph_entry_nested(struct trace_iterator *iter,
        /* Function */
        for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) {
-                ret = trace_seq_printf(s, " ");
+                ret = trace_seq_putc(s, ' ');
                if (!ret)
                        return TRACE_TYPE_PARTIAL_LINE;
        }
@@ -917,7 +917,7 @@ print_graph_prologue(struct trace_iterator *iter, struct trace_seq *s,
                if (ret == TRACE_TYPE_PARTIAL_LINE)
                        return TRACE_TYPE_PARTIAL_LINE;
-                ret = trace_seq_printf(s, " | ");
+                ret = trace_seq_puts(s, " | ");
                if (!ret)
                        return TRACE_TYPE_PARTIAL_LINE;
        }
@@ -1117,7 +1117,7 @@ print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s,
        /* Closing brace */
        for (i = 0; i < trace->depth * TRACE_GRAPH_INDENT; i++) {
-                ret = trace_seq_printf(s, " ");
+                ret = trace_seq_putc(s, ' ');
                if (!ret)
                        return TRACE_TYPE_PARTIAL_LINE;
        }
@@ -1129,7 +1129,7 @@ print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s,
         * belongs to, write out the function name.
         */
        if (func_match) {
-                ret = trace_seq_printf(s, "}\n");
+                ret = trace_seq_puts(s, "}\n");
                if (!ret)
                        return TRACE_TYPE_PARTIAL_LINE;
        } else {
@@ -1179,13 +1179,13 @@ print_graph_comment(struct trace_seq *s, struct trace_entry *ent,
        /* Indentation */
        if (depth > 0)
                for (i = 0; i < (depth + 1) * TRACE_GRAPH_INDENT; i++) {
-                        ret = trace_seq_printf(s, " ");
+                        ret = trace_seq_putc(s, ' ');
                        if (!ret)
                                return TRACE_TYPE_PARTIAL_LINE;
                }
        /* The comment */
-        ret = trace_seq_printf(s, "/* ");
+        ret = trace_seq_puts(s, "/* ");
        if (!ret)
                return TRACE_TYPE_PARTIAL_LINE;
@@ -1216,7 +1216,7 @@ print_graph_comment(struct trace_seq *s, struct trace_entry *ent,
                s->len--;
        }
-        ret = trace_seq_printf(s, " */\n");
+        ret = trace_seq_puts(s, " */\n");
        if (!ret)
                return TRACE_TYPE_PARTIAL_LINE;
@@ -1448,7 +1448,7 @@ static struct trace_event graph_trace_ret_event = {
        .funcs          = &graph_functions
 };
-static struct tracer graph_trace __read_mostly = {
+static struct tracer graph_trace __tracer_data = {
        .name           = "function_graph",
        .open           = graph_trace_open,
        .pipe_open      = graph_trace_open,
diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c
index 7ed6976493c8..243f6834d026 100644
--- a/kernel/trace/trace_kprobe.c
+++ b/kernel/trace/trace_kprobe.c
@@ -95,7 +95,7 @@ static __kprobes bool trace_probe_is_on_module(struct trace_probe *tp)
 }
 static int register_probe_event(struct trace_probe *tp);
-static void unregister_probe_event(struct trace_probe *tp);
+static int unregister_probe_event(struct trace_probe *tp);
 static DEFINE_MUTEX(probe_lock);
 static LIST_HEAD(probe_list);
@@ -243,11 +243,11 @@ find_event_file_link(struct trace_probe *tp, struct ftrace_event_file *file)
 static int
 disable_trace_probe(struct trace_probe *tp, struct ftrace_event_file *file)
 {
+        struct event_file_link *link = NULL;
+        int wait = 0;
        int ret = 0;
        if (file) {
-                struct event_file_link *link;
                link = find_event_file_link(tp, file);
                if (!link) {
                        ret = -EINVAL;
@@ -255,10 +255,7 @@ disable_trace_probe(struct trace_probe *tp, struct ftrace_event_file *file)
                }
                list_del_rcu(&link->list);
-                /* synchronize with kprobe_trace_func/kretprobe_trace_func */
+                wait = 1;
-                synchronize_sched();
-                kfree(link);
                if (!list_empty(&tp->files))
                        goto out;
@@ -271,8 +268,22 @@ disable_trace_probe(struct trace_probe *tp, struct ftrace_event_file *file)
                        disable_kretprobe(&tp->rp);
                else
                        disable_kprobe(&tp->rp.kp);
+                wait = 1;
        }
 out:
+        if (wait) {
+                /*
+                 * Synchronize with kprobe_trace_func/kretprobe_trace_func
+                 * to ensure disabled (all running handlers are finished).
+                 * This is not only for kfree(), but also the caller,
+                 * trace_remove_event_call() supposes it for releasing
+                 * event_call related objects, which will be accessed in
+                 * the kprobe_trace_func/kretprobe_trace_func.
+                 */
+                synchronize_sched();
+                kfree(link);    /* Ignored if link == NULL */
+        }
        return ret;
 }
@@ -340,9 +351,12 @@ static int unregister_trace_probe(struct trace_probe *tp)
        if (trace_probe_is_enabled(tp))
                return -EBUSY;
+        /* Will fail if probe is being used by ftrace or perf */
+        if (unregister_probe_event(tp))
+                return -EBUSY;
        __unregister_trace_probe(tp);
        list_del(&tp->list);
-        unregister_probe_event(tp);
        return 0;
 }
@@ -621,7 +635,9 @@ static int release_all_trace_probes(void)
        /* TODO: Use batch unregistration */
        while (!list_empty(&probe_list)) {
                tp = list_entry(probe_list.next, struct trace_probe, list);
-                unregister_trace_probe(tp);
+                ret = unregister_trace_probe(tp);
+                if (ret)
+                        goto end;
                free_trace_probe(tp);
        }
@@ -1087,9 +1103,6 @@ kprobe_perf_func(struct trace_probe *tp, struct pt_regs *regs)
        __size = sizeof(*entry) + tp->size + dsize;
        size = ALIGN(__size + sizeof(u32), sizeof(u64));
        size -= sizeof(u32);
-        if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
-                     "profile buffer not large enough"))
-                return;
        entry = perf_trace_buf_prepare(size, call->event.type, regs, &rctx);
        if (!entry)
@@ -1120,9 +1133,6 @@ kretprobe_perf_func(struct trace_probe *tp, struct kretprobe_instance *ri,
        __size = sizeof(*entry) + tp->size + dsize;
        size = ALIGN(__size + sizeof(u32), sizeof(u64));
        size -= sizeof(u32);
-        if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
-                     "profile buffer not large enough"))
-                return;
        entry = perf_trace_buf_prepare(size, call->event.type, regs, &rctx);
        if (!entry)
@@ -1242,11 +1252,15 @@ static int register_probe_event(struct trace_probe *tp)
        return ret;
 }
-static void unregister_probe_event(struct trace_probe *tp)
+static int unregister_probe_event(struct trace_probe *tp)
 {
+        int ret;
        /* tp->event is unregistered in trace_remove_event_call() */
-        trace_remove_event_call(&tp->call);
+        ret = trace_remove_event_call(&tp->call);
-        kfree(tp->call.print_fmt);
+        if (!ret)
+                kfree(tp->call.print_fmt);
+        return ret;
 }
 /* Make a debugfs interface for controlling probe points */
diff --git a/kernel/trace/trace_mmiotrace.c b/kernel/trace/trace_mmiotrace.c
index a5e8f4878bfa..b3dcfb2f0fef 100644
--- a/kernel/trace/trace_mmiotrace.c
+++ b/kernel/trace/trace_mmiotrace.c
@@ -90,7 +90,7 @@ static int mmio_print_pcidev(struct trace_seq *s, const struct pci_dev *dev)
        if (drv)
                ret += trace_seq_printf(s, " %s\n", drv->name);
        else
-                ret += trace_seq_printf(s, " \n");
+                ret += trace_seq_puts(s, " \n");
        return ret;
 }
@@ -107,7 +107,7 @@ static void mmio_pipe_open(struct trace_iterator *iter)
        struct header_iter *hiter;
        struct trace_seq *s = &iter->seq;
-        trace_seq_printf(s, "VERSION 20070824\n");
+        trace_seq_puts(s, "VERSION 20070824\n");
        hiter = kzalloc(sizeof(*hiter), GFP_KERNEL);
        if (!hiter)
@@ -209,7 +209,7 @@ static enum print_line_t mmio_print_rw(struct trace_iterator *iter)
                        (rw->value >> 0) & 0xff, rw->pc, 0);
                break;
        default:
-                ret = trace_seq_printf(s, "rw what?\n");
+                ret = trace_seq_puts(s, "rw what?\n");
                break;
        }
        if (ret)
@@ -245,7 +245,7 @@ static enum print_line_t mmio_print_map(struct trace_iterator *iter)
                        secs, usec_rem, m->map_id, 0UL, 0);
                break;
        default:
-                ret = trace_seq_printf(s, "map what?\n");
+                ret = trace_seq_puts(s, "map what?\n");
                break;
        }
        if (ret)
diff --git a/kernel/trace/trace_output.c b/kernel/trace/trace_output.c
index bb922d9ee51b..34e7cbac0c9c 100644
--- a/kernel/trace/trace_output.c
+++ b/kernel/trace/trace_output.c
@@ -78,7 +78,7 @@ enum print_line_t trace_print_printk_msg_only(struct trace_iterator *iter)
        trace_assign_type(field, entry);
-        ret = trace_seq_printf(s, "%s", field->buf);
+        ret = trace_seq_puts(s, field->buf);
        if (!ret)
                return TRACE_TYPE_PARTIAL_LINE;
@@ -558,14 +558,14 @@ seq_print_userip_objs(const struct userstack_entry *entry, struct trace_seq *s,
                        if (ret)
                                ret = trace_seq_puts(s, "??");
                        if (ret)
-                                ret = trace_seq_puts(s, "\n");
+                                ret = trace_seq_putc(s, '\n');
                        continue;
                }
                if (!ret)
                        break;
                if (ret)
                        ret = seq_print_user_ip(s, mm, ip, sym_flags);
-                ret = trace_seq_puts(s, "\n");
+                ret = trace_seq_putc(s, '\n');
        }
        if (mm)
@@ -579,7 +579,7 @@ seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags)
        int ret;
        if (!ip)
-                return trace_seq_printf(s, "0");
+                return trace_seq_putc(s, '0');
        if (sym_flags & TRACE_ITER_SYM_OFFSET)
                ret = seq_print_sym_offset(s, "%s", ip);
@@ -964,14 +964,14 @@ static enum print_line_t trace_fn_trace(struct trace_iterator *iter, int flags,
                goto partial;
        if ((flags & TRACE_ITER_PRINT_PARENT) && field->parent_ip) {
-                if (!trace_seq_printf(s, " <-"))
+                if (!trace_seq_puts(s, " <-"))
                        goto partial;
                if (!seq_print_ip_sym(s,
                                      field->parent_ip,
                                      flags))
                        goto partial;
        }
-        if (!trace_seq_printf(s, "\n"))
+        if (!trace_seq_putc(s, '\n'))
                goto partial;
        return TRACE_TYPE_HANDLED;
@@ -1210,7 +1210,7 @@ static enum print_line_t trace_stack_print(struct trace_iterator *iter,
                if (!seq_print_ip_sym(s, *p, flags))
                        goto partial;
-                if (!trace_seq_puts(s, "\n"))
+                if (!trace_seq_putc(s, '\n'))
                        goto partial;
        }
diff --git a/kernel/trace/trace_printk.c b/kernel/trace/trace_printk.c
index a9077c1b4ad3..2900817ba65c 100644
--- a/kernel/trace/trace_printk.c
+++ b/kernel/trace/trace_printk.c
@@ -244,12 +244,31 @@ static const char **find_next(void *v, loff_t *pos)
 {
        const char **fmt = v;
        int start_index;
+        int last_index;
        start_index = __stop___trace_bprintk_fmt - __start___trace_bprintk_fmt;
        if (*pos < start_index)
                return __start___trace_bprintk_fmt + *pos;
+        /*
+         * The __tracepoint_str section is treated the same as the
+         * __trace_printk_fmt section. The difference is that the
+         * __trace_printk_fmt section should only be used by trace_printk()
+         * in a debugging environment, as if anything exists in that section
+         * the trace_prink() helper buffers are allocated, which would just
+         * waste space in a production environment.
+         *
+         * The __tracepoint_str sections on the other hand are used by
+         * tracepoints which need to map pointers to their strings to
+         * the ASCII text for userspace.
+         */
+        last_index = start_index;
+        start_index = __stop___tracepoint_str - __start___tracepoint_str;
+        if (*pos < last_index + start_index)
+                return __start___tracepoint_str + (*pos - last_index);
        return find_next_mod_format(start_index, v, fmt, pos);
 }
diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c
index 322e16461072..8fd03657bc7d 100644
--- a/kernel/trace/trace_syscalls.c
+++ b/kernel/trace/trace_syscalls.c
@@ -175,7 +175,7 @@ print_syscall_exit(struct trace_iterator *iter, int flags,
        entry = syscall_nr_to_meta(syscall);
        if (!entry) {
-                trace_seq_printf(s, "\n");
+                trace_seq_putc(s, '\n');
                return TRACE_TYPE_HANDLED;
        }
@@ -566,15 +566,15 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
        if (!sys_data)
                return;
+        head = this_cpu_ptr(sys_data->enter_event->perf_events);
+        if (hlist_empty(head))
+                return;
        /* get the size after alignment with the u32 buffer size field */
        size = sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
        size = ALIGN(size + sizeof(u32), sizeof(u64));
        size -= sizeof(u32);
-        if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
-                      "perf buffer not large enough"))
-                return;
        rec = (struct syscall_trace_enter *)perf_trace_buf_prepare(size,
                                sys_data->enter_event->event.type, regs, &rctx);
        if (!rec)
@@ -583,8 +583,6 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
        rec->nr = syscall_nr;
        syscall_get_arguments(current, regs, 0, sys_data->nb_args,
                               (unsigned long *)&rec->args);
-        head = this_cpu_ptr(sys_data->enter_event->perf_events);
        perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head, NULL);
 }
@@ -642,18 +640,14 @@ static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
        if (!sys_data)
                return;
+        head = this_cpu_ptr(sys_data->exit_event->perf_events);
+        if (hlist_empty(head))
+                return;
        /* We can probably do that at build time */
        size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
        size -= sizeof(u32);
-        /*
-         * Impossible, but be paranoid with the future
-         * How to put this check outside runtime?
-         */
-        if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
-                "exit event has grown above perf buffer size"))
-                return;
        rec = (struct syscall_trace_exit *)perf_trace_buf_prepare(size,
                                sys_data->exit_event->event.type, regs, &rctx);
        if (!rec)
@@ -661,8 +655,6 @@ static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
        rec->nr = syscall_nr;
        rec->ret = syscall_get_return_value(current, regs);
-        head = this_cpu_ptr(sys_data->exit_event->perf_events);
        perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head, NULL);
 }
diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c
index d5d0cd368a56..272261b5f94f 100644
--- a/kernel/trace/trace_uprobe.c
+++ b/kernel/trace/trace_uprobe.c
@@ -70,7 +70,7 @@ struct trace_uprobe {
        (sizeof(struct probe_arg) * (n)))
 static int register_uprobe_event(struct trace_uprobe *tu);
-static void unregister_uprobe_event(struct trace_uprobe *tu);
+static int unregister_uprobe_event(struct trace_uprobe *tu);
 static DEFINE_MUTEX(uprobe_lock);
 static LIST_HEAD(uprobe_list);
@@ -164,11 +164,17 @@ static struct trace_uprobe *find_probe_event(const char *event, const char *grou
 }
 /* Unregister a trace_uprobe and probe_event: call with locking uprobe_lock */
-static void unregister_trace_uprobe(struct trace_uprobe *tu)
+static int unregister_trace_uprobe(struct trace_uprobe *tu)
 {
+        int ret;
+        ret = unregister_uprobe_event(tu);
+        if (ret)
+                return ret;
        list_del(&tu->list);
-        unregister_uprobe_event(tu);
        free_trace_uprobe(tu);
+        return 0;
 }
 /* Register a trace_uprobe and probe_event */
@@ -181,9 +187,12 @@ static int register_trace_uprobe(struct trace_uprobe *tu)
        /* register as an event */
        old_tp = find_probe_event(tu->call.name, tu->call.class->system);
-        if (old_tp)
+        if (old_tp) {
                /* delete old event */
-                unregister_trace_uprobe(old_tp);
+                ret = unregister_trace_uprobe(old_tp);
+                if (ret)
+                        goto end;
+        }
        ret = register_uprobe_event(tu);
        if (ret) {
@@ -256,6 +265,8 @@ static int create_trace_uprobe(int argc, char **argv)
                group = UPROBE_EVENT_SYSTEM;
        if (is_delete) {
+                int ret;
                if (!event) {
                        pr_info("Delete command needs an event name.\n");
                        return -EINVAL;
@@ -269,9 +280,9 @@ static int create_trace_uprobe(int argc, char **argv)
                        return -ENOENT;
                }
                /* delete an event */
-                unregister_trace_uprobe(tu);
+                ret = unregister_trace_uprobe(tu);
                mutex_unlock(&uprobe_lock);
-                return 0;
+                return ret;
        }
        if (argc < 2) {
@@ -408,16 +419,20 @@ fail_address_parse:
        return ret;
 }
-static void cleanup_all_probes(void)
+static int cleanup_all_probes(void)
 {
        struct trace_uprobe *tu;
+        int ret = 0;
        mutex_lock(&uprobe_lock);
        while (!list_empty(&uprobe_list)) {
                tu = list_entry(uprobe_list.next, struct trace_uprobe, list);
-                unregister_trace_uprobe(tu);
+                ret = unregister_trace_uprobe(tu);
+                if (ret)
+                        break;
        }
        mutex_unlock(&uprobe_lock);
+        return ret;
 }
 /* Probes listing interfaces */
@@ -462,8 +477,13 @@ static const struct seq_operations probes_seq_op = {
 static int probes_open(struct inode *inode, struct file *file)
 {
-        if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC))
+        int ret;
-                cleanup_all_probes();
+        if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
+                ret = cleanup_all_probes();
+                if (ret)
+                        return ret;
+        }
        return seq_open(file, &probes_seq_op);
 }
@@ -818,8 +838,6 @@ static void uprobe_perf_print(struct trace_uprobe *tu,
        size = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
        size = ALIGN(size + tu->size + sizeof(u32), sizeof(u64)) - sizeof(u32);
-        if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE, "profile buffer not large enough"))
-                return;
        preempt_disable();
        head = this_cpu_ptr(call->perf_events);
@@ -970,12 +988,17 @@ static int register_uprobe_event(struct trace_uprobe *tu)
        return ret;
 }
-static void unregister_uprobe_event(struct trace_uprobe *tu)
+static int unregister_uprobe_event(struct trace_uprobe *tu)
 {
+        int ret;
        /* tu->event is unregistered in trace_remove_event_call() */
-        trace_remove_event_call(&tu->call);
+        ret = trace_remove_event_call(&tu->call);
+        if (ret)
+                return ret;
        kfree(tu->call.print_fmt);
        tu->call.print_fmt = NULL;
+        return 0;
 }
 /* Make a trace interface for controling probe points */
diff --git a/kernel/user_namespace.c b/kernel/user_namespace.c
index d8c30db06c5b..9064b919a406 100644
--- a/kernel/user_namespace.c
+++ b/kernel/user_namespace.c
@@ -62,6 +62,9 @@ int create_user_ns(struct cred *new)
        kgid_t group = new->egid;
        int ret;
+        if (parent_ns->level > 32)
+                return -EUSERS;
        /*
         * Verify that we can not violate the policy of which files
         * may be accessed that is specified by the root directory,
@@ -92,6 +95,7 @@ int create_user_ns(struct cred *new)
        atomic_set(&ns->count, 1);
        /* Leave the new->user_ns reference with the new user namespace. */
        ns->parent = parent_ns;
+        ns->level = parent_ns->level + 1;
        ns->owner = owner;
        ns->group = group;
@@ -105,16 +109,21 @@ int create_user_ns(struct cred *new)
 int unshare_userns(unsigned long unshare_flags, struct cred **new_cred)
 {
        struct cred *cred;
+        int err = -ENOMEM;
        if (!(unshare_flags & CLONE_NEWUSER))
                return 0;
        cred = prepare_creds();
-        if (!cred)
+        if (cred) {
-                return -ENOMEM;
+                err = create_user_ns(cred);
+                if (err)
+                        put_cred(cred);
+                else
+                        *new_cred = cred;
+        }
-        *new_cred = cred;
+        return err;
-        return create_user_ns(cred);
 }
 void free_user_ns(struct user_namespace *ns)
diff --git a/kernel/wait.c b/kernel/wait.c
index ce0daa320a26..d550920e040c 100644
--- a/kernel/wait.c
+++ b/kernel/wait.c
@@ -333,7 +333,8 @@ int __wait_on_atomic_t(wait_queue_head_t *wq, struct wait_bit_queue *q,
                prepare_to_wait(wq, &q->wait, mode);
                val = q->key.flags;
                if (atomic_read(val) == 0)
-                        ret = (*action)(val);
+                        break;
+                ret = (*action)(val);
        } while (!ret && atomic_read(val) != 0);
        finish_wait(wq, &q->wait);
        return ret;
@@ -362,8 +363,7 @@ EXPORT_SYMBOL(out_of_line_wait_on_atomic_t);
 /**
 * wake_up_atomic_t - Wake up a waiter on a atomic_t
- * @word: The word being waited on, a kernel virtual address
+ * @p: The atomic_t being waited on, a kernel virtual address
- * @bit: The bit of the word being waited on
 *
 * Wake up anyone waiting for the atomic_t to go to zero.
 *
diff --git a/kernel/watchdog.c b/kernel/watchdog.c
index 1241d8c91d5e..51c4f34d258e 100644
--- a/kernel/watchdog.c
+++ b/kernel/watchdog.c
@@ -553,14 +553,6 @@ void __init lockup_detector_init(void)
 {
        set_sample_period();
-#ifdef CONFIG_NO_HZ_FULL
-        if (watchdog_user_enabled) {
-                watchdog_user_enabled = 0;
-                pr_warning("Disabled lockup detectors by default for full dynticks\n");
-                pr_warning("You can reactivate it with 'sysctl -w kernel.watchdog=1'\n");
-        }
-#endif
        if (watchdog_user_enabled)
                watchdog_enable_all_cpus();
 }
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index f02c4a4a0c3c..29b79852a845 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -16,9 +16,10 @@
 *
 * This is the generic async execution mechanism.  Work items as are
 * executed in process context.  The worker pool is shared and
- * automatically managed.  There is one worker pool for each CPU and
+ * automatically managed.  There are two worker pools for each CPU (one for
- * one extra for works which are better served by workers which are
+ * normal work items and the other for high priority ones) and some extra
- * not bound to any specific CPU.
+ * pools for workqueues which are not bound to any specific CPU - the
+ * number of these backing pools is dynamic.
 *
 * Please read Documentation/workqueue.txt for details.
 */
@@ -2033,8 +2034,11 @@ static bool maybe_destroy_workers(struct worker_pool *pool)
 * multiple times.  Does GFP_KERNEL allocations.
 *
 * RETURNS:
- * spin_lock_irq(pool->lock) which may be released and regrabbed
+ * %false if the pool don't need management and the caller can safely start
- * multiple times.  Does GFP_KERNEL allocations.
+ * processing works, %true indicates that the function released pool->lock
+ * and reacquired it to perform some management function and that the
+ * conditions that the caller verified while holding the lock before
+ * calling the function might no longer be true.
 */
 static bool manage_workers(struct worker *worker)
 {
@@ -2201,6 +2205,15 @@ __acquires(&pool->lock)
                dump_stack();
        }
+        /*
+         * The following prevents a kworker from hogging CPU on !PREEMPT
+         * kernels, where a requeueing work item waiting for something to
+         * happen could deadlock with stop_machine as such work item could
+         * indefinitely requeue itself while all other CPUs are trapped in
+         * stop_machine.
+         */
+        cond_resched();
        spin_lock_irq(&pool->lock);
        /* clear cpu intensive status */
@@ -2817,6 +2830,19 @@ already_gone:
        return false;
 }
+static bool __flush_work(struct work_struct *work)
+{
+        struct wq_barrier barr;
+        if (start_flush_work(work, &barr)) {
+                wait_for_completion(&barr.done);
+                destroy_work_on_stack(&barr.work);
+                return true;
+        } else {
+                return false;
+        }
+}
 /**
 * flush_work - wait for a work to finish executing the last queueing instance
 * @work: the work to flush
@@ -2830,18 +2856,10 @@ already_gone:
 */
 bool flush_work(struct work_struct *work)
 {
-        struct wq_barrier barr;
        lock_map_acquire(&work->lockdep_map);
        lock_map_release(&work->lockdep_map);
-        if (start_flush_work(work, &barr)) {
+        return __flush_work(work);
-                wait_for_completion(&barr.done);
-                destroy_work_on_stack(&barr.work);
-                return true;
-        } else {
-                return false;
-        }
 }
 EXPORT_SYMBOL_GPL(flush_work);
@@ -3081,25 +3099,26 @@ static struct workqueue_struct *dev_to_wq(struct device *dev)
        return wq_dev->wq;
 }
-static ssize_t wq_per_cpu_show(struct device *dev,
+static ssize_t per_cpu_show(struct device *dev, struct device_attribute *attr,
-                               struct device_attribute *attr, char *buf)
+                            char *buf)
 {
        struct workqueue_struct *wq = dev_to_wq(dev);
        return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND));
 }
+static DEVICE_ATTR_RO(per_cpu);
-static ssize_t wq_max_active_show(struct device *dev,
+static ssize_t max_active_show(struct device *dev,
-                                  struct device_attribute *attr, char *buf)
+                               struct device_attribute *attr, char *buf)
 {
        struct workqueue_struct *wq = dev_to_wq(dev);
        return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active);
 }
-static ssize_t wq_max_active_store(struct device *dev,
+static ssize_t max_active_store(struct device *dev,
-                                   struct device_attribute *attr,
+                                struct device_attribute *attr, const char *buf,
-                                   const char *buf, size_t count)
+                                size_t count)
 {
        struct workqueue_struct *wq = dev_to_wq(dev);
        int val;
@@ -3110,12 +3129,14 @@ static ssize_t wq_max_active_store(struct device *dev,
        workqueue_set_max_active(wq, val);
        return count;
 }
+static DEVICE_ATTR_RW(max_active);
-static struct device_attribute wq_sysfs_attrs[] = {
+static struct attribute *wq_sysfs_attrs[] = {
-        __ATTR(per_cpu, 0444, wq_per_cpu_show, NULL),
+        &dev_attr_per_cpu.attr,
-        __ATTR(max_active, 0644, wq_max_active_show, wq_max_active_store),
+        &dev_attr_max_active.attr,
-        __ATTR_NULL,
+        NULL,
 };
+ATTRIBUTE_GROUPS(wq_sysfs);
 static ssize_t wq_pool_ids_show(struct device *dev,
                                struct device_attribute *attr, char *buf)
@@ -3265,7 +3286,7 @@ static struct device_attribute wq_sysfs_unbound_attrs[] = {
 static struct bus_type wq_subsys = {
        .name                           = "workqueue",
-        .dev_attrs                      = wq_sysfs_attrs,
+        .dev_groups                     = wq_sysfs_groups,
 };
 static int __init wq_sysfs_init(void)
@@ -3411,6 +3432,12 @@ static void copy_workqueue_attrs(struct workqueue_attrs *to,
 {
        to->nice = from->nice;
        cpumask_copy(to->cpumask, from->cpumask);
+        /*
+         * Unlike hash and equality test, this function doesn't ignore
+         * ->no_numa as it is used for both pool and wq attrs.  Instead,
+         * get_unbound_pool() explicitly clears ->no_numa after copying.
+         */
+        to->no_numa = from->no_numa;
 }
 /* hash value of the content of @attr */
@@ -3578,6 +3605,12 @@ static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs)
        lockdep_set_subclass(&pool->lock, 1);   /* see put_pwq() */
        copy_workqueue_attrs(pool->attrs, attrs);
+        /*
+         * no_numa isn't a worker_pool attribute, always clear it.  See
+         * 'struct workqueue_attrs' comments for detail.
+         */
+        pool->attrs->no_numa = false;
        /* if cpumask is contained inside a NUMA node, we belong to that node */
        if (wq_numa_enabled) {
                for_each_node(node) {
@@ -4644,7 +4677,7 @@ static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu)
 * Workqueues should be brought up before normal priority CPU notifiers.
 * This will be registered high priority CPU notifier.
 */
-static int __cpuinit workqueue_cpu_up_callback(struct notifier_block *nfb,
+static int workqueue_cpu_up_callback(struct notifier_block *nfb,
                                               unsigned long action,
                                               void *hcpu)
 {
@@ -4697,7 +4730,7 @@ static int __cpuinit workqueue_cpu_up_callback(struct notifier_block *nfb,
 * Workqueues should be brought down after normal priority CPU notifiers.
 * This will be registered as low priority CPU notifier.
 */
-static int __cpuinit workqueue_cpu_down_callback(struct notifier_block *nfb,
+static int workqueue_cpu_down_callback(struct notifier_block *nfb,
                                                 unsigned long action,
                                                 void *hcpu)
 {
@@ -4756,7 +4789,14 @@ long work_on_cpu(int cpu, long (*fn)(void *), void *arg)
        INIT_WORK_ONSTACK(&wfc.work, work_for_cpu_fn);
        schedule_work_on(cpu, &wfc.work);
-        flush_work(&wfc.work);
+        /*
+         * The work item is on-stack and can't lead to deadlock through
+         * flushing.  Use __flush_work() to avoid spurious lockdep warnings
+         * when work_on_cpu()s are nested.
+         */
+        __flush_work(&wfc.work);
        return wfc.ret;
 }
 EXPORT_SYMBOL_GPL(work_on_cpu);