37 files changed, 3010 insertions, 2160 deletions
diff --git a/kernel/cgroup.c b/kernel/cgroup.c
index 781845a013ab..e0aeb32415ff 100644
--- a/kernel/cgroup.c
+++ b/kernel/cgroup.c
@@ -81,7 +81,7 @@
 */
 #ifdef CONFIG_PROVE_RCU
 DEFINE_MUTEX(cgroup_mutex);
-EXPORT_SYMBOL_GPL(cgroup_mutex);        /* only for task_subsys_state_check() */
+EXPORT_SYMBOL_GPL(cgroup_mutex);        /* only for lockdep */
 #else
 static DEFINE_MUTEX(cgroup_mutex);
 #endif
@@ -117,6 +117,7 @@ struct cfent {
        struct list_head                node;
        struct dentry                   *dentry;
        struct cftype                   *type;
+        struct cgroup_subsys_state      *css;
        /* file xattrs */
        struct simple_xattrs            xattrs;
@@ -159,9 +160,9 @@ struct css_id {
 */
 struct cgroup_event {
        /*
-         * Cgroup which the event belongs to.
+         * css which the event belongs to.
         */
-        struct cgroup *cgrp;
+        struct cgroup_subsys_state *css;
        /*
         * Control file which the event associated.
         */
@@ -215,10 +216,33 @@ static u64 cgroup_serial_nr_next = 1;
 */
 static int need_forkexit_callback __read_mostly;
-static void cgroup_offline_fn(struct work_struct *work);
+static struct cftype cgroup_base_files[];
+static void cgroup_destroy_css_killed(struct cgroup *cgrp);
 static int cgroup_destroy_locked(struct cgroup *cgrp);
-static int cgroup_addrm_files(struct cgroup *cgrp, struct cgroup_subsys *subsys,
+static int cgroup_addrm_files(struct cgroup *cgrp, struct cftype cfts[],
-                              struct cftype cfts[], bool is_add);
+                              bool is_add);
+/**
+ * cgroup_css - obtain a cgroup's css for the specified subsystem
+ * @cgrp: the cgroup of interest
+ * @ss: the subsystem of interest (%NULL returns the dummy_css)
+ *
+ * Return @cgrp's css (cgroup_subsys_state) associated with @ss.  This
+ * function must be called either under cgroup_mutex or rcu_read_lock() and
+ * the caller is responsible for pinning the returned css if it wants to
+ * keep accessing it outside the said locks.  This function may return
+ * %NULL if @cgrp doesn't have @subsys_id enabled.
+ */
+static struct cgroup_subsys_state *cgroup_css(struct cgroup *cgrp,
+                                              struct cgroup_subsys *ss)
+{
+        if (ss)
+                return rcu_dereference_check(cgrp->subsys[ss->subsys_id],
+                                             lockdep_is_held(&cgroup_mutex));
+        else
+                return &cgrp->dummy_css;
+}
 /* convenient tests for these bits */
 static inline bool cgroup_is_dead(const struct cgroup *cgrp)
@@ -365,9 +389,11 @@ static struct cgrp_cset_link init_cgrp_cset_link;
 static int cgroup_init_idr(struct cgroup_subsys *ss,
                           struct cgroup_subsys_state *css);
-/* css_set_lock protects the list of css_set objects, and the
+/*
- * chain of tasks off each css_set.  Nests outside task->alloc_lock
+ * css_set_lock protects the list of css_set objects, and the chain of
- * due to cgroup_iter_start() */
+ * tasks off each css_set.  Nests outside task->alloc_lock due to
+ * css_task_iter_start().
+ */
 static DEFINE_RWLOCK(css_set_lock);
 static int css_set_count;
@@ -392,10 +418,12 @@ static unsigned long css_set_hash(struct cgroup_subsys_state *css[])
        return key;
 }
-/* We don't maintain the lists running through each css_set to its
+/*
- * task until after the first call to cgroup_iter_start(). This
+ * We don't maintain the lists running through each css_set to its task
- * reduces the fork()/exit() overhead for people who have cgroups
+ * until after the first call to css_task_iter_start().  This reduces the
- * compiled into their kernel but not actually in use */
+ * fork()/exit() overhead for people who have cgroups compiled into their
+ * kernel but not actually in use.
+ */
 static int use_task_css_set_links __read_mostly;
 static void __put_css_set(struct css_set *cset, int taskexit)
@@ -464,7 +492,7 @@ static inline void put_css_set_taskexit(struct css_set *cset)
 * @new_cgrp: cgroup that's being entered by the task
 * @template: desired set of css pointers in css_set (pre-calculated)
 *
- * Returns true if "cg" matches "old_cg" except for the hierarchy
+ * Returns true if "cset" matches "old_cset" except for the hierarchy
 * which "new_cgrp" belongs to, for which it should match "new_cgrp".
 */
 static bool compare_css_sets(struct css_set *cset,
@@ -555,7 +583,7 @@ static struct css_set *find_existing_css_set(struct css_set *old_cset,
                        /* Subsystem is in this hierarchy. So we want
                         * the subsystem state from the new
                         * cgroup */
-                        template[i] = cgrp->subsys[i];
+                        template[i] = cgroup_css(cgrp, ss);
                } else {
                        /* Subsystem is not in this hierarchy, so we
                         * don't want to change the subsystem state */
@@ -803,8 +831,7 @@ static struct cgroup *task_cgroup_from_root(struct task_struct *task,
 static int cgroup_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode);
 static int cgroup_rmdir(struct inode *unused_dir, struct dentry *dentry);
-static int cgroup_populate_dir(struct cgroup *cgrp, bool base_files,
+static int cgroup_populate_dir(struct cgroup *cgrp, unsigned long subsys_mask);
-                               unsigned long subsys_mask);
 static const struct inode_operations cgroup_dir_inode_operations;
 static const struct file_operations proc_cgroupstats_operations;
@@ -813,8 +840,7 @@ static struct backing_dev_info cgroup_backing_dev_info = {
        .capabilities   = BDI_CAP_NO_ACCT_AND_WRITEBACK,
 };
-static int alloc_css_id(struct cgroup_subsys *ss,
+static int alloc_css_id(struct cgroup_subsys_state *child_css);
-                        struct cgroup *parent, struct cgroup *child);
 static struct inode *cgroup_new_inode(umode_t mode, struct super_block *sb)
 {
@@ -845,15 +871,8 @@ static struct cgroup_name *cgroup_alloc_name(struct dentry *dentry)
 static void cgroup_free_fn(struct work_struct *work)
 {
        struct cgroup *cgrp = container_of(work, struct cgroup, destroy_work);
-        struct cgroup_subsys *ss;
        mutex_lock(&cgroup_mutex);
-        /*
-         * Release the subsystem state objects.
-         */
-        for_each_root_subsys(cgrp->root, ss)
-                ss->css_free(cgrp);
        cgrp->root->number_of_cgroups--;
        mutex_unlock(&cgroup_mutex);
@@ -864,8 +883,6 @@ static void cgroup_free_fn(struct work_struct *work)
         */
        dput(cgrp->parent->dentry);
-        ida_simple_remove(&cgrp->root->cgroup_ida, cgrp->id);
        /*
         * Drop the active superblock reference that we took when we
         * created the cgroup. This will free cgrp->root, if we are
@@ -956,27 +973,22 @@ static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft)
 }
 /**
- * cgroup_clear_directory - selective removal of base and subsystem files
+ * cgroup_clear_dir - remove subsys files in a cgroup directory
- * @dir: directory containing the files
+ * @cgrp: target cgroup
- * @base_files: true if the base files should be removed
 * @subsys_mask: mask of the subsystem ids whose files should be removed
 */
-static void cgroup_clear_directory(struct dentry *dir, bool base_files,
+static void cgroup_clear_dir(struct cgroup *cgrp, unsigned long subsys_mask)
-                                   unsigned long subsys_mask)
 {
-        struct cgroup *cgrp = __d_cgrp(dir);
        struct cgroup_subsys *ss;
+        int i;
-        for_each_root_subsys(cgrp->root, ss) {
+        for_each_subsys(ss, i) {
                struct cftype_set *set;
-                if (!test_bit(ss->subsys_id, &subsys_mask))
+                if (!test_bit(i, &subsys_mask))
                        continue;
                list_for_each_entry(set, &ss->cftsets, node)
-                        cgroup_addrm_files(cgrp, NULL, set->cfts, false);
+                        cgroup_addrm_files(cgrp, set->cfts, false);
-        }
-        if (base_files) {
-                while (!list_empty(&cgrp->files))
-                        cgroup_rm_file(cgrp, NULL);
        }
 }
@@ -986,9 +998,6 @@ static void cgroup_clear_directory(struct dentry *dir, bool base_files,
 static void cgroup_d_remove_dir(struct dentry *dentry)
 {
        struct dentry *parent;
-        struct cgroupfs_root *root = dentry->d_sb->s_fs_info;
-        cgroup_clear_directory(dentry, true, root->subsys_mask);
        parent = dentry->d_parent;
        spin_lock(&parent->d_lock);
@@ -1009,79 +1018,84 @@ static int rebind_subsystems(struct cgroupfs_root *root,
 {
        struct cgroup *cgrp = &root->top_cgroup;
        struct cgroup_subsys *ss;
-        int i;
+        unsigned long pinned = 0;
+        int i, ret;
        BUG_ON(!mutex_is_locked(&cgroup_mutex));
        BUG_ON(!mutex_is_locked(&cgroup_root_mutex));
        /* Check that any added subsystems are currently free */
        for_each_subsys(ss, i) {
-                unsigned long bit = 1UL << i;
+                if (!(added_mask & (1 << i)))
-                if (!(bit & added_mask))
                        continue;
+                /* is the subsystem mounted elsewhere? */
                if (ss->root != &cgroup_dummy_root) {
-                        /* Subsystem isn't free */
+                        ret = -EBUSY;
-                        return -EBUSY;
+                        goto out_put;
+                }
+                /* pin the module */
+                if (!try_module_get(ss->module)) {
+                        ret = -ENOENT;
+                        goto out_put;
                }
+                pinned |= 1 << i;
        }
-        /* Currently we don't handle adding/removing subsystems when
+        /* subsys could be missing if unloaded between parsing and here */
-         * any child cgroups exist. This is theoretically supportable
+        if (added_mask != pinned) {
-         * but involves complex error handling, so it's being left until
+                ret = -ENOENT;
-         * later */
+                goto out_put;
-        if (root->number_of_cgroups > 1)
+        }
-                return -EBUSY;
+        ret = cgroup_populate_dir(cgrp, added_mask);
+        if (ret)
+                goto out_put;
+        /*
+         * Nothing can fail from this point on.  Remove files for the
+         * removed subsystems and rebind each subsystem.
+         */
+        cgroup_clear_dir(cgrp, removed_mask);
-        /* Process each subsystem */
        for_each_subsys(ss, i) {
                unsigned long bit = 1UL << i;
                if (bit & added_mask) {
                        /* We're binding this subsystem to this hierarchy */
-                        BUG_ON(cgrp->subsys[i]);
+                        BUG_ON(cgroup_css(cgrp, ss));
-                        BUG_ON(!cgroup_dummy_top->subsys[i]);
+                        BUG_ON(!cgroup_css(cgroup_dummy_top, ss));
-                        BUG_ON(cgroup_dummy_top->subsys[i]->cgroup != cgroup_dummy_top);
+                        BUG_ON(cgroup_css(cgroup_dummy_top, ss)->cgroup != cgroup_dummy_top);
+                        rcu_assign_pointer(cgrp->subsys[i],
+                                           cgroup_css(cgroup_dummy_top, ss));
+                        cgroup_css(cgrp, ss)->cgroup = cgrp;
-                        cgrp->subsys[i] = cgroup_dummy_top->subsys[i];
-                        cgrp->subsys[i]->cgroup = cgrp;
                        list_move(&ss->sibling, &root->subsys_list);
                        ss->root = root;
                        if (ss->bind)
-                                ss->bind(cgrp);
+                                ss->bind(cgroup_css(cgrp, ss));
                        /* refcount was already taken, and we're keeping it */
                        root->subsys_mask |= bit;
                } else if (bit & removed_mask) {
                        /* We're removing this subsystem */
-                        BUG_ON(cgrp->subsys[i] != cgroup_dummy_top->subsys[i]);
+                        BUG_ON(cgroup_css(cgrp, ss) != cgroup_css(cgroup_dummy_top, ss));
-                        BUG_ON(cgrp->subsys[i]->cgroup != cgrp);
+                        BUG_ON(cgroup_css(cgrp, ss)->cgroup != cgrp);
                        if (ss->bind)
-                                ss->bind(cgroup_dummy_top);
+                                ss->bind(cgroup_css(cgroup_dummy_top, ss));
-                        cgroup_dummy_top->subsys[i]->cgroup = cgroup_dummy_top;
-                        cgrp->subsys[i] = NULL;
+                        cgroup_css(cgroup_dummy_top, ss)->cgroup = cgroup_dummy_top;
+                        RCU_INIT_POINTER(cgrp->subsys[i], NULL);
                        cgroup_subsys[i]->root = &cgroup_dummy_root;
                        list_move(&ss->sibling, &cgroup_dummy_root.subsys_list);
                        /* subsystem is now free - drop reference on module */
                        module_put(ss->module);
                        root->subsys_mask &= ~bit;
-                } else if (bit & root->subsys_mask) {
-                        /* Subsystem state should already exist */
-                        BUG_ON(!cgrp->subsys[i]);
-                        /*
-                         * a refcount was taken, but we already had one, so
-                         * drop the extra reference.
-                         */
-                        module_put(ss->module);
-#ifdef CONFIG_MODULE_UNLOAD
-                        BUG_ON(ss->module && !module_refcount(ss->module));
-#endif
-                } else {
-                        /* Subsystem state shouldn't exist */
-                        BUG_ON(cgrp->subsys[i]);
                }
        }
@@ -1092,6 +1106,12 @@ static int rebind_subsystems(struct cgroupfs_root *root,
        root->flags |= CGRP_ROOT_SUBSYS_BOUND;
        return 0;
+out_put:
+        for_each_subsys(ss, i)
+                if (pinned & (1 << i))
+                        module_put(ss->module);
+        return ret;
 }
 static int cgroup_show_options(struct seq_file *seq, struct dentry *dentry)
@@ -1142,7 +1162,6 @@ static int parse_cgroupfs_options(char *data, struct cgroup_sb_opts *opts)
        char *token, *o = data;
        bool all_ss = false, one_ss = false;
        unsigned long mask = (unsigned long)-1;
-        bool module_pin_failed = false;
        struct cgroup_subsys *ss;
        int i;
@@ -1285,52 +1304,9 @@ static int parse_cgroupfs_options(char *data, struct cgroup_sb_opts *opts)
        if (!opts->subsys_mask && !opts->name)
                return -EINVAL;
-        /*
-         * Grab references on all the modules we'll need, so the subsystems
-         * don't dance around before rebind_subsystems attaches them. This may
-         * take duplicate reference counts on a subsystem that's already used,
-         * but rebind_subsystems handles this case.
-         */
-        for_each_subsys(ss, i) {
-                if (!(opts->subsys_mask & (1UL << i)))
-                        continue;
-                if (!try_module_get(cgroup_subsys[i]->module)) {
-                        module_pin_failed = true;
-                        break;
-                }
-        }
-        if (module_pin_failed) {
-                /*
-                 * oops, one of the modules was going away. this means that we
-                 * raced with a module_delete call, and to the user this is
-                 * essentially a "subsystem doesn't exist" case.
-                 */
-                for (i--; i >= 0; i--) {
-                        /* drop refcounts only on the ones we took */
-                        unsigned long bit = 1UL << i;
-                        if (!(bit & opts->subsys_mask))
-                                continue;
-                        module_put(cgroup_subsys[i]->module);
-                }
-                return -ENOENT;
-        }
        return 0;
 }
-static void drop_parsed_module_refcounts(unsigned long subsys_mask)
-{
-        struct cgroup_subsys *ss;
-        int i;
-        mutex_lock(&cgroup_mutex);
-        for_each_subsys(ss, i)
-                if (subsys_mask & (1UL << i))
-                        module_put(cgroup_subsys[i]->module);
-        mutex_unlock(&cgroup_mutex);
-}
 static int cgroup_remount(struct super_block *sb, int *flags, char *data)
 {
        int ret = 0;
@@ -1370,22 +1346,15 @@ static int cgroup_remount(struct super_block *sb, int *flags, char *data)
                goto out_unlock;
        }
-        /*
+        /* remounting is not allowed for populated hierarchies */
-         * Clear out the files of subsystems that should be removed, do
+        if (root->number_of_cgroups > 1) {
-         * this before rebind_subsystems, since rebind_subsystems may
+                ret = -EBUSY;
-         * change this hierarchy's subsys_list.
-         */
-        cgroup_clear_directory(cgrp->dentry, false, removed_mask);
-        ret = rebind_subsystems(root, added_mask, removed_mask);
-        if (ret) {
-                /* rebind_subsystems failed, re-populate the removed files */
-                cgroup_populate_dir(cgrp, false, removed_mask);
                goto out_unlock;
        }
-        /* re-populate subsystem files */
+        ret = rebind_subsystems(root, added_mask, removed_mask);
-        cgroup_populate_dir(cgrp, false, added_mask);
+        if (ret)
+                goto out_unlock;
        if (opts.release_agent)
                strcpy(root->release_agent_path, opts.release_agent);
@@ -1395,8 +1364,6 @@ static int cgroup_remount(struct super_block *sb, int *flags, char *data)
        mutex_unlock(&cgroup_root_mutex);
        mutex_unlock(&cgroup_mutex);
        mutex_unlock(&cgrp->dentry->d_inode->i_mutex);
-        if (ret)
-                drop_parsed_module_refcounts(opts.subsys_mask);
        return ret;
 }
@@ -1416,6 +1383,7 @@ static void init_cgroup_housekeeping(struct cgroup *cgrp)
        INIT_LIST_HEAD(&cgrp->release_list);
        INIT_LIST_HEAD(&cgrp->pidlists);
        mutex_init(&cgrp->pidlist_mutex);
+        cgrp->dummy_css.cgroup = cgrp;
        INIT_LIST_HEAD(&cgrp->event_list);
        spin_lock_init(&cgrp->event_list_lock);
        simple_xattrs_init(&cgrp->xattrs);
@@ -1431,6 +1399,7 @@ static void init_cgroup_root(struct cgroupfs_root *root)
        cgrp->root = root;
        RCU_INIT_POINTER(cgrp->name, &root_cgroup_name);
        init_cgroup_housekeeping(cgrp);
+        idr_init(&root->cgroup_idr);
 }
 static int cgroup_init_root_id(struct cgroupfs_root *root, int start, int end)
@@ -1503,7 +1472,6 @@ static struct cgroupfs_root *cgroup_root_from_opts(struct cgroup_sb_opts *opts)
         */
        root->subsys_mask = opts->subsys_mask;
        root->flags = opts->flags;
-        ida_init(&root->cgroup_ida);
        if (opts->release_agent)
                strcpy(root->release_agent_path, opts->release_agent);
        if (opts->name)
@@ -1519,7 +1487,7 @@ static void cgroup_free_root(struct cgroupfs_root *root)
                /* hierarhcy ID shoulid already have been released */
                WARN_ON_ONCE(root->hierarchy_id);
-                ida_destroy(&root->cgroup_ida);
+                idr_destroy(&root->cgroup_idr);
                kfree(root);
        }
 }
@@ -1584,7 +1552,9 @@ static struct dentry *cgroup_mount(struct file_system_type *fs_type,
        int ret = 0;
        struct super_block *sb;
        struct cgroupfs_root *new_root;
+        struct list_head tmp_links;
        struct inode *inode;
+        const struct cred *cred;
        /* First find the desired set of subsystems */
        mutex_lock(&cgroup_mutex);
@@ -1600,7 +1570,7 @@ static struct dentry *cgroup_mount(struct file_system_type *fs_type,
        new_root = cgroup_root_from_opts(&opts);
        if (IS_ERR(new_root)) {
                ret = PTR_ERR(new_root);
-                goto drop_modules;
+                goto out_err;
        }
        opts.new_root = new_root;
@@ -1609,17 +1579,15 @@ static struct dentry *cgroup_mount(struct file_system_type *fs_type,
        if (IS_ERR(sb)) {
                ret = PTR_ERR(sb);
                cgroup_free_root(opts.new_root);
-                goto drop_modules;
+                goto out_err;
        }
        root = sb->s_fs_info;
        BUG_ON(!root);
        if (root == opts.new_root) {
                /* We used the new root structure, so this is a new hierarchy */
-                struct list_head tmp_links;
                struct cgroup *root_cgrp = &root->top_cgroup;
                struct cgroupfs_root *existing_root;
-                const struct cred *cred;
                int i;
                struct css_set *cset;
@@ -1634,6 +1602,11 @@ static struct dentry *cgroup_mount(struct file_system_type *fs_type,
                mutex_lock(&cgroup_mutex);
                mutex_lock(&cgroup_root_mutex);
+                root_cgrp->id = idr_alloc(&root->cgroup_idr, root_cgrp,
+                                           0, 1, GFP_KERNEL);
+                if (root_cgrp->id < 0)
+                        goto unlock_drop;
                /* Check for name clashes with existing mounts */
                ret = -EBUSY;
                if (strlen(root->name))
@@ -1657,26 +1630,37 @@ static struct dentry *cgroup_mount(struct file_system_type *fs_type,
                if (ret)
                        goto unlock_drop;
+                sb->s_root->d_fsdata = root_cgrp;
+                root_cgrp->dentry = sb->s_root;
+                /*
+                 * We're inside get_sb() and will call lookup_one_len() to
+                 * create the root files, which doesn't work if SELinux is
+                 * in use.  The following cred dancing somehow works around
+                 * it.  See 2ce9738ba ("cgroupfs: use init_cred when
+                 * populating new cgroupfs mount") for more details.
+                 */
+                cred = override_creds(&init_cred);
+                ret = cgroup_addrm_files(root_cgrp, cgroup_base_files, true);
+                if (ret)
+                        goto rm_base_files;
                ret = rebind_subsystems(root, root->subsys_mask, 0);
-                if (ret == -EBUSY) {
+                if (ret)
-                        free_cgrp_cset_links(&tmp_links);
+                        goto rm_base_files;
-                        goto unlock_drop;
-                }
+                revert_creds(cred);
                /*
                 * There must be no failure case after here, since rebinding
                 * takes care of subsystems' refcounts, which are explicitly
                 * dropped in the failure exit path.
                 */
-                /* EBUSY should be the only error here */
-                BUG_ON(ret);
                list_add(&root->root_list, &cgroup_roots);
                cgroup_root_count++;
-                sb->s_root->d_fsdata = root_cgrp;
-                root->top_cgroup.dentry = sb->s_root;
                /* Link the top cgroup in this hierarchy into all
                 * the css_set objects */
                write_lock(&css_set_lock);
@@ -1689,9 +1673,6 @@ static struct dentry *cgroup_mount(struct file_system_type *fs_type,
                BUG_ON(!list_empty(&root_cgrp->children));
                BUG_ON(root->number_of_cgroups != 1);
-                cred = override_creds(&init_cred);
-                cgroup_populate_dir(root_cgrp, true, root->subsys_mask);
-                revert_creds(cred);
                mutex_unlock(&cgroup_root_mutex);
                mutex_unlock(&cgroup_mutex);
                mutex_unlock(&inode->i_mutex);
@@ -1711,15 +1692,16 @@ static struct dentry *cgroup_mount(struct file_system_type *fs_type,
                                pr_warning("cgroup: new mount options do not match the existing superblock, will be ignored\n");
                        }
                }
-                /* no subsys rebinding, so refcounts don't change */
-                drop_parsed_module_refcounts(opts.subsys_mask);
        }
        kfree(opts.release_agent);
        kfree(opts.name);
        return dget(sb->s_root);
+ rm_base_files:
+        free_cgrp_cset_links(&tmp_links);
+        cgroup_addrm_files(&root->top_cgroup, cgroup_base_files, false);
+        revert_creds(cred);
 unlock_drop:
        cgroup_exit_root_id(root);
        mutex_unlock(&cgroup_root_mutex);
@@ -1727,8 +1709,6 @@ static struct dentry *cgroup_mount(struct file_system_type *fs_type,
        mutex_unlock(&inode->i_mutex);
 drop_new_super:
        deactivate_locked_super(sb);
- drop_modules:
-        drop_parsed_module_refcounts(opts.subsys_mask);
 out_err:
        kfree(opts.release_agent);
        kfree(opts.name);
@@ -1746,6 +1726,7 @@ static void cgroup_kill_sb(struct super_block *sb) {
        BUG_ON(root->number_of_cgroups != 1);
        BUG_ON(!list_empty(&cgrp->children));
+        mutex_lock(&cgrp->dentry->d_inode->i_mutex);
        mutex_lock(&cgroup_mutex);
        mutex_lock(&cgroup_root_mutex);
@@ -1778,6 +1759,7 @@ static void cgroup_kill_sb(struct super_block *sb) {
        mutex_unlock(&cgroup_root_mutex);
        mutex_unlock(&cgroup_mutex);
+        mutex_unlock(&cgrp->dentry->d_inode->i_mutex);
        simple_xattrs_free(&cgrp->xattrs);
@@ -1889,7 +1871,7 @@ EXPORT_SYMBOL_GPL(task_cgroup_path);
 struct task_and_cgroup {
        struct task_struct      *task;
        struct cgroup           *cgrp;
-        struct css_set          *cg;
+        struct css_set          *cset;
 };
 struct cgroup_taskset {
@@ -1939,18 +1921,20 @@ struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset)
 EXPORT_SYMBOL_GPL(cgroup_taskset_next);
 /**
- * cgroup_taskset_cur_cgroup - return the matching cgroup for the current task
+ * cgroup_taskset_cur_css - return the matching css for the current task
 * @tset: taskset of interest
+ * @subsys_id: the ID of the target subsystem
 *
- * Return the cgroup for the current (last returned) task of @tset.  This
+ * Return the css for the current (last returned) task of @tset for
- * function must be preceded by either cgroup_taskset_first() or
+ * subsystem specified by @subsys_id.  This function must be preceded by
- * cgroup_taskset_next().
+ * either cgroup_taskset_first() or cgroup_taskset_next().
 */
-struct cgroup *cgroup_taskset_cur_cgroup(struct cgroup_taskset *tset)
+struct cgroup_subsys_state *cgroup_taskset_cur_css(struct cgroup_taskset *tset,
+                                                   int subsys_id)
 {
-        return tset->cur_cgrp;
+        return cgroup_css(tset->cur_cgrp, cgroup_subsys[subsys_id]);
 }
-EXPORT_SYMBOL_GPL(cgroup_taskset_cur_cgroup);
+EXPORT_SYMBOL_GPL(cgroup_taskset_cur_css);
 /**
 * cgroup_taskset_size - return the number of tasks in taskset
@@ -2089,8 +2073,10 @@ static int cgroup_attach_task(struct cgroup *cgrp, struct task_struct *tsk,
         * step 1: check that we can legitimately attach to the cgroup.
         */
        for_each_root_subsys(root, ss) {
+                struct cgroup_subsys_state *css = cgroup_css(cgrp, ss);
                if (ss->can_attach) {
-                        retval = ss->can_attach(cgrp, &tset);
+                        retval = ss->can_attach(css, &tset);
                        if (retval) {
                                failed_ss = ss;
                                goto out_cancel_attach;
@@ -2107,8 +2093,8 @@ static int cgroup_attach_task(struct cgroup *cgrp, struct task_struct *tsk,
                tc = flex_array_get(group, i);
                old_cset = task_css_set(tc->task);
-                tc->cg = find_css_set(old_cset, cgrp);
+                tc->cset = find_css_set(old_cset, cgrp);
-                if (!tc->cg) {
+                if (!tc->cset) {
                        retval = -ENOMEM;
                        goto out_put_css_set_refs;
                }
@@ -2121,7 +2107,7 @@ static int cgroup_attach_task(struct cgroup *cgrp, struct task_struct *tsk,
         */
        for (i = 0; i < group_size; i++) {
                tc = flex_array_get(group, i);
-                cgroup_task_migrate(tc->cgrp, tc->task, tc->cg);
+                cgroup_task_migrate(tc->cgrp, tc->task, tc->cset);
        }
        /* nothing is sensitive to fork() after this point. */
@@ -2129,8 +2115,10 @@ static int cgroup_attach_task(struct cgroup *cgrp, struct task_struct *tsk,
         * step 4: do subsystem attach callbacks.
         */
        for_each_root_subsys(root, ss) {
+                struct cgroup_subsys_state *css = cgroup_css(cgrp, ss);
                if (ss->attach)
-                        ss->attach(cgrp, &tset);
+                        ss->attach(css, &tset);
        }
        /*
@@ -2141,18 +2129,20 @@ out_put_css_set_refs:
        if (retval) {
                for (i = 0; i < group_size; i++) {
                        tc = flex_array_get(group, i);
-                        if (!tc->cg)
+                        if (!tc->cset)
                                break;
-                        put_css_set(tc->cg);
+                        put_css_set(tc->cset);
                }
        }
 out_cancel_attach:
        if (retval) {
                for_each_root_subsys(root, ss) {
+                        struct cgroup_subsys_state *css = cgroup_css(cgrp, ss);
                        if (ss == failed_ss)
                                break;
                        if (ss->cancel_attach)
-                                ss->cancel_attach(cgrp, &tset);
+                                ss->cancel_attach(css, &tset);
                }
        }
 out_free_group_list:
@@ -2253,9 +2243,9 @@ int cgroup_attach_task_all(struct task_struct *from, struct task_struct *tsk)
        mutex_lock(&cgroup_mutex);
        for_each_active_root(root) {
-                struct cgroup *from_cg = task_cgroup_from_root(from, root);
+                struct cgroup *from_cgrp = task_cgroup_from_root(from, root);
-                retval = cgroup_attach_task(from_cg, tsk, false);
+                retval = cgroup_attach_task(from_cgrp, tsk, false);
                if (retval)
                        break;
        }
@@ -2265,34 +2255,38 @@ int cgroup_attach_task_all(struct task_struct *from, struct task_struct *tsk)
 }
 EXPORT_SYMBOL_GPL(cgroup_attach_task_all);
-static int cgroup_tasks_write(struct cgroup *cgrp, struct cftype *cft, u64 pid)
+static int cgroup_tasks_write(struct cgroup_subsys_state *css,
+                              struct cftype *cft, u64 pid)
 {
-        return attach_task_by_pid(cgrp, pid, false);
+        return attach_task_by_pid(css->cgroup, pid, false);
 }
-static int cgroup_procs_write(struct cgroup *cgrp, struct cftype *cft, u64 tgid)
+static int cgroup_procs_write(struct cgroup_subsys_state *css,
+                              struct cftype *cft, u64 tgid)
 {
-        return attach_task_by_pid(cgrp, tgid, true);
+        return attach_task_by_pid(css->cgroup, tgid, true);
 }
-static int cgroup_release_agent_write(struct cgroup *cgrp, struct cftype *cft,
+static int cgroup_release_agent_write(struct cgroup_subsys_state *css,
-                                      const char *buffer)
+                                      struct cftype *cft, const char *buffer)
 {
-        BUILD_BUG_ON(sizeof(cgrp->root->release_agent_path) < PATH_MAX);
+        BUILD_BUG_ON(sizeof(css->cgroup->root->release_agent_path) < PATH_MAX);
        if (strlen(buffer) >= PATH_MAX)
                return -EINVAL;
-        if (!cgroup_lock_live_group(cgrp))
+        if (!cgroup_lock_live_group(css->cgroup))
                return -ENODEV;
        mutex_lock(&cgroup_root_mutex);
-        strcpy(cgrp->root->release_agent_path, buffer);
+        strcpy(css->cgroup->root->release_agent_path, buffer);
        mutex_unlock(&cgroup_root_mutex);
        mutex_unlock(&cgroup_mutex);
        return 0;
 }
-static int cgroup_release_agent_show(struct cgroup *cgrp, struct cftype *cft,
+static int cgroup_release_agent_show(struct cgroup_subsys_state *css,
-                                     struct seq_file *seq)
+                                     struct cftype *cft, struct seq_file *seq)
 {
+        struct cgroup *cgrp = css->cgroup;
        if (!cgroup_lock_live_group(cgrp))
                return -ENODEV;
        seq_puts(seq, cgrp->root->release_agent_path);
@@ -2301,20 +2295,20 @@ static int cgroup_release_agent_show(struct cgroup *cgrp, struct cftype *cft,
        return 0;
 }
-static int cgroup_sane_behavior_show(struct cgroup *cgrp, struct cftype *cft,
+static int cgroup_sane_behavior_show(struct cgroup_subsys_state *css,
-                                     struct seq_file *seq)
+                                     struct cftype *cft, struct seq_file *seq)
 {
-        seq_printf(seq, "%d\n", cgroup_sane_behavior(cgrp));
+        seq_printf(seq, "%d\n", cgroup_sane_behavior(css->cgroup));
        return 0;
 }
 /* A buffer size big enough for numbers or short strings */
 #define CGROUP_LOCAL_BUFFER_SIZE 64
-static ssize_t cgroup_write_X64(struct cgroup *cgrp, struct cftype *cft,
+static ssize_t cgroup_write_X64(struct cgroup_subsys_state *css,
-                                struct file *file,
+                                struct cftype *cft, struct file *file,
-                                const char __user *userbuf,
+                                const char __user *userbuf, size_t nbytes,
-                                size_t nbytes, loff_t *unused_ppos)
+                                loff_t *unused_ppos)
 {
        char buffer[CGROUP_LOCAL_BUFFER_SIZE];
        int retval = 0;
@@ -2332,22 +2326,22 @@ static ssize_t cgroup_write_X64(struct cgroup *cgrp, struct cftype *cft,
                u64 val = simple_strtoull(strstrip(buffer), &end, 0);
                if (*end)
                        return -EINVAL;
-                retval = cft->write_u64(cgrp, cft, val);
+                retval = cft->write_u64(css, cft, val);
        } else {
                s64 val = simple_strtoll(strstrip(buffer), &end, 0);
                if (*end)
                        return -EINVAL;
-                retval = cft->write_s64(cgrp, cft, val);
+                retval = cft->write_s64(css, cft, val);
        }
        if (!retval)
                retval = nbytes;
        return retval;
 }
-static ssize_t cgroup_write_string(struct cgroup *cgrp, struct cftype *cft,
+static ssize_t cgroup_write_string(struct cgroup_subsys_state *css,
-                                   struct file *file,
+                                   struct cftype *cft, struct file *file,
-                                   const char __user *userbuf,
+                                   const char __user *userbuf, size_t nbytes,
-                                   size_t nbytes, loff_t *unused_ppos)
+                                   loff_t *unused_ppos)
 {
        char local_buffer[CGROUP_LOCAL_BUFFER_SIZE];
        int retval = 0;
@@ -2370,7 +2364,7 @@ static ssize_t cgroup_write_string(struct cgroup *cgrp, struct cftype *cft,
        }
        buffer[nbytes] = 0;     /* nul-terminate */
-        retval = cft->write_string(cgrp, cft, strstrip(buffer));
+        retval = cft->write_string(css, cft, strstrip(buffer));
        if (!retval)
                retval = nbytes;
 out:
@@ -2380,65 +2374,60 @@ out:
 }
 static ssize_t cgroup_file_write(struct file *file, const char __user *buf,
-                                                size_t nbytes, loff_t *ppos)
+                                 size_t nbytes, loff_t *ppos)
 {
+        struct cfent *cfe = __d_cfe(file->f_dentry);
        struct cftype *cft = __d_cft(file->f_dentry);
-        struct cgroup *cgrp = __d_cgrp(file->f_dentry->d_parent);
+        struct cgroup_subsys_state *css = cfe->css;
-        if (cgroup_is_dead(cgrp))
-                return -ENODEV;
        if (cft->write)
-                return cft->write(cgrp, cft, file, buf, nbytes, ppos);
+                return cft->write(css, cft, file, buf, nbytes, ppos);
        if (cft->write_u64 || cft->write_s64)
-                return cgroup_write_X64(cgrp, cft, file, buf, nbytes, ppos);
+                return cgroup_write_X64(css, cft, file, buf, nbytes, ppos);
        if (cft->write_string)
-                return cgroup_write_string(cgrp, cft, file, buf, nbytes, ppos);
+                return cgroup_write_string(css, cft, file, buf, nbytes, ppos);
        if (cft->trigger) {
-                int ret = cft->trigger(cgrp, (unsigned int)cft->private);
+                int ret = cft->trigger(css, (unsigned int)cft->private);
                return ret ? ret : nbytes;
        }
        return -EINVAL;
 }
-static ssize_t cgroup_read_u64(struct cgroup *cgrp, struct cftype *cft,
+static ssize_t cgroup_read_u64(struct cgroup_subsys_state *css,
-                               struct file *file,
+                               struct cftype *cft, struct file *file,
-                               char __user *buf, size_t nbytes,
+                               char __user *buf, size_t nbytes, loff_t *ppos)
-                               loff_t *ppos)
 {
        char tmp[CGROUP_LOCAL_BUFFER_SIZE];
-        u64 val = cft->read_u64(cgrp, cft);
+        u64 val = cft->read_u64(css, cft);
        int len = sprintf(tmp, "%llu\n", (unsigned long long) val);
        return simple_read_from_buffer(buf, nbytes, ppos, tmp, len);
 }
-static ssize_t cgroup_read_s64(struct cgroup *cgrp, struct cftype *cft,
+static ssize_t cgroup_read_s64(struct cgroup_subsys_state *css,
-                               struct file *file,
+                               struct cftype *cft, struct file *file,
-                               char __user *buf, size_t nbytes,
+                               char __user *buf, size_t nbytes, loff_t *ppos)
-                               loff_t *ppos)
 {
        char tmp[CGROUP_LOCAL_BUFFER_SIZE];
-        s64 val = cft->read_s64(cgrp, cft);
+        s64 val = cft->read_s64(css, cft);
        int len = sprintf(tmp, "%lld\n", (long long) val);
        return simple_read_from_buffer(buf, nbytes, ppos, tmp, len);
 }
 static ssize_t cgroup_file_read(struct file *file, char __user *buf,
-                                   size_t nbytes, loff_t *ppos)
+                                size_t nbytes, loff_t *ppos)
 {
+        struct cfent *cfe = __d_cfe(file->f_dentry);
        struct cftype *cft = __d_cft(file->f_dentry);
-        struct cgroup *cgrp = __d_cgrp(file->f_dentry->d_parent);
+        struct cgroup_subsys_state *css = cfe->css;
-        if (cgroup_is_dead(cgrp))
-                return -ENODEV;
        if (cft->read)
-                return cft->read(cgrp, cft, file, buf, nbytes, ppos);
+                return cft->read(css, cft, file, buf, nbytes, ppos);
        if (cft->read_u64)
-                return cgroup_read_u64(cgrp, cft, file, buf, nbytes, ppos);
+                return cgroup_read_u64(css, cft, file, buf, nbytes, ppos);
        if (cft->read_s64)
-                return cgroup_read_s64(cgrp, cft, file, buf, nbytes, ppos);
+                return cgroup_read_s64(css, cft, file, buf, nbytes, ppos);
        return -EINVAL;
 }
@@ -2447,11 +2436,6 @@ static ssize_t cgroup_file_read(struct file *file, char __user *buf,
 * supports string->u64 maps, but can be extended in future.
 */
-struct cgroup_seqfile_state {
-        struct cftype *cft;
-        struct cgroup *cgroup;
-};
 static int cgroup_map_add(struct cgroup_map_cb *cb, const char *key, u64 value)
 {
        struct seq_file *sf = cb->state;
@@ -2460,69 +2444,86 @@ static int cgroup_map_add(struct cgroup_map_cb *cb, const char *key, u64 value)
 static int cgroup_seqfile_show(struct seq_file *m, void *arg)
 {
-        struct cgroup_seqfile_state *state = m->private;
+        struct cfent *cfe = m->private;
-        struct cftype *cft = state->cft;
+        struct cftype *cft = cfe->type;
+        struct cgroup_subsys_state *css = cfe->css;
        if (cft->read_map) {
                struct cgroup_map_cb cb = {
                        .fill = cgroup_map_add,
                        .state = m,
                };
-                return cft->read_map(state->cgroup, cft, &cb);
+                return cft->read_map(css, cft, &cb);
        }
-        return cft->read_seq_string(state->cgroup, cft, m);
+        return cft->read_seq_string(css, cft, m);
-}
-static int cgroup_seqfile_release(struct inode *inode, struct file *file)
-{
-        struct seq_file *seq = file->private_data;
-        kfree(seq->private);
-        return single_release(inode, file);
 }
 static const struct file_operations cgroup_seqfile_operations = {
        .read = seq_read,
        .write = cgroup_file_write,
        .llseek = seq_lseek,
-        .release = cgroup_seqfile_release,
+        .release = single_release,
 };
 static int cgroup_file_open(struct inode *inode, struct file *file)
 {
+        struct cfent *cfe = __d_cfe(file->f_dentry);
+        struct cftype *cft = __d_cft(file->f_dentry);
+        struct cgroup *cgrp = __d_cgrp(cfe->dentry->d_parent);
+        struct cgroup_subsys_state *css;
        int err;
-        struct cftype *cft;
        err = generic_file_open(inode, file);
        if (err)
                return err;
-        cft = __d_cft(file->f_dentry);
-        if (cft->read_map || cft->read_seq_string) {
+        /*
-                struct cgroup_seqfile_state *state;
+         * If the file belongs to a subsystem, pin the css.  Will be
+         * unpinned either on open failure or release.  This ensures that
+         * @css stays alive for all file operations.
+         */
+        rcu_read_lock();
+        css = cgroup_css(cgrp, cft->ss);
+        if (cft->ss && !css_tryget(css))
+                css = NULL;
+        rcu_read_unlock();
-                state = kzalloc(sizeof(*state), GFP_USER);
+        if (!css)
-                if (!state)
+                return -ENODEV;
-                        return -ENOMEM;
-                state->cft = cft;
+        /*
-                state->cgroup = __d_cgrp(file->f_dentry->d_parent);
+         * @cfe->css is used by read/write/close to determine the
+         * associated css.  @file->private_data would be a better place but
+         * that's already used by seqfile.  Multiple accessors may use it
+         * simultaneously which is okay as the association never changes.
+         */
+        WARN_ON_ONCE(cfe->css && cfe->css != css);
+        cfe->css = css;
+        if (cft->read_map || cft->read_seq_string) {
                file->f_op = &cgroup_seqfile_operations;
-                err = single_open(file, cgroup_seqfile_show, state);
+                err = single_open(file, cgroup_seqfile_show, cfe);
-                if (err < 0)
+        } else if (cft->open) {
-                        kfree(state);
-        } else if (cft->open)
                err = cft->open(inode, file);
-        else
+        }
-                err = 0;
+        if (css->ss && err)
+                css_put(css);
        return err;
 }
 static int cgroup_file_release(struct inode *inode, struct file *file)
 {
+        struct cfent *cfe = __d_cfe(file->f_dentry);
        struct cftype *cft = __d_cft(file->f_dentry);
+        struct cgroup_subsys_state *css = cfe->css;
+        int ret = 0;
        if (cft->release)
-                return cft->release(inode, file);
+                ret = cft->release(inode, file);
-        return 0;
+        if (css->ss)
+                css_put(css);
+        return ret;
 }
 /*
@@ -2736,8 +2737,7 @@ static umode_t cgroup_file_mode(const struct cftype *cft)
        return mode;
 }
-static int cgroup_add_file(struct cgroup *cgrp, struct cgroup_subsys *subsys,
+static int cgroup_add_file(struct cgroup *cgrp, struct cftype *cft)
-                           struct cftype *cft)
 {
        struct dentry *dir = cgrp->dentry;
        struct cgroup *parent = __d_cgrp(dir);
@@ -2747,8 +2747,9 @@ static int cgroup_add_file(struct cgroup *cgrp, struct cgroup_subsys *subsys,
        umode_t mode;
        char name[MAX_CGROUP_TYPE_NAMELEN + MAX_CFTYPE_NAME + 2] = { 0 };
-        if (subsys && !(cgrp->root->flags & CGRP_ROOT_NOPREFIX)) {
+        if (cft->ss && !(cft->flags & CFTYPE_NO_PREFIX) &&
-                strcpy(name, subsys->name);
+            !(cgrp->root->flags & CGRP_ROOT_NOPREFIX)) {
+                strcpy(name, cft->ss->name);
                strcat(name, ".");
        }
        strcat(name, cft->name);
@@ -2782,11 +2783,25 @@ out:
        return error;
 }
-static int cgroup_addrm_files(struct cgroup *cgrp, struct cgroup_subsys *subsys,
+/**
-                              struct cftype cfts[], bool is_add)
+ * cgroup_addrm_files - add or remove files to a cgroup directory
+ * @cgrp: the target cgroup
+ * @cfts: array of cftypes to be added
+ * @is_add: whether to add or remove
+ *
+ * Depending on @is_add, add or remove files defined by @cfts on @cgrp.
+ * For removals, this function never fails.  If addition fails, this
+ * function doesn't remove files already added.  The caller is responsible
+ * for cleaning up.
+ */
+static int cgroup_addrm_files(struct cgroup *cgrp, struct cftype cfts[],
+                              bool is_add)
 {
        struct cftype *cft;
-        int err, ret = 0;
+        int ret;
+        lockdep_assert_held(&cgrp->dentry->d_inode->i_mutex);
+        lockdep_assert_held(&cgroup_mutex);
        for (cft = cfts; cft->name[0] != '\0'; cft++) {
                /* does cft->flags tell us to skip this file on @cgrp? */
@@ -2798,16 +2813,17 @@ static int cgroup_addrm_files(struct cgroup *cgrp, struct cgroup_subsys *subsys,
                        continue;
                if (is_add) {
-                        err = cgroup_add_file(cgrp, subsys, cft);
+                        ret = cgroup_add_file(cgrp, cft);
-                        if (err)
+                        if (ret) {
                                pr_warn("cgroup_addrm_files: failed to add %s, err=%d\n",
-                                        cft->name, err);
+                                        cft->name, ret);
-                        ret = err;
+                                return ret;
+                        }
                } else {
                        cgroup_rm_file(cgrp, cft);
                }
        }
-        return ret;
+        return 0;
 }
 static void cgroup_cfts_prepare(void)
@@ -2816,28 +2832,30 @@ static void cgroup_cfts_prepare(void)
        /*
         * Thanks to the entanglement with vfs inode locking, we can't walk
         * the existing cgroups under cgroup_mutex and create files.
-         * Instead, we use cgroup_for_each_descendant_pre() and drop RCU
+         * Instead, we use css_for_each_descendant_pre() and drop RCU read
-         * read lock before calling cgroup_addrm_files().
+         * lock before calling cgroup_addrm_files().
         */
        mutex_lock(&cgroup_mutex);
 }
-static void cgroup_cfts_commit(struct cgroup_subsys *ss,
+static int cgroup_cfts_commit(struct cftype *cfts, bool is_add)
-                               struct cftype *cfts, bool is_add)
        __releases(&cgroup_mutex)
 {
        LIST_HEAD(pending);
-        struct cgroup *cgrp, *root = &ss->root->top_cgroup;
+        struct cgroup_subsys *ss = cfts[0].ss;
+        struct cgroup *root = &ss->root->top_cgroup;
        struct super_block *sb = ss->root->sb;
        struct dentry *prev = NULL;
        struct inode *inode;
+        struct cgroup_subsys_state *css;
        u64 update_before;
+        int ret = 0;
        /* %NULL @cfts indicates abort and don't bother if @ss isn't attached */
        if (!cfts || ss->root == &cgroup_dummy_root ||
            !atomic_inc_not_zero(&sb->s_active)) {
                mutex_unlock(&cgroup_mutex);
-                return;
+                return 0;
        }
        /*
@@ -2849,17 +2867,11 @@ static void cgroup_cfts_commit(struct cgroup_subsys *ss,
        mutex_unlock(&cgroup_mutex);
-        /* @root always needs to be updated */
-        inode = root->dentry->d_inode;
-        mutex_lock(&inode->i_mutex);
-        mutex_lock(&cgroup_mutex);
-        cgroup_addrm_files(root, ss, cfts, is_add);
-        mutex_unlock(&cgroup_mutex);
-        mutex_unlock(&inode->i_mutex);
        /* add/rm files for all cgroups created before */
        rcu_read_lock();
-        cgroup_for_each_descendant_pre(cgrp, root) {
+        css_for_each_descendant_pre(css, cgroup_css(root, ss)) {
+                struct cgroup *cgrp = css->cgroup;
                if (cgroup_is_dead(cgrp))
                        continue;
@@ -2873,15 +2885,18 @@ static void cgroup_cfts_commit(struct cgroup_subsys *ss,
                mutex_lock(&inode->i_mutex);
                mutex_lock(&cgroup_mutex);
                if (cgrp->serial_nr < update_before && !cgroup_is_dead(cgrp))
-                        cgroup_addrm_files(cgrp, ss, cfts, is_add);
+                        ret = cgroup_addrm_files(cgrp, cfts, is_add);
                mutex_unlock(&cgroup_mutex);
                mutex_unlock(&inode->i_mutex);
                rcu_read_lock();
+                if (ret)
+                        break;
        }
        rcu_read_unlock();
        dput(prev);
        deactivate_super(sb);
+        return ret;
 }
 /**
@@ -2901,49 +2916,56 @@ static void cgroup_cfts_commit(struct cgroup_subsys *ss,
 int cgroup_add_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
 {
        struct cftype_set *set;
+        struct cftype *cft;
+        int ret;
        set = kzalloc(sizeof(*set), GFP_KERNEL);
        if (!set)
                return -ENOMEM;
+        for (cft = cfts; cft->name[0] != '\0'; cft++)
+                cft->ss = ss;
        cgroup_cfts_prepare();
        set->cfts = cfts;
        list_add_tail(&set->node, &ss->cftsets);
-        cgroup_cfts_commit(ss, cfts, true);
+        ret = cgroup_cfts_commit(cfts, true);
+        if (ret)
-        return 0;
+                cgroup_rm_cftypes(cfts);
+        return ret;
 }
 EXPORT_SYMBOL_GPL(cgroup_add_cftypes);
 /**
 * cgroup_rm_cftypes - remove an array of cftypes from a subsystem
- * @ss: target cgroup subsystem
 * @cfts: zero-length name terminated array of cftypes
 *
- * Unregister @cfts from @ss.  Files described by @cfts are removed from
+ * Unregister @cfts.  Files described by @cfts are removed from all
- * all existing cgroups to which @ss is attached and all future cgroups
+ * existing cgroups and all future cgroups won't have them either.  This
- * won't have them either.  This function can be called anytime whether @ss
+ * function can be called anytime whether @cfts' subsys is attached or not.
- * is attached or not.
 *
 * Returns 0 on successful unregistration, -ENOENT if @cfts is not
- * registered with @ss.
+ * registered.
 */
-int cgroup_rm_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
+int cgroup_rm_cftypes(struct cftype *cfts)
 {
        struct cftype_set *set;
+        if (!cfts || !cfts[0].ss)
+                return -ENOENT;
        cgroup_cfts_prepare();
-        list_for_each_entry(set, &ss->cftsets, node) {
+        list_for_each_entry(set, &cfts[0].ss->cftsets, node) {
                if (set->cfts == cfts) {
                        list_del(&set->node);
                        kfree(set);
-                        cgroup_cfts_commit(ss, cfts, false);
+                        cgroup_cfts_commit(cfts, false);
                        return 0;
                }
        }
-        cgroup_cfts_commit(ss, NULL, false);
+        cgroup_cfts_commit(NULL, false);
        return -ENOENT;
 }
@@ -2966,34 +2988,10 @@ int cgroup_task_count(const struct cgroup *cgrp)
 }
 /*
- * Advance a list_head iterator.  The iterator should be positioned at
+ * To reduce the fork() overhead for systems that are not actually using
- * the start of a css_set
+ * their cgroups capability, we don't maintain the lists running through
- */
+ * each css_set to its tasks until we see the list actually used - in other
-static void cgroup_advance_iter(struct cgroup *cgrp, struct cgroup_iter *it)
+ * words after the first call to css_task_iter_start().
-{
-        struct list_head *l = it->cset_link;
-        struct cgrp_cset_link *link;
-        struct css_set *cset;
-        /* Advance to the next non-empty css_set */
-        do {
-                l = l->next;
-                if (l == &cgrp->cset_links) {
-                        it->cset_link = NULL;
-                        return;
-                }
-                link = list_entry(l, struct cgrp_cset_link, cset_link);
-                cset = link->cset;
-        } while (list_empty(&cset->tasks));
-        it->cset_link = l;
-        it->task = cset->tasks.next;
-}
-/*
- * To reduce the fork() overhead for systems that are not actually
- * using their cgroups capability, we don't maintain the lists running
- * through each css_set to its tasks until we see the list actually
- * used - in other words after the first call to cgroup_iter_start().
 */
 static void cgroup_enable_task_cg_lists(void)
 {
@@ -3024,16 +3022,21 @@ static void cgroup_enable_task_cg_lists(void)
 }
 /**
- * cgroup_next_sibling - find the next sibling of a given cgroup
+ * css_next_child - find the next child of a given css
- * @pos: the current cgroup
+ * @pos_css: the current position (%NULL to initiate traversal)
+ * @parent_css: css whose children to walk
 *
- * This function returns the next sibling of @pos and should be called
+ * This function returns the next child of @parent_css and should be called
- * under RCU read lock.  The only requirement is that @pos is accessible.
+ * under RCU read lock.  The only requirement is that @parent_css and
- * The next sibling is guaranteed to be returned regardless of @pos's
+ * @pos_css are accessible.  The next sibling is guaranteed to be returned
- * state.
+ * regardless of their states.
 */
-struct cgroup *cgroup_next_sibling(struct cgroup *pos)
+struct cgroup_subsys_state *
+css_next_child(struct cgroup_subsys_state *pos_css,
+               struct cgroup_subsys_state *parent_css)
 {
+        struct cgroup *pos = pos_css ? pos_css->cgroup : NULL;
+        struct cgroup *cgrp = parent_css->cgroup;
        struct cgroup *next;
        WARN_ON_ONCE(!rcu_read_lock_held());
@@ -3048,78 +3051,81 @@ struct cgroup *cgroup_next_sibling(struct cgroup *pos)
         * safe to dereference from this RCU critical section.  If
         * ->sibling.next is inaccessible, cgroup_is_dead() is guaranteed
         * to be visible as %true here.
+         *
+         * If @pos is dead, its next pointer can't be dereferenced;
+         * however, as each cgroup is given a monotonically increasing
+         * unique serial number and always appended to the sibling list,
+         * the next one can be found by walking the parent's children until
+         * we see a cgroup with higher serial number than @pos's.  While
+         * this path can be slower, it's taken only when either the current
+         * cgroup is removed or iteration and removal race.
         */
-        if (likely(!cgroup_is_dead(pos))) {
+        if (!pos) {
+                next = list_entry_rcu(cgrp->children.next, struct cgroup, sibling);
+        } else if (likely(!cgroup_is_dead(pos))) {
                next = list_entry_rcu(pos->sibling.next, struct cgroup, sibling);
-                if (&next->sibling != &pos->parent->children)
+        } else {
-                        return next;
+                list_for_each_entry_rcu(next, &cgrp->children, sibling)
-                return NULL;
+                        if (next->serial_nr > pos->serial_nr)
+                                break;
        }
-        /*
+        if (&next->sibling == &cgrp->children)
-         * Can't dereference the next pointer.  Each cgroup is given a
+                return NULL;
-         * monotonically increasing unique serial number and always
-         * appended to the sibling list, so the next one can be found by
+        return cgroup_css(next, parent_css->ss);
-         * walking the parent's children until we see a cgroup with higher
-         * serial number than @pos's.
-         *
-         * While this path can be slow, it's taken only when either the
-         * current cgroup is removed or iteration and removal race.
-         */
-        list_for_each_entry_rcu(next, &pos->parent->children, sibling)
-                if (next->serial_nr > pos->serial_nr)
-                        return next;
-        return NULL;
 }
-EXPORT_SYMBOL_GPL(cgroup_next_sibling);
+EXPORT_SYMBOL_GPL(css_next_child);
 /**
- * cgroup_next_descendant_pre - find the next descendant for pre-order walk
+ * css_next_descendant_pre - find the next descendant for pre-order walk
 * @pos: the current position (%NULL to initiate traversal)
- * @cgroup: cgroup whose descendants to walk
+ * @root: css whose descendants to walk
 *
- * To be used by cgroup_for_each_descendant_pre().  Find the next
+ * To be used by css_for_each_descendant_pre().  Find the next descendant
- * descendant to visit for pre-order traversal of @cgroup's descendants.
+ * to visit for pre-order traversal of @root's descendants.  @root is
+ * included in the iteration and the first node to be visited.
 *
 * While this function requires RCU read locking, it doesn't require the
 * whole traversal to be contained in a single RCU critical section.  This
 * function will return the correct next descendant as long as both @pos
- * and @cgroup are accessible and @pos is a descendant of @cgroup.
+ * and @root are accessible and @pos is a descendant of @root.
 */
-struct cgroup *cgroup_next_descendant_pre(struct cgroup *pos,
+struct cgroup_subsys_state *
-                                          struct cgroup *cgroup)
+css_next_descendant_pre(struct cgroup_subsys_state *pos,
+                        struct cgroup_subsys_state *root)
 {
-        struct cgroup *next;
+        struct cgroup_subsys_state *next;
        WARN_ON_ONCE(!rcu_read_lock_held());
-        /* if first iteration, pretend we just visited @cgroup */
+        /* if first iteration, visit @root */
        if (!pos)
-                pos = cgroup;
+                return root;
        /* visit the first child if exists */
-        next = list_first_or_null_rcu(&pos->children, struct cgroup, sibling);
+        next = css_next_child(NULL, pos);
        if (next)
                return next;
        /* no child, visit my or the closest ancestor's next sibling */
-        while (pos != cgroup) {
+        while (pos != root) {
-                next = cgroup_next_sibling(pos);
+                next = css_next_child(pos, css_parent(pos));
                if (next)
                        return next;
-                pos = pos->parent;
+                pos = css_parent(pos);
        }
        return NULL;
 }
-EXPORT_SYMBOL_GPL(cgroup_next_descendant_pre);
+EXPORT_SYMBOL_GPL(css_next_descendant_pre);
 /**
- * cgroup_rightmost_descendant - return the rightmost descendant of a cgroup
+ * css_rightmost_descendant - return the rightmost descendant of a css
- * @pos: cgroup of interest
+ * @pos: css of interest
 *
- * Return the rightmost descendant of @pos.  If there's no descendant,
+ * Return the rightmost descendant of @pos.  If there's no descendant, @pos
- * @pos is returned.  This can be used during pre-order traversal to skip
+ * is returned.  This can be used during pre-order traversal to skip
 * subtree of @pos.
 *
 * While this function requires RCU read locking, it doesn't require the
@@ -3127,9 +3133,10 @@ EXPORT_SYMBOL_GPL(cgroup_next_descendant_pre);
 * function will return the correct rightmost descendant as long as @pos is
 * accessible.
 */
-struct cgroup *cgroup_rightmost_descendant(struct cgroup *pos)
+struct cgroup_subsys_state *
+css_rightmost_descendant(struct cgroup_subsys_state *pos)
 {
-        struct cgroup *last, *tmp;
+        struct cgroup_subsys_state *last, *tmp;
        WARN_ON_ONCE(!rcu_read_lock_held());
@@ -3137,82 +3144,138 @@ struct cgroup *cgroup_rightmost_descendant(struct cgroup *pos)
                last = pos;
                /* ->prev isn't RCU safe, walk ->next till the end */
                pos = NULL;
-                list_for_each_entry_rcu(tmp, &last->children, sibling)
+                css_for_each_child(tmp, last)
                        pos = tmp;
        } while (pos);
        return last;
 }
-EXPORT_SYMBOL_GPL(cgroup_rightmost_descendant);
+EXPORT_SYMBOL_GPL(css_rightmost_descendant);
-static struct cgroup *cgroup_leftmost_descendant(struct cgroup *pos)
+static struct cgroup_subsys_state *
+css_leftmost_descendant(struct cgroup_subsys_state *pos)
 {
-        struct cgroup *last;
+        struct cgroup_subsys_state *last;
        do {
                last = pos;
-                pos = list_first_or_null_rcu(&pos->children, struct cgroup,
+                pos = css_next_child(NULL, pos);
-                                             sibling);
        } while (pos);
        return last;
 }
 /**
- * cgroup_next_descendant_post - find the next descendant for post-order walk
+ * css_next_descendant_post - find the next descendant for post-order walk
 * @pos: the current position (%NULL to initiate traversal)
- * @cgroup: cgroup whose descendants to walk
+ * @root: css whose descendants to walk
 *
- * To be used by cgroup_for_each_descendant_post().  Find the next
+ * To be used by css_for_each_descendant_post().  Find the next descendant
- * descendant to visit for post-order traversal of @cgroup's descendants.
+ * to visit for post-order traversal of @root's descendants.  @root is
+ * included in the iteration and the last node to be visited.
 *
 * While this function requires RCU read locking, it doesn't require the
 * whole traversal to be contained in a single RCU critical section.  This
 * function will return the correct next descendant as long as both @pos
 * and @cgroup are accessible and @pos is a descendant of @cgroup.
 */
-struct cgroup *cgroup_next_descendant_post(struct cgroup *pos,
+struct cgroup_subsys_state *
-                                           struct cgroup *cgroup)
+css_next_descendant_post(struct cgroup_subsys_state *pos,
+                         struct cgroup_subsys_state *root)
 {
-        struct cgroup *next;
+        struct cgroup_subsys_state *next;
        WARN_ON_ONCE(!rcu_read_lock_held());
        /* if first iteration, visit the leftmost descendant */
        if (!pos) {
-                next = cgroup_leftmost_descendant(cgroup);
+                next = css_leftmost_descendant(root);
-                return next != cgroup ? next : NULL;
+                return next != root ? next : NULL;
        }
+        /* if we visited @root, we're done */
+        if (pos == root)
+                return NULL;
        /* if there's an unvisited sibling, visit its leftmost descendant */
-        next = cgroup_next_sibling(pos);
+        next = css_next_child(pos, css_parent(pos));
        if (next)
-                return cgroup_leftmost_descendant(next);
+                return css_leftmost_descendant(next);
        /* no sibling left, visit parent */
-        next = pos->parent;
+        return css_parent(pos);
-        return next != cgroup ? next : NULL;
+}
+EXPORT_SYMBOL_GPL(css_next_descendant_post);
+/**
+ * css_advance_task_iter - advance a task itererator to the next css_set
+ * @it: the iterator to advance
+ *
+ * Advance @it to the next css_set to walk.
+ */
+static void css_advance_task_iter(struct css_task_iter *it)
+{
+        struct list_head *l = it->cset_link;
+        struct cgrp_cset_link *link;
+        struct css_set *cset;
+        /* Advance to the next non-empty css_set */
+        do {
+                l = l->next;
+                if (l == &it->origin_css->cgroup->cset_links) {
+                        it->cset_link = NULL;
+                        return;
+                }
+                link = list_entry(l, struct cgrp_cset_link, cset_link);
+                cset = link->cset;
+        } while (list_empty(&cset->tasks));
+        it->cset_link = l;
+        it->task = cset->tasks.next;
 }
-EXPORT_SYMBOL_GPL(cgroup_next_descendant_post);
-void cgroup_iter_start(struct cgroup *cgrp, struct cgroup_iter *it)
+/**
+ * css_task_iter_start - initiate task iteration
+ * @css: the css to walk tasks of
+ * @it: the task iterator to use
+ *
+ * Initiate iteration through the tasks of @css.  The caller can call
+ * css_task_iter_next() to walk through the tasks until the function
+ * returns NULL.  On completion of iteration, css_task_iter_end() must be
+ * called.
+ *
+ * Note that this function acquires a lock which is released when the
+ * iteration finishes.  The caller can't sleep while iteration is in
+ * progress.
+ */
+void css_task_iter_start(struct cgroup_subsys_state *css,
+                         struct css_task_iter *it)
        __acquires(css_set_lock)
 {
        /*
-         * The first time anyone tries to iterate across a cgroup,
+         * The first time anyone tries to iterate across a css, we need to
-         * we need to enable the list linking each css_set to its
+         * enable the list linking each css_set to its tasks, and fix up
-         * tasks, and fix up all existing tasks.
+         * all existing tasks.
         */
        if (!use_task_css_set_links)
                cgroup_enable_task_cg_lists();
        read_lock(&css_set_lock);
-        it->cset_link = &cgrp->cset_links;
-        cgroup_advance_iter(cgrp, it);
+        it->origin_css = css;
+        it->cset_link = &css->cgroup->cset_links;
+        css_advance_task_iter(it);
 }
-struct task_struct *cgroup_iter_next(struct cgroup *cgrp,
+/**
-                                        struct cgroup_iter *it)
+ * css_task_iter_next - return the next task for the iterator
+ * @it: the task iterator being iterated
+ *
+ * The "next" function for task iteration.  @it should have been
+ * initialized via css_task_iter_start().  Returns NULL when the iteration
+ * reaches the end.
+ */
+struct task_struct *css_task_iter_next(struct css_task_iter *it)
 {
        struct task_struct *res;
        struct list_head *l = it->task;
@@ -3226,16 +3289,24 @@ struct task_struct *cgroup_iter_next(struct cgroup *cgrp,
        l = l->next;
        link = list_entry(it->cset_link, struct cgrp_cset_link, cset_link);
        if (l == &link->cset->tasks) {
-                /* We reached the end of this task list - move on to
+                /*
-                 * the next cg_cgroup_link */
+                 * We reached the end of this task list - move on to the
-                cgroup_advance_iter(cgrp, it);
+                 * next cgrp_cset_link.
+                 */
+                css_advance_task_iter(it);
        } else {
                it->task = l;
        }
        return res;
 }
-void cgroup_iter_end(struct cgroup *cgrp, struct cgroup_iter *it)
+/**
+ * css_task_iter_end - finish task iteration
+ * @it: the task iterator to finish
+ *
+ * Finish task iteration started by css_task_iter_start().
+ */
+void css_task_iter_end(struct css_task_iter *it)
        __releases(css_set_lock)
 {
        read_unlock(&css_set_lock);
@@ -3276,46 +3347,49 @@ static inline int started_after(void *p1, void *p2)
 }
 /**
- * cgroup_scan_tasks - iterate though all the tasks in a cgroup
+ * css_scan_tasks - iterate though all the tasks in a css
- * @scan: struct cgroup_scanner containing arguments for the scan
+ * @css: the css to iterate tasks of
+ * @test: optional test callback
+ * @process: process callback
+ * @data: data passed to @test and @process
+ * @heap: optional pre-allocated heap used for task iteration
+ *
+ * Iterate through all the tasks in @css, calling @test for each, and if it
+ * returns %true, call @process for it also.
+ *
+ * @test may be NULL, meaning always true (select all tasks), which
+ * effectively duplicates css_task_iter_{start,next,end}() but does not
+ * lock css_set_lock for the call to @process.
 *
- * Arguments include pointers to callback functions test_task() and
+ * It is guaranteed that @process will act on every task that is a member
- * process_task().
+ * of @css for the duration of this call.  This function may or may not
- * Iterate through all the tasks in a cgroup, calling test_task() for each,
+ * call @process for tasks that exit or move to a different css during the
- * and if it returns true, call process_task() for it also.
+ * call, or are forked or move into the css during the call.
- * The test_task pointer may be NULL, meaning always true (select all tasks).
- * Effectively duplicates cgroup_iter_{start,next,end}()
- * but does not lock css_set_lock for the call to process_task().
- * The struct cgroup_scanner may be embedded in any structure of the caller's
- * creation.
- * It is guaranteed that process_task() will act on every task that
- * is a member of the cgroup for the duration of this call. This
- * function may or may not call process_task() for tasks that exit
- * or move to a different cgroup during the call, or are forked or
- * move into the cgroup during the call.
 *
- * Note that test_task() may be called with locks held, and may in some
+ * Note that @test may be called with locks held, and may in some
- * situations be called multiple times for the same task, so it should
+ * situations be called multiple times for the same task, so it should be
- * be cheap.
+ * cheap.
- * If the heap pointer in the struct cgroup_scanner is non-NULL, a heap has been
+ *
- * pre-allocated and will be used for heap operations (and its "gt" member will
+ * If @heap is non-NULL, a heap has been pre-allocated and will be used for
- * be overwritten), else a temporary heap will be used (allocation of which
+ * heap operations (and its "gt" member will be overwritten), else a
- * may cause this function to fail).
+ * temporary heap will be used (allocation of which may cause this function
+ * to fail).
 */
-int cgroup_scan_tasks(struct cgroup_scanner *scan)
+int css_scan_tasks(struct cgroup_subsys_state *css,
+                   bool (*test)(struct task_struct *, void *),
+                   void (*process)(struct task_struct *, void *),
+                   void *data, struct ptr_heap *heap)
 {
        int retval, i;
-        struct cgroup_iter it;
+        struct css_task_iter it;
        struct task_struct *p, *dropped;
        /* Never dereference latest_task, since it's not refcounted */
        struct task_struct *latest_task = NULL;
        struct ptr_heap tmp_heap;
-        struct ptr_heap *heap;
        struct timespec latest_time = { 0, 0 };
-        if (scan->heap) {
+        if (heap) {
                /* The caller supplied our heap and pre-allocated its memory */
-                heap = scan->heap;
                heap->gt = &started_after;
        } else {
                /* We need to allocate our own heap memory */
@@ -3328,25 +3402,24 @@ int cgroup_scan_tasks(struct cgroup_scanner *scan)
 again:
        /*
-         * Scan tasks in the cgroup, using the scanner's "test_task" callback
+         * Scan tasks in the css, using the @test callback to determine
-         * to determine which are of interest, and using the scanner's
+         * which are of interest, and invoking @process callback on the
-         * "process_task" callback to process any of them that need an update.
+         * ones which need an update.  Since we don't want to hold any
-         * Since we don't want to hold any locks during the task updates,
+         * locks during the task updates, gather tasks to be processed in a
-         * gather tasks to be processed in a heap structure.
+         * heap structure.  The heap is sorted by descending task start
-         * The heap is sorted by descending task start time.
+         * time.  If the statically-sized heap fills up, we overflow tasks
-         * If the statically-sized heap fills up, we overflow tasks that
+         * that started later, and in future iterations only consider tasks
-         * started later, and in future iterations only consider tasks that
+         * that started after the latest task in the previous pass. This
-         * started after the latest task in the previous pass. This
         * guarantees forward progress and that we don't miss any tasks.
         */
        heap->size = 0;
-        cgroup_iter_start(scan->cg, &it);
+        css_task_iter_start(css, &it);
-        while ((p = cgroup_iter_next(scan->cg, &it))) {
+        while ((p = css_task_iter_next(&it))) {
                /*
                 * Only affect tasks that qualify per the caller's callback,
                 * if he provided one
                 */
-                if (scan->test_task && !scan->test_task(p, scan))
+                if (test && !test(p, data))
                        continue;
                /*
                 * Only process tasks that started after the last task
@@ -3374,7 +3447,7 @@ int cgroup_scan_tasks(struct cgroup_scanner *scan)
                 * the heap and wasn't inserted
                 */
        }
-        cgroup_iter_end(scan->cg, &it);
+        css_task_iter_end(&it);
        if (heap->size) {
                for (i = 0; i < heap->size; i++) {
@@ -3384,7 +3457,7 @@ int cgroup_scan_tasks(struct cgroup_scanner *scan)
                                latest_task = q;
                        }
                        /* Process the task per the caller's callback */
-                        scan->process_task(q, scan);
+                        process(q, data);
                        put_task_struct(q);
                }
                /*
@@ -3401,10 +3474,9 @@ int cgroup_scan_tasks(struct cgroup_scanner *scan)
        return 0;
 }
-static void cgroup_transfer_one_task(struct task_struct *task,
+static void cgroup_transfer_one_task(struct task_struct *task, void *data)
-                                     struct cgroup_scanner *scan)
 {
-        struct cgroup *new_cgroup = scan->data;
+        struct cgroup *new_cgroup = data;
        mutex_lock(&cgroup_mutex);
        cgroup_attach_task(new_cgroup, task, false);
@@ -3418,15 +3490,8 @@ static void cgroup_transfer_one_task(struct task_struct *task,
 */
 int cgroup_transfer_tasks(struct cgroup *to, struct cgroup *from)
 {
-        struct cgroup_scanner scan;
+        return css_scan_tasks(&from->dummy_css, NULL, cgroup_transfer_one_task,
+                              to, NULL);
-        scan.cg = from;
-        scan.test_task = NULL; /* select all tasks in cgroup */
-        scan.process_task = cgroup_transfer_one_task;
-        scan.heap = NULL;
-        scan.data = to;
-        return cgroup_scan_tasks(&scan);
 }
 /*
@@ -3468,7 +3533,7 @@ struct cgroup_pidlist {
        /* pointer to the cgroup we belong to, for list removal purposes */
        struct cgroup *owner;
        /* protects the other fields */
-        struct rw_semaphore mutex;
+        struct rw_semaphore rwsem;
 };
 /*
@@ -3541,7 +3606,7 @@ static struct cgroup_pidlist *cgroup_pidlist_find(struct cgroup *cgrp,
        struct pid_namespace *ns = task_active_pid_ns(current);
        /*
-         * We can't drop the pidlist_mutex before taking the l->mutex in case
+         * We can't drop the pidlist_mutex before taking the l->rwsem in case
         * the last ref-holder is trying to remove l from the list at the same
         * time. Holding the pidlist_mutex precludes somebody taking whichever
         * list we find out from under us - compare release_pid_array().
@@ -3550,7 +3615,7 @@ static struct cgroup_pidlist *cgroup_pidlist_find(struct cgroup *cgrp,
        list_for_each_entry(l, &cgrp->pidlists, links) {
                if (l->key.type == type && l->key.ns == ns) {
                        /* make sure l doesn't vanish out from under us */
-                        down_write(&l->mutex);
+                        down_write(&l->rwsem);
                        mutex_unlock(&cgrp->pidlist_mutex);
                        return l;
                }
@@ -3561,8 +3626,8 @@ static struct cgroup_pidlist *cgroup_pidlist_find(struct cgroup *cgrp,
                mutex_unlock(&cgrp->pidlist_mutex);
                return l;
        }
-        init_rwsem(&l->mutex);
+        init_rwsem(&l->rwsem);
-        down_write(&l->mutex);
+        down_write(&l->rwsem);
        l->key.type = type;
        l->key.ns = get_pid_ns(ns);
        l->owner = cgrp;
@@ -3580,7 +3645,7 @@ static int pidlist_array_load(struct cgroup *cgrp, enum cgroup_filetype type,
        pid_t *array;
        int length;
        int pid, n = 0; /* used for populating the array */
-        struct cgroup_iter it;
+        struct css_task_iter it;
        struct task_struct *tsk;
        struct cgroup_pidlist *l;
@@ -3595,8 +3660,8 @@ static int pidlist_array_load(struct cgroup *cgrp, enum cgroup_filetype type,
        if (!array)
                return -ENOMEM;
        /* now, populate the array */
-        cgroup_iter_start(cgrp, &it);
+        css_task_iter_start(&cgrp->dummy_css, &it);
-        while ((tsk = cgroup_iter_next(cgrp, &it))) {
+        while ((tsk = css_task_iter_next(&it))) {
                if (unlikely(n == length))
                        break;
                /* get tgid or pid for procs or tasks file respectively */
@@ -3607,7 +3672,7 @@ static int pidlist_array_load(struct cgroup *cgrp, enum cgroup_filetype type,
                if (pid > 0) /* make sure to only use valid results */
                        array[n++] = pid;
        }
-        cgroup_iter_end(cgrp, &it);
+        css_task_iter_end(&it);
        length = n;
        /* now sort & (if procs) strip out duplicates */
        sort(array, length, sizeof(pid_t), cmppid, NULL);
@@ -3623,7 +3688,7 @@ static int pidlist_array_load(struct cgroup *cgrp, enum cgroup_filetype type,
        l->list = array;
        l->length = length;
        l->use_count++;
-        up_write(&l->mutex);
+        up_write(&l->rwsem);
        *lp = l;
        return 0;
 }
@@ -3641,7 +3706,7 @@ int cgroupstats_build(struct cgroupstats *stats, struct dentry *dentry)
 {
        int ret = -EINVAL;
        struct cgroup *cgrp;
-        struct cgroup_iter it;
+        struct css_task_iter it;
        struct task_struct *tsk;
        /*
@@ -3655,8 +3720,8 @@ int cgroupstats_build(struct cgroupstats *stats, struct dentry *dentry)
        ret = 0;
        cgrp = dentry->d_fsdata;
-        cgroup_iter_start(cgrp, &it);
+        css_task_iter_start(&cgrp->dummy_css, &it);
-        while ((tsk = cgroup_iter_next(cgrp, &it))) {
+        while ((tsk = css_task_iter_next(&it))) {
                switch (tsk->state) {
                case TASK_RUNNING:
                        stats->nr_running++;
@@ -3676,7 +3741,7 @@ int cgroupstats_build(struct cgroupstats *stats, struct dentry *dentry)
                        break;
                }
        }
-        cgroup_iter_end(cgrp, &it);
+        css_task_iter_end(&it);
 err:
        return ret;
@@ -3701,7 +3766,7 @@ static void *cgroup_pidlist_start(struct seq_file *s, loff_t *pos)
        int index = 0, pid = *pos;
        int *iter;
-        down_read(&l->mutex);
+        down_read(&l->rwsem);
        if (pid) {
                int end = l->length;
@@ -3728,7 +3793,7 @@ static void *cgroup_pidlist_start(struct seq_file *s, loff_t *pos)
 static void cgroup_pidlist_stop(struct seq_file *s, void *v)
 {
        struct cgroup_pidlist *l = s->private;
-        up_read(&l->mutex);
+        up_read(&l->rwsem);
 }
 static void *cgroup_pidlist_next(struct seq_file *s, void *v, loff_t *pos)
@@ -3774,7 +3839,7 @@ static void cgroup_release_pid_array(struct cgroup_pidlist *l)
         * pidlist_mutex, we have to take pidlist_mutex first.
         */
        mutex_lock(&l->owner->pidlist_mutex);
-        down_write(&l->mutex);
+        down_write(&l->rwsem);
        BUG_ON(!l->use_count);
        if (!--l->use_count) {
                /* we're the last user if refcount is 0; remove and free */
@@ -3782,12 +3847,12 @@ static void cgroup_release_pid_array(struct cgroup_pidlist *l)
                mutex_unlock(&l->owner->pidlist_mutex);
                pidlist_free(l->list);
                put_pid_ns(l->key.ns);
-                up_write(&l->mutex);
+                up_write(&l->rwsem);
                kfree(l);
                return;
        }
        mutex_unlock(&l->owner->pidlist_mutex);
-        up_write(&l->mutex);
+        up_write(&l->rwsem);
 }
 static int cgroup_pidlist_release(struct inode *inode, struct file *file)
@@ -3851,21 +3916,20 @@ static int cgroup_procs_open(struct inode *unused, struct file *file)
        return cgroup_pidlist_open(file, CGROUP_FILE_PROCS);
 }
-static u64 cgroup_read_notify_on_release(struct cgroup *cgrp,
+static u64 cgroup_read_notify_on_release(struct cgroup_subsys_state *css,
-                                            struct cftype *cft)
+                                         struct cftype *cft)
 {
-        return notify_on_release(cgrp);
+        return notify_on_release(css->cgroup);
 }
-static int cgroup_write_notify_on_release(struct cgroup *cgrp,
+static int cgroup_write_notify_on_release(struct cgroup_subsys_state *css,
-                                          struct cftype *cft,
+                                          struct cftype *cft, u64 val)
-                                          u64 val)
 {
-        clear_bit(CGRP_RELEASABLE, &cgrp->flags);
+        clear_bit(CGRP_RELEASABLE, &css->cgroup->flags);
        if (val)
-                set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
+                set_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags);
        else
-                clear_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
+                clear_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags);
        return 0;
 }
@@ -3895,18 +3959,18 @@ static void cgroup_event_remove(struct work_struct *work)
 {
        struct cgroup_event *event = container_of(work, struct cgroup_event,
                        remove);
-        struct cgroup *cgrp = event->cgrp;
+        struct cgroup_subsys_state *css = event->css;
        remove_wait_queue(event->wqh, &event->wait);
-        event->cft->unregister_event(cgrp, event->cft, event->eventfd);
+        event->cft->unregister_event(css, event->cft, event->eventfd);
        /* Notify userspace the event is going away. */
        eventfd_signal(event->eventfd, 1);
        eventfd_ctx_put(event->eventfd);
        kfree(event);
-        cgroup_dput(cgrp);
+        css_put(css);
 }
 /*
@@ -3919,7 +3983,7 @@ static int cgroup_event_wake(wait_queue_t *wait, unsigned mode,
 {
        struct cgroup_event *event = container_of(wait,
                        struct cgroup_event, wait);
-        struct cgroup *cgrp = event->cgrp;
+        struct cgroup *cgrp = event->css->cgroup;
        unsigned long flags = (unsigned long)key;
        if (flags & POLLHUP) {
@@ -3963,14 +4027,15 @@ static void cgroup_event_ptable_queue_proc(struct file *file,
 * Input must be in format '<event_fd> <control_fd> <args>'.
 * Interpretation of args is defined by control file implementation.
 */
-static int cgroup_write_event_control(struct cgroup *cgrp, struct cftype *cft,
+static int cgroup_write_event_control(struct cgroup_subsys_state *dummy_css,
-                                      const char *buffer)
+                                      struct cftype *cft, const char *buffer)
 {
-        struct cgroup_event *event = NULL;
+        struct cgroup *cgrp = dummy_css->cgroup;
-        struct cgroup *cgrp_cfile;
+        struct cgroup_event *event;
+        struct cgroup_subsys_state *cfile_css;
        unsigned int efd, cfd;
-        struct file *efile = NULL;
+        struct file *efile;
-        struct file *cfile = NULL;
+        struct file *cfile;
        char *endp;
        int ret;
@@ -3987,7 +4052,7 @@ static int cgroup_write_event_control(struct cgroup *cgrp, struct cftype *cft,
        event = kzalloc(sizeof(*event), GFP_KERNEL);
        if (!event)
                return -ENOMEM;
-        event->cgrp = cgrp;
        INIT_LIST_HEAD(&event->list);
        init_poll_funcptr(&event->pt, cgroup_event_ptable_queue_proc);
        init_waitqueue_func_entry(&event->wait, cgroup_event_wake);
@@ -3996,62 +4061,68 @@ static int cgroup_write_event_control(struct cgroup *cgrp, struct cftype *cft,
        efile = eventfd_fget(efd);
        if (IS_ERR(efile)) {
                ret = PTR_ERR(efile);
-                goto fail;
+                goto out_kfree;
        }
        event->eventfd = eventfd_ctx_fileget(efile);
        if (IS_ERR(event->eventfd)) {
                ret = PTR_ERR(event->eventfd);
-                goto fail;
+                goto out_put_efile;
        }
        cfile = fget(cfd);
        if (!cfile) {
                ret = -EBADF;
-                goto fail;
+                goto out_put_eventfd;
        }
        /* the process need read permission on control file */
        /* AV: shouldn't we check that it's been opened for read instead? */
        ret = inode_permission(file_inode(cfile), MAY_READ);
        if (ret < 0)
-                goto fail;
+                goto out_put_cfile;
        event->cft = __file_cft(cfile);
        if (IS_ERR(event->cft)) {
                ret = PTR_ERR(event->cft);
-                goto fail;
+                goto out_put_cfile;
+        }
+        if (!event->cft->ss) {
+                ret = -EBADF;
+                goto out_put_cfile;
        }
        /*
-         * The file to be monitored must be in the same cgroup as
+         * Determine the css of @cfile, verify it belongs to the same
-         * cgroup.event_control is.
+         * cgroup as cgroup.event_control, and associate @event with it.
+         * Remaining events are automatically removed on cgroup destruction
+         * but the removal is asynchronous, so take an extra ref.
         */
-        cgrp_cfile = __d_cgrp(cfile->f_dentry->d_parent);
+        rcu_read_lock();
-        if (cgrp_cfile != cgrp) {
-                ret = -EINVAL;
+        ret = -EINVAL;
-                goto fail;
+        event->css = cgroup_css(cgrp, event->cft->ss);
-        }
+        cfile_css = css_from_dir(cfile->f_dentry->d_parent, event->cft->ss);
+        if (event->css && event->css == cfile_css && css_tryget(event->css))
+                ret = 0;
+        rcu_read_unlock();
+        if (ret)
+                goto out_put_cfile;
        if (!event->cft->register_event || !event->cft->unregister_event) {
                ret = -EINVAL;
-                goto fail;
+                goto out_put_css;
        }
-        ret = event->cft->register_event(cgrp, event->cft,
+        ret = event->cft->register_event(event->css, event->cft,
                        event->eventfd, buffer);
        if (ret)
-                goto fail;
+                goto out_put_css;
        efile->f_op->poll(efile, &event->pt);
-        /*
-         * Events should be removed after rmdir of cgroup directory, but before
-         * destroying subsystem state objects. Let's take reference to cgroup
-         * directory dentry to do that.
-         */
-        dget(cgrp->dentry);
        spin_lock(&cgrp->event_list_lock);
        list_add(&event->list, &cgrp->event_list);
        spin_unlock(&cgrp->event_list_lock);
@@ -4061,35 +4132,33 @@ static int cgroup_write_event_control(struct cgroup *cgrp, struct cftype *cft,
        return 0;
-fail:
+out_put_css:
-        if (cfile)
+        css_put(event->css);
-                fput(cfile);
+out_put_cfile:
+        fput(cfile);
-        if (event && event->eventfd && !IS_ERR(event->eventfd))
+out_put_eventfd:
-                eventfd_ctx_put(event->eventfd);
+        eventfd_ctx_put(event->eventfd);
+out_put_efile:
-        if (!IS_ERR_OR_NULL(efile))
+        fput(efile);
-                fput(efile);
+out_kfree:
        kfree(event);
        return ret;
 }
-static u64 cgroup_clone_children_read(struct cgroup *cgrp,
+static u64 cgroup_clone_children_read(struct cgroup_subsys_state *css,
-                                    struct cftype *cft)
+                                      struct cftype *cft)
 {
-        return test_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags);
+        return test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
 }
-static int cgroup_clone_children_write(struct cgroup *cgrp,
+static int cgroup_clone_children_write(struct cgroup_subsys_state *css,
-                                     struct cftype *cft,
+                                       struct cftype *cft, u64 val)
-                                     u64 val)
 {
        if (val)
-                set_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags);
+                set_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
        else
-                clear_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags);
+                clear_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
        return 0;
 }
@@ -4148,36 +4217,34 @@ static struct cftype cgroup_base_files[] = {
 };
 /**
- * cgroup_populate_dir - selectively creation of files in a directory
+ * cgroup_populate_dir - create subsys files in a cgroup directory
 * @cgrp: target cgroup
- * @base_files: true if the base files should be added
 * @subsys_mask: mask of the subsystem ids whose files should be added
+ *
+ * On failure, no file is added.
 */
-static int cgroup_populate_dir(struct cgroup *cgrp, bool base_files,
+static int cgroup_populate_dir(struct cgroup *cgrp, unsigned long subsys_mask)
-                               unsigned long subsys_mask)
 {
-        int err;
        struct cgroup_subsys *ss;
+        int i, ret = 0;
-        if (base_files) {
-                err = cgroup_addrm_files(cgrp, NULL, cgroup_base_files, true);
-                if (err < 0)
-                        return err;
-        }
        /* process cftsets of each subsystem */
-        for_each_root_subsys(cgrp->root, ss) {
+        for_each_subsys(ss, i) {
                struct cftype_set *set;
-                if (!test_bit(ss->subsys_id, &subsys_mask))
+                if (!test_bit(i, &subsys_mask))
                        continue;
-                list_for_each_entry(set, &ss->cftsets, node)
+                list_for_each_entry(set, &ss->cftsets, node) {
-                        cgroup_addrm_files(cgrp, ss, set->cfts, true);
+                        ret = cgroup_addrm_files(cgrp, set->cfts, true);
+                        if (ret < 0)
+                                goto err;
+                }
        }
        /* This cgroup is ready now */
        for_each_root_subsys(cgrp->root, ss) {
-                struct cgroup_subsys_state *css = cgrp->subsys[ss->subsys_id];
+                struct cgroup_subsys_state *css = cgroup_css(cgrp, ss);
                struct css_id *id = rcu_dereference_protected(css->id, true);
                /*
@@ -4190,14 +4257,57 @@ static int cgroup_populate_dir(struct cgroup *cgrp, bool base_files,
        }
        return 0;
+err:
+        cgroup_clear_dir(cgrp, subsys_mask);
+        return ret;
+}
+/*
+ * css destruction is four-stage process.
+ *
+ * 1. Destruction starts.  Killing of the percpu_ref is initiated.
+ *    Implemented in kill_css().
+ *
+ * 2. When the percpu_ref is confirmed to be visible as killed on all CPUs
+ *    and thus css_tryget() is guaranteed to fail, the css can be offlined
+ *    by invoking offline_css().  After offlining, the base ref is put.
+ *    Implemented in css_killed_work_fn().
+ *
+ * 3. When the percpu_ref reaches zero, the only possible remaining
+ *    accessors are inside RCU read sections.  css_release() schedules the
+ *    RCU callback.
+ *
+ * 4. After the grace period, the css can be freed.  Implemented in
+ *    css_free_work_fn().
+ *
+ * It is actually hairier because both step 2 and 4 require process context
+ * and thus involve punting to css->destroy_work adding two additional
+ * steps to the already complex sequence.
+ */
+static void css_free_work_fn(struct work_struct *work)
+{
+        struct cgroup_subsys_state *css =
+                container_of(work, struct cgroup_subsys_state, destroy_work);
+        struct cgroup *cgrp = css->cgroup;
+        if (css->parent)
+                css_put(css->parent);
+        css->ss->css_free(css);
+        cgroup_dput(cgrp);
 }
-static void css_dput_fn(struct work_struct *work)
+static void css_free_rcu_fn(struct rcu_head *rcu_head)
 {
        struct cgroup_subsys_state *css =
-                container_of(work, struct cgroup_subsys_state, dput_work);
+                container_of(rcu_head, struct cgroup_subsys_state, rcu_head);
-        cgroup_dput(css->cgroup);
+        /*
+         * css holds an extra ref to @cgrp->dentry which is put on the last
+         * css_put().  dput() requires process context which we don't have.
+         */
+        INIT_WORK(&css->destroy_work, css_free_work_fn);
+        schedule_work(&css->destroy_work);
 }
 static void css_release(struct percpu_ref *ref)
@@ -4205,49 +4315,47 @@ static void css_release(struct percpu_ref *ref)
        struct cgroup_subsys_state *css =
                container_of(ref, struct cgroup_subsys_state, refcnt);
-        schedule_work(&css->dput_work);
+        call_rcu(&css->rcu_head, css_free_rcu_fn);
 }
-static void init_cgroup_css(struct cgroup_subsys_state *css,
+static void init_css(struct cgroup_subsys_state *css, struct cgroup_subsys *ss,
-                               struct cgroup_subsys *ss,
+                     struct cgroup *cgrp)
-                               struct cgroup *cgrp)
 {
        css->cgroup = cgrp;
+        css->ss = ss;
        css->flags = 0;
        css->id = NULL;
-        if (cgrp == cgroup_dummy_top)
+        if (cgrp->parent)
+                css->parent = cgroup_css(cgrp->parent, ss);
+        else
                css->flags |= CSS_ROOT;
-        BUG_ON(cgrp->subsys[ss->subsys_id]);
-        cgrp->subsys[ss->subsys_id] = css;
-        /*
+        BUG_ON(cgroup_css(cgrp, ss));
-         * css holds an extra ref to @cgrp->dentry which is put on the last
-         * css_put().  dput() requires process context, which css_put() may
-         * be called without.  @css->dput_work will be used to invoke
-         * dput() asynchronously from css_put().
-         */
-        INIT_WORK(&css->dput_work, css_dput_fn);
 }
-/* invoke ->post_create() on a new CSS and mark it online if successful */
+/* invoke ->css_online() on a new CSS and mark it online if successful */
-static int online_css(struct cgroup_subsys *ss, struct cgroup *cgrp)
+static int online_css(struct cgroup_subsys_state *css)
 {
+        struct cgroup_subsys *ss = css->ss;
        int ret = 0;
        lockdep_assert_held(&cgroup_mutex);
        if (ss->css_online)
-                ret = ss->css_online(cgrp);
+                ret = ss->css_online(css);
-        if (!ret)
+        if (!ret) {
-                cgrp->subsys[ss->subsys_id]->flags |= CSS_ONLINE;
+                css->flags |= CSS_ONLINE;
+                css->cgroup->nr_css++;
+                rcu_assign_pointer(css->cgroup->subsys[ss->subsys_id], css);
+        }
        return ret;
 }
-/* if the CSS is online, invoke ->pre_destory() on it and mark it offline */
+/* if the CSS is online, invoke ->css_offline() on it and mark it offline */
-static void offline_css(struct cgroup_subsys *ss, struct cgroup *cgrp)
+static void offline_css(struct cgroup_subsys_state *css)
-        __releases(&cgroup_mutex) __acquires(&cgroup_mutex)
 {
-        struct cgroup_subsys_state *css = cgrp->subsys[ss->subsys_id];
+        struct cgroup_subsys *ss = css->ss;
        lockdep_assert_held(&cgroup_mutex);
@@ -4255,9 +4363,11 @@ static void offline_css(struct cgroup_subsys *ss, struct cgroup *cgrp)
                return;
        if (ss->css_offline)
-                ss->css_offline(cgrp);
+                ss->css_offline(css);
-        cgrp->subsys[ss->subsys_id]->flags &= ~CSS_ONLINE;
+        css->flags &= ~CSS_ONLINE;
+        css->cgroup->nr_css--;
+        RCU_INIT_POINTER(css->cgroup->subsys[ss->subsys_id], css);
 }
 /*
@@ -4271,6 +4381,7 @@ static void offline_css(struct cgroup_subsys *ss, struct cgroup *cgrp)
 static long cgroup_create(struct cgroup *parent, struct dentry *dentry,
                             umode_t mode)
 {
+        struct cgroup_subsys_state *css_ar[CGROUP_SUBSYS_COUNT] = { };
        struct cgroup *cgrp;
        struct cgroup_name *name;
        struct cgroupfs_root *root = parent->root;
@@ -4288,7 +4399,11 @@ static long cgroup_create(struct cgroup *parent, struct dentry *dentry,
                goto err_free_cgrp;
        rcu_assign_pointer(cgrp->name, name);
-        cgrp->id = ida_simple_get(&root->cgroup_ida, 1, 0, GFP_KERNEL);
+        /*
+         * Temporarily set the pointer to NULL, so idr_find() won't return
+         * a half-baked cgroup.
+         */
+        cgrp->id = idr_alloc(&root->cgroup_idr, NULL, 1, 0, GFP_KERNEL);
        if (cgrp->id < 0)
                goto err_free_name;
@@ -4317,6 +4432,7 @@ static long cgroup_create(struct cgroup *parent, struct dentry *dentry,
        cgrp->dentry = dentry;
        cgrp->parent = parent;
+        cgrp->dummy_css.parent = &parent->dummy_css;
        cgrp->root = parent->root;
        if (notify_on_release(parent))
@@ -4328,22 +4444,21 @@ static long cgroup_create(struct cgroup *parent, struct dentry *dentry,
        for_each_root_subsys(root, ss) {
                struct cgroup_subsys_state *css;
-                css = ss->css_alloc(cgrp);
+                css = ss->css_alloc(cgroup_css(parent, ss));
                if (IS_ERR(css)) {
                        err = PTR_ERR(css);
                        goto err_free_all;
                }
+                css_ar[ss->subsys_id] = css;
                err = percpu_ref_init(&css->refcnt, css_release);
-                if (err) {
+                if (err)
-                        ss->css_free(cgrp);
                        goto err_free_all;
-                }
-                init_cgroup_css(css, ss, cgrp);
+                init_css(css, ss, cgrp);
                if (ss->use_id) {
-                        err = alloc_css_id(ss, parent, cgrp);
+                        err = alloc_css_id(css);
                        if (err)
                                goto err_free_all;
                }
@@ -4365,16 +4480,22 @@ static long cgroup_create(struct cgroup *parent, struct dentry *dentry,
        list_add_tail_rcu(&cgrp->sibling, &cgrp->parent->children);
        root->number_of_cgroups++;
-        /* each css holds a ref to the cgroup's dentry */
+        /* each css holds a ref to the cgroup's dentry and the parent css */
-        for_each_root_subsys(root, ss)
+        for_each_root_subsys(root, ss) {
+                struct cgroup_subsys_state *css = css_ar[ss->subsys_id];
                dget(dentry);
+                css_get(css->parent);
+        }
        /* hold a ref to the parent's dentry */
        dget(parent->dentry);
        /* creation succeeded, notify subsystems */
        for_each_root_subsys(root, ss) {
-                err = online_css(ss, cgrp);
+                struct cgroup_subsys_state *css = css_ar[ss->subsys_id];
+                err = online_css(css);
                if (err)
                        goto err_destroy;
@@ -4388,7 +4509,13 @@ static long cgroup_create(struct cgroup *parent, struct dentry *dentry,
                }
        }
-        err = cgroup_populate_dir(cgrp, true, root->subsys_mask);
+        idr_replace(&root->cgroup_idr, cgrp, cgrp->id);
+        err = cgroup_addrm_files(cgrp, cgroup_base_files, true);
+        if (err)
+                goto err_destroy;
+        err = cgroup_populate_dir(cgrp, root->subsys_mask);
        if (err)
                goto err_destroy;
@@ -4399,18 +4526,18 @@ static long cgroup_create(struct cgroup *parent, struct dentry *dentry,
 err_free_all:
        for_each_root_subsys(root, ss) {
-                struct cgroup_subsys_state *css = cgrp->subsys[ss->subsys_id];
+                struct cgroup_subsys_state *css = css_ar[ss->subsys_id];
                if (css) {
                        percpu_ref_cancel_init(&css->refcnt);
-                        ss->css_free(cgrp);
+                        ss->css_free(css);
                }
        }
        mutex_unlock(&cgroup_mutex);
        /* Release the reference count that we took on the superblock */
        deactivate_super(sb);
 err_free_id:
-        ida_simple_remove(&root->cgroup_ida, cgrp->id);
+        idr_remove(&root->cgroup_idr, cgrp->id);
 err_free_name:
        kfree(rcu_dereference_raw(cgrp->name));
 err_free_cgrp:
@@ -4432,22 +4559,84 @@ static int cgroup_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
        return cgroup_create(c_parent, dentry, mode | S_IFDIR);
 }
-static void cgroup_css_killed(struct cgroup *cgrp)
+/*
+ * This is called when the refcnt of a css is confirmed to be killed.
+ * css_tryget() is now guaranteed to fail.
+ */
+static void css_killed_work_fn(struct work_struct *work)
 {
-        if (!atomic_dec_and_test(&cgrp->css_kill_cnt))
+        struct cgroup_subsys_state *css =
-                return;
+                container_of(work, struct cgroup_subsys_state, destroy_work);
+        struct cgroup *cgrp = css->cgroup;
-        /* percpu ref's of all css's are killed, kick off the next step */
+        mutex_lock(&cgroup_mutex);
-        INIT_WORK(&cgrp->destroy_work, cgroup_offline_fn);
-        schedule_work(&cgrp->destroy_work);
+        /*
+         * css_tryget() is guaranteed to fail now.  Tell subsystems to
+         * initate destruction.
+         */
+        offline_css(css);
+        /*
+         * If @cgrp is marked dead, it's waiting for refs of all css's to
+         * be disabled before proceeding to the second phase of cgroup
+         * destruction.  If we are the last one, kick it off.
+         */
+        if (!cgrp->nr_css && cgroup_is_dead(cgrp))
+                cgroup_destroy_css_killed(cgrp);
+        mutex_unlock(&cgroup_mutex);
+        /*
+         * Put the css refs from kill_css().  Each css holds an extra
+         * reference to the cgroup's dentry and cgroup removal proceeds
+         * regardless of css refs.  On the last put of each css, whenever
+         * that may be, the extra dentry ref is put so that dentry
+         * destruction happens only after all css's are released.
+         */
+        css_put(css);
 }
-static void css_ref_killed_fn(struct percpu_ref *ref)
+/* css kill confirmation processing requires process context, bounce */
+static void css_killed_ref_fn(struct percpu_ref *ref)
 {
        struct cgroup_subsys_state *css =
                container_of(ref, struct cgroup_subsys_state, refcnt);
-        cgroup_css_killed(css->cgroup);
+        INIT_WORK(&css->destroy_work, css_killed_work_fn);
+        schedule_work(&css->destroy_work);
+}
+/**
+ * kill_css - destroy a css
+ * @css: css to destroy
+ *
+ * This function initiates destruction of @css by removing cgroup interface
+ * files and putting its base reference.  ->css_offline() will be invoked
+ * asynchronously once css_tryget() is guaranteed to fail and when the
+ * reference count reaches zero, @css will be released.
+ */
+static void kill_css(struct cgroup_subsys_state *css)
+{
+        cgroup_clear_dir(css->cgroup, 1 << css->ss->subsys_id);
+        /*
+         * Killing would put the base ref, but we need to keep it alive
+         * until after ->css_offline().
+         */
+        css_get(css);
+        /*
+         * cgroup core guarantees that, by the time ->css_offline() is
+         * invoked, no new css reference will be given out via
+         * css_tryget().  We can't simply call percpu_ref_kill() and
+         * proceed to offlining css's because percpu_ref_kill() doesn't
+         * guarantee that the ref is seen as killed on all CPUs on return.
+         *
+         * Use percpu_ref_kill_and_confirm() to get notifications as each
+         * css is confirmed to be seen as killed on all CPUs.
+         */
+        percpu_ref_kill_and_confirm(&css->refcnt, css_killed_ref_fn);
 }
 /**
@@ -4480,6 +4669,7 @@ static int cgroup_destroy_locked(struct cgroup *cgrp)
        struct dentry *d = cgrp->dentry;
        struct cgroup_event *event, *tmp;
        struct cgroup_subsys *ss;
+        struct cgroup *child;
        bool empty;
        lockdep_assert_held(&d->d_inode->i_mutex);
@@ -4490,47 +4680,41 @@ static int cgroup_destroy_locked(struct cgroup *cgrp)
         * @cgrp from being removed while __put_css_set() is in progress.
         */
        read_lock(&css_set_lock);
-        empty = list_empty(&cgrp->cset_links) && list_empty(&cgrp->children);
+        empty = list_empty(&cgrp->cset_links);
        read_unlock(&css_set_lock);
        if (!empty)
                return -EBUSY;
        /*
-         * Block new css_tryget() by killing css refcnts.  cgroup core
+         * Make sure there's no live children.  We can't test ->children
-         * guarantees that, by the time ->css_offline() is invoked, no new
+         * emptiness as dead children linger on it while being destroyed;
-         * css reference will be given out via css_tryget().  We can't
+         * otherwise, "rmdir parent/child parent" may fail with -EBUSY.
-         * simply call percpu_ref_kill() and proceed to offlining css's
-         * because percpu_ref_kill() doesn't guarantee that the ref is seen
-         * as killed on all CPUs on return.
-         *
-         * Use percpu_ref_kill_and_confirm() to get notifications as each
-         * css is confirmed to be seen as killed on all CPUs.  The
-         * notification callback keeps track of the number of css's to be
-         * killed and schedules cgroup_offline_fn() to perform the rest of
-         * destruction once the percpu refs of all css's are confirmed to
-         * be killed.
         */
-        atomic_set(&cgrp->css_kill_cnt, 1);
+        empty = true;
-        for_each_root_subsys(cgrp->root, ss) {
+        rcu_read_lock();
-                struct cgroup_subsys_state *css = cgrp->subsys[ss->subsys_id];
+        list_for_each_entry_rcu(child, &cgrp->children, sibling) {
+                empty = cgroup_is_dead(child);
-                /*
+                if (!empty)
-                 * Killing would put the base ref, but we need to keep it
+                        break;
-                 * alive until after ->css_offline.
-                 */
-                percpu_ref_get(&css->refcnt);
-                atomic_inc(&cgrp->css_kill_cnt);
-                percpu_ref_kill_and_confirm(&css->refcnt, css_ref_killed_fn);
        }
-        cgroup_css_killed(cgrp);
+        rcu_read_unlock();
+        if (!empty)
+                return -EBUSY;
+        /*
+         * Initiate massacre of all css's.  cgroup_destroy_css_killed()
+         * will be invoked to perform the rest of destruction once the
+         * percpu refs of all css's are confirmed to be killed.
+         */
+        for_each_root_subsys(cgrp->root, ss)
+                kill_css(cgroup_css(cgrp, ss));
        /*
         * Mark @cgrp dead.  This prevents further task migration and child
         * creation by disabling cgroup_lock_live_group().  Note that
-         * CGRP_DEAD assertion is depended upon by cgroup_next_sibling() to
+         * CGRP_DEAD assertion is depended upon by css_next_child() to
         * resume iteration after dropping RCU read lock.  See
-         * cgroup_next_sibling() for details.
+         * css_next_child() for details.
         */
        set_bit(CGRP_DEAD, &cgrp->flags);
@@ -4541,9 +4725,20 @@ static int cgroup_destroy_locked(struct cgroup *cgrp)
        raw_spin_unlock(&release_list_lock);
        /*
-         * Remove @cgrp directory.  The removal puts the base ref but we
+         * If @cgrp has css's attached, the second stage of cgroup
-         * aren't quite done with @cgrp yet, so hold onto it.
+         * destruction is kicked off from css_killed_work_fn() after the
+         * refs of all attached css's are killed.  If @cgrp doesn't have
+         * any css, we kick it off here.
         */
+        if (!cgrp->nr_css)
+                cgroup_destroy_css_killed(cgrp);
+        /*
+         * Clear the base files and remove @cgrp directory.  The removal
+         * puts the base ref but we aren't quite done with @cgrp yet, so
+         * hold onto it.
+         */
+        cgroup_addrm_files(cgrp, cgroup_base_files, false);
        dget(d);
        cgroup_d_remove_dir(d);
@@ -4563,50 +4758,36 @@ static int cgroup_destroy_locked(struct cgroup *cgrp)
 };
 /**
- * cgroup_offline_fn - the second step of cgroup destruction
+ * cgroup_destroy_css_killed - the second step of cgroup destruction
 * @work: cgroup->destroy_free_work
 *
 * This function is invoked from a work item for a cgroup which is being
- * destroyed after the percpu refcnts of all css's are guaranteed to be
+ * destroyed after all css's are offlined and performs the rest of
- * seen as killed on all CPUs, and performs the rest of destruction.  This
+ * destruction.  This is the second step of destruction described in the
- * is the second step of destruction described in the comment above
+ * comment above cgroup_destroy_locked().
- * cgroup_destroy_locked().
 */
-static void cgroup_offline_fn(struct work_struct *work)
+static void cgroup_destroy_css_killed(struct cgroup *cgrp)
 {
-        struct cgroup *cgrp = container_of(work, struct cgroup, destroy_work);
        struct cgroup *parent = cgrp->parent;
        struct dentry *d = cgrp->dentry;
-        struct cgroup_subsys *ss;
-        mutex_lock(&cgroup_mutex);
+        lockdep_assert_held(&cgroup_mutex);
-        /*
+        /* delete this cgroup from parent->children */
-         * css_tryget() is guaranteed to fail now.  Tell subsystems to
+        list_del_rcu(&cgrp->sibling);
-         * initate destruction.
-         */
-        for_each_root_subsys(cgrp->root, ss)
-                offline_css(ss, cgrp);
        /*
-         * Put the css refs from cgroup_destroy_locked().  Each css holds
+         * We should remove the cgroup object from idr before its grace
-         * an extra reference to the cgroup's dentry and cgroup removal
+         * period starts, so we won't be looking up a cgroup while the
-         * proceeds regardless of css refs.  On the last put of each css,
+         * cgroup is being freed.
-         * whenever that may be, the extra dentry ref is put so that dentry
-         * destruction happens only after all css's are released.
         */
-        for_each_root_subsys(cgrp->root, ss)
+        idr_remove(&cgrp->root->cgroup_idr, cgrp->id);
-                css_put(cgrp->subsys[ss->subsys_id]);
+        cgrp->id = -1;
-        /* delete this cgroup from parent->children */
-        list_del_rcu(&cgrp->sibling);
        dput(d);
        set_bit(CGRP_RELEASABLE, &parent->flags);
        check_for_release(parent);
-        mutex_unlock(&cgroup_mutex);
 }
 static int cgroup_rmdir(struct inode *unused_dir, struct dentry *dentry)
@@ -4629,6 +4810,11 @@ static void __init_or_module cgroup_init_cftsets(struct cgroup_subsys *ss)
         * deregistration.
         */
        if (ss->base_cftypes) {
+                struct cftype *cft;
+                for (cft = ss->base_cftypes; cft->name[0] != '\0'; cft++)
+                        cft->ss = ss;
                ss->base_cftset.cfts = ss->base_cftypes;
                list_add_tail(&ss->base_cftset.node, &ss->cftsets);
        }
@@ -4648,10 +4834,10 @@ static void __init cgroup_init_subsys(struct cgroup_subsys *ss)
        /* Create the top cgroup state for this subsystem */
        list_add(&ss->sibling, &cgroup_dummy_root.subsys_list);
        ss->root = &cgroup_dummy_root;
-        css = ss->css_alloc(cgroup_dummy_top);
+        css = ss->css_alloc(cgroup_css(cgroup_dummy_top, ss));
        /* We don't handle early failures gracefully */
        BUG_ON(IS_ERR(css));
-        init_cgroup_css(css, ss, cgroup_dummy_top);
+        init_css(css, ss, cgroup_dummy_top);
        /* Update the init_css_set to contain a subsys
         * pointer to this state - since the subsystem is
@@ -4666,7 +4852,7 @@ static void __init cgroup_init_subsys(struct cgroup_subsys *ss)
         * need to invoke fork callbacks here. */
        BUG_ON(!list_empty(&init_task.tasks));
-        BUG_ON(online_css(ss, cgroup_dummy_top));
+        BUG_ON(online_css(css));
        mutex_unlock(&cgroup_mutex);
@@ -4727,7 +4913,7 @@ int __init_or_module cgroup_load_subsys(struct cgroup_subsys *ss)
         * struct, so this can happen first (i.e. before the dummy root
         * attachment).
         */
-        css = ss->css_alloc(cgroup_dummy_top);
+        css = ss->css_alloc(cgroup_css(cgroup_dummy_top, ss));
        if (IS_ERR(css)) {
                /* failure case - need to deassign the cgroup_subsys[] slot. */
                cgroup_subsys[ss->subsys_id] = NULL;
@@ -4739,8 +4925,8 @@ int __init_or_module cgroup_load_subsys(struct cgroup_subsys *ss)
        ss->root = &cgroup_dummy_root;
        /* our new subsystem will be attached to the dummy hierarchy. */
-        init_cgroup_css(css, ss, cgroup_dummy_top);
+        init_css(css, ss, cgroup_dummy_top);
-        /* init_idr must be after init_cgroup_css because it sets css->id. */
+        /* init_idr must be after init_css() because it sets css->id. */
        if (ss->use_id) {
                ret = cgroup_init_idr(ss, css);
                if (ret)
@@ -4770,7 +4956,7 @@ int __init_or_module cgroup_load_subsys(struct cgroup_subsys *ss)
        }
        write_unlock(&css_set_lock);
-        ret = online_css(ss, cgroup_dummy_top);
+        ret = online_css(css);
        if (ret)
                goto err_unload;
@@ -4802,14 +4988,14 @@ void cgroup_unload_subsys(struct cgroup_subsys *ss)
        /*
         * we shouldn't be called if the subsystem is in use, and the use of
-         * try_module_get in parse_cgroupfs_options should ensure that it
+         * try_module_get() in rebind_subsystems() should ensure that it
         * doesn't start being used while we're killing it off.
         */
        BUG_ON(ss->root != &cgroup_dummy_root);
        mutex_lock(&cgroup_mutex);
-        offline_css(ss, cgroup_dummy_top);
+        offline_css(cgroup_css(cgroup_dummy_top, ss));
        if (ss->use_id)
                idr_destroy(&ss->idr);
@@ -4843,8 +5029,8 @@ void cgroup_unload_subsys(struct cgroup_subsys *ss)
         * the cgrp->subsys pointer to find their state. note that this
         * also takes care of freeing the css_id.
         */
-        ss->css_free(cgroup_dummy_top);
+        ss->css_free(cgroup_css(cgroup_dummy_top, ss));
-        cgroup_dummy_top->subsys[ss->subsys_id] = NULL;
+        RCU_INIT_POINTER(cgroup_dummy_top->subsys[ss->subsys_id], NULL);
        mutex_unlock(&cgroup_mutex);
 }
@@ -4926,6 +5112,10 @@ int __init cgroup_init(void)
        BUG_ON(cgroup_init_root_id(&cgroup_dummy_root, 0, 1));
+        err = idr_alloc(&cgroup_dummy_root.cgroup_idr, cgroup_dummy_top,
+                        0, 1, GFP_KERNEL);
+        BUG_ON(err < 0);
        mutex_unlock(&cgroup_root_mutex);
        mutex_unlock(&cgroup_mutex);
@@ -5082,7 +5272,7 @@ void cgroup_fork(struct task_struct *child)
 * Adds the task to the list running through its css_set if necessary and
 * call the subsystem fork() callbacks.  Has to be after the task is
 * visible on the task list in case we race with the first call to
- * cgroup_iter_start() - to guarantee that the new task ends up on its
+ * cgroup_task_iter_start() - to guarantee that the new task ends up on its
 * list.
 */
 void cgroup_post_fork(struct task_struct *child)
@@ -5195,10 +5385,10 @@ void cgroup_exit(struct task_struct *tsk, int run_callbacks)
                 */
                for_each_builtin_subsys(ss, i) {
                        if (ss->exit) {
-                                struct cgroup *old_cgrp = cset->subsys[i]->cgroup;
+                                struct cgroup_subsys_state *old_css = cset->subsys[i];
-                                struct cgroup *cgrp = task_cgroup(tsk, i);
+                                struct cgroup_subsys_state *css = task_css(tsk, i);
-                                ss->exit(cgrp, old_cgrp, tsk);
+                                ss->exit(css, old_css, tsk);
                        }
                }
        }
@@ -5457,20 +5647,16 @@ static int __init_or_module cgroup_init_idr(struct cgroup_subsys *ss,
        return 0;
 }
-static int alloc_css_id(struct cgroup_subsys *ss, struct cgroup *parent,
+static int alloc_css_id(struct cgroup_subsys_state *child_css)
-                        struct cgroup *child)
 {
-        int subsys_id, i, depth = 0;
+        struct cgroup_subsys_state *parent_css = css_parent(child_css);
-        struct cgroup_subsys_state *parent_css, *child_css;
        struct css_id *child_id, *parent_id;
+        int i, depth;
-        subsys_id = ss->subsys_id;
-        parent_css = parent->subsys[subsys_id];
-        child_css = child->subsys[subsys_id];
        parent_id = rcu_dereference_protected(parent_css->id, true);
        depth = parent_id->depth + 1;
-        child_id = get_new_cssid(ss, depth);
+        child_id = get_new_cssid(child_css->ss, depth);
        if (IS_ERR(child_id))
                return PTR_ERR(child_id);
@@ -5508,31 +5694,56 @@ struct cgroup_subsys_state *css_lookup(struct cgroup_subsys *ss, int id)
 }
 EXPORT_SYMBOL_GPL(css_lookup);
-/*
+/**
- * get corresponding css from file open on cgroupfs directory
+ * css_from_dir - get corresponding css from the dentry of a cgroup dir
+ * @dentry: directory dentry of interest
+ * @ss: subsystem of interest
+ *
+ * Must be called under RCU read lock.  The caller is responsible for
+ * pinning the returned css if it needs to be accessed outside the RCU
+ * critical section.
 */
-struct cgroup_subsys_state *cgroup_css_from_dir(struct file *f, int id)
+struct cgroup_subsys_state *css_from_dir(struct dentry *dentry,
+                                         struct cgroup_subsys *ss)
 {
        struct cgroup *cgrp;
-        struct inode *inode;
-        struct cgroup_subsys_state *css;
-        inode = file_inode(f);
+        WARN_ON_ONCE(!rcu_read_lock_held());
-        /* check in cgroup filesystem dir */
-        if (inode->i_op != &cgroup_dir_inode_operations)
+        /* is @dentry a cgroup dir? */
+        if (!dentry->d_inode ||
+            dentry->d_inode->i_op != &cgroup_dir_inode_operations)
                return ERR_PTR(-EBADF);
-        if (id < 0 || id >= CGROUP_SUBSYS_COUNT)
+        cgrp = __d_cgrp(dentry);
-                return ERR_PTR(-EINVAL);
+        return cgroup_css(cgrp, ss) ?: ERR_PTR(-ENOENT);
+}
-        /* get cgroup */
+/**
-        cgrp = __d_cgrp(f->f_dentry);
+ * css_from_id - lookup css by id
-        css = cgrp->subsys[id];
+ * @id: the cgroup id
-        return css ? css : ERR_PTR(-ENOENT);
+ * @ss: cgroup subsys to be looked into
+ *
+ * Returns the css if there's valid one with @id, otherwise returns NULL.
+ * Should be called under rcu_read_lock().
+ */
+struct cgroup_subsys_state *css_from_id(int id, struct cgroup_subsys *ss)
+{
+        struct cgroup *cgrp;
+        rcu_lockdep_assert(rcu_read_lock_held() ||
+                           lockdep_is_held(&cgroup_mutex),
+                           "css_from_id() needs proper protection");
+        cgrp = idr_find(&ss->root->cgroup_idr, id);
+        if (cgrp)
+                return cgroup_css(cgrp, ss);
+        return NULL;
 }
 #ifdef CONFIG_CGROUP_DEBUG
-static struct cgroup_subsys_state *debug_css_alloc(struct cgroup *cgrp)
+static struct cgroup_subsys_state *
+debug_css_alloc(struct cgroup_subsys_state *parent_css)
 {
        struct cgroup_subsys_state *css = kzalloc(sizeof(*css), GFP_KERNEL);
@@ -5542,22 +5753,24 @@ static struct cgroup_subsys_state *debug_css_alloc(struct cgroup *cgrp)
        return css;
 }
-static void debug_css_free(struct cgroup *cgrp)
+static void debug_css_free(struct cgroup_subsys_state *css)
 {
-        kfree(cgrp->subsys[debug_subsys_id]);
+        kfree(css);
 }
-static u64 debug_taskcount_read(struct cgroup *cgrp, struct cftype *cft)
+static u64 debug_taskcount_read(struct cgroup_subsys_state *css,
+                                struct cftype *cft)
 {
-        return cgroup_task_count(cgrp);
+        return cgroup_task_count(css->cgroup);
 }
-static u64 current_css_set_read(struct cgroup *cgrp, struct cftype *cft)
+static u64 current_css_set_read(struct cgroup_subsys_state *css,
+                                struct cftype *cft)
 {
        return (u64)(unsigned long)current->cgroups;
 }
-static u64 current_css_set_refcount_read(struct cgroup *cgrp,
+static u64 current_css_set_refcount_read(struct cgroup_subsys_state *css,
                                         struct cftype *cft)
 {
        u64 count;
@@ -5568,7 +5781,7 @@ static u64 current_css_set_refcount_read(struct cgroup *cgrp,
        return count;
 }
-static int current_css_set_cg_links_read(struct cgroup *cgrp,
+static int current_css_set_cg_links_read(struct cgroup_subsys_state *css,
                                         struct cftype *cft,
                                         struct seq_file *seq)
 {
@@ -5595,14 +5808,13 @@ static int current_css_set_cg_links_read(struct cgroup *cgrp,
 }
 #define MAX_TASKS_SHOWN_PER_CSS 25
-static int cgroup_css_links_read(struct cgroup *cgrp,
+static int cgroup_css_links_read(struct cgroup_subsys_state *css,
-                                 struct cftype *cft,
+                                 struct cftype *cft, struct seq_file *seq)
-                                 struct seq_file *seq)
 {
        struct cgrp_cset_link *link;
        read_lock(&css_set_lock);
-        list_for_each_entry(link, &cgrp->cset_links, cset_link) {
+        list_for_each_entry(link, &css->cgroup->cset_links, cset_link) {
                struct css_set *cset = link->cset;
                struct task_struct *task;
                int count = 0;
@@ -5621,9 +5833,9 @@ static int cgroup_css_links_read(struct cgroup *cgrp,
        return 0;
 }
-static u64 releasable_read(struct cgroup *cgrp, struct cftype *cft)
+static u64 releasable_read(struct cgroup_subsys_state *css, struct cftype *cft)
 {
-        return test_bit(CGRP_RELEASABLE, &cgrp->flags);
+        return test_bit(CGRP_RELEASABLE, &css->cgroup->flags);
 }
 static struct cftype debug_files[] =  {
diff --git a/kernel/cgroup_freezer.c b/kernel/cgroup_freezer.c
index 75dda1ea5026..f0ff64d0ebaa 100644
--- a/kernel/cgroup_freezer.c
+++ b/kernel/cgroup_freezer.c
@@ -45,25 +45,19 @@ struct freezer {
        spinlock_t                      lock;
 };
-static inline struct freezer *cgroup_freezer(struct cgroup *cgroup)
+static inline struct freezer *css_freezer(struct cgroup_subsys_state *css)
 {
-        return container_of(cgroup_subsys_state(cgroup, freezer_subsys_id),
+        return css ? container_of(css, struct freezer, css) : NULL;
-                            struct freezer, css);
 }
 static inline struct freezer *task_freezer(struct task_struct *task)
 {
-        return container_of(task_subsys_state(task, freezer_subsys_id),
+        return css_freezer(task_css(task, freezer_subsys_id));
-                            struct freezer, css);
 }
 static struct freezer *parent_freezer(struct freezer *freezer)
 {
-        struct cgroup *pcg = freezer->css.cgroup->parent;
+        return css_freezer(css_parent(&freezer->css));
-        if (pcg)
-                return cgroup_freezer(pcg);
-        return NULL;
 }
 bool cgroup_freezing(struct task_struct *task)
@@ -92,7 +86,8 @@ static const char *freezer_state_strs(unsigned int state)
 struct cgroup_subsys freezer_subsys;
-static struct cgroup_subsys_state *freezer_css_alloc(struct cgroup *cgroup)
+static struct cgroup_subsys_state *
+freezer_css_alloc(struct cgroup_subsys_state *parent_css)
 {
        struct freezer *freezer;
@@ -105,22 +100,22 @@ static struct cgroup_subsys_state *freezer_css_alloc(struct cgroup *cgroup)
 }
 /**
- * freezer_css_online - commit creation of a freezer cgroup
+ * freezer_css_online - commit creation of a freezer css
- * @cgroup: cgroup being created
+ * @css: css being created
 *
- * We're committing to creation of @cgroup.  Mark it online and inherit
+ * We're committing to creation of @css.  Mark it online and inherit
 * parent's freezing state while holding both parent's and our
 * freezer->lock.
 */
-static int freezer_css_online(struct cgroup *cgroup)
+static int freezer_css_online(struct cgroup_subsys_state *css)
 {
-        struct freezer *freezer = cgroup_freezer(cgroup);
+        struct freezer *freezer = css_freezer(css);
        struct freezer *parent = parent_freezer(freezer);
        /*
         * The following double locking and freezing state inheritance
         * guarantee that @cgroup can never escape ancestors' freezing
-         * states.  See cgroup_for_each_descendant_pre() for details.
+         * states.  See css_for_each_descendant_pre() for details.
         */
        if (parent)
                spin_lock_irq(&parent->lock);
@@ -141,15 +136,15 @@ static int freezer_css_online(struct cgroup *cgroup)
 }
 /**
- * freezer_css_offline - initiate destruction of @cgroup
+ * freezer_css_offline - initiate destruction of a freezer css
- * @cgroup: cgroup being destroyed
+ * @css: css being destroyed
 *
- * @cgroup is going away.  Mark it dead and decrement system_freezing_count
+ * @css is going away.  Mark it dead and decrement system_freezing_count if
- * if it was holding one.
+ * it was holding one.
 */
-static void freezer_css_offline(struct cgroup *cgroup)
+static void freezer_css_offline(struct cgroup_subsys_state *css)
 {
-        struct freezer *freezer = cgroup_freezer(cgroup);
+        struct freezer *freezer = css_freezer(css);
        spin_lock_irq(&freezer->lock);
@@ -161,9 +156,9 @@ static void freezer_css_offline(struct cgroup *cgroup)
        spin_unlock_irq(&freezer->lock);
 }
-static void freezer_css_free(struct cgroup *cgroup)
+static void freezer_css_free(struct cgroup_subsys_state *css)
 {
-        kfree(cgroup_freezer(cgroup));
+        kfree(css_freezer(css));
 }
 /*
@@ -175,25 +170,26 @@ static void freezer_css_free(struct cgroup *cgroup)
 * @freezer->lock.  freezer_attach() makes the new tasks conform to the
 * current state and all following state changes can see the new tasks.
 */
-static void freezer_attach(struct cgroup *new_cgrp, struct cgroup_taskset *tset)
+static void freezer_attach(struct cgroup_subsys_state *new_css,
+                           struct cgroup_taskset *tset)
 {
-        struct freezer *freezer = cgroup_freezer(new_cgrp);
+        struct freezer *freezer = css_freezer(new_css);
        struct task_struct *task;
        bool clear_frozen = false;
        spin_lock_irq(&freezer->lock);
        /*
-         * Make the new tasks conform to the current state of @new_cgrp.
+         * Make the new tasks conform to the current state of @new_css.
         * For simplicity, when migrating any task to a FROZEN cgroup, we
         * revert it to FREEZING and let update_if_frozen() determine the
         * correct state later.
         *
-         * Tasks in @tset are on @new_cgrp but may not conform to its
+         * Tasks in @tset are on @new_css but may not conform to its
         * current state before executing the following - !frozen tasks may
         * be visible in a FROZEN cgroup and frozen tasks in a THAWED one.
         */
-        cgroup_taskset_for_each(task, new_cgrp, tset) {
+        cgroup_taskset_for_each(task, new_css, tset) {
                if (!(freezer->state & CGROUP_FREEZING)) {
                        __thaw_task(task);
                } else {
@@ -231,7 +227,7 @@ static void freezer_fork(struct task_struct *task)
         * The root cgroup is non-freezable, so we can skip the
         * following check.
         */
-        if (!freezer->css.cgroup->parent)
+        if (!parent_freezer(freezer))
                goto out;
        spin_lock_irq(&freezer->lock);
@@ -244,7 +240,7 @@ out:
 /**
 * update_if_frozen - update whether a cgroup finished freezing
- * @cgroup: cgroup of interest
+ * @css: css of interest
 *
 * Once FREEZING is initiated, transition to FROZEN is lazily updated by
 * calling this function.  If the current state is FREEZING but not FROZEN,
@@ -255,14 +251,14 @@ out:
 * update_if_frozen() on all descendants prior to invoking this function.
 *
 * Task states and freezer state might disagree while tasks are being
- * migrated into or out of @cgroup, so we can't verify task states against
+ * migrated into or out of @css, so we can't verify task states against
 * @freezer state here.  See freezer_attach() for details.
 */
-static void update_if_frozen(struct cgroup *cgroup)
+static void update_if_frozen(struct cgroup_subsys_state *css)
 {
-        struct freezer *freezer = cgroup_freezer(cgroup);
+        struct freezer *freezer = css_freezer(css);
-        struct cgroup *pos;
+        struct cgroup_subsys_state *pos;
-        struct cgroup_iter it;
+        struct css_task_iter it;
        struct task_struct *task;
        WARN_ON_ONCE(!rcu_read_lock_held());
@@ -274,8 +270,8 @@ static void update_if_frozen(struct cgroup *cgroup)
                goto out_unlock;
        /* are all (live) children frozen? */
-        cgroup_for_each_child(pos, cgroup) {
+        css_for_each_child(pos, css) {
-                struct freezer *child = cgroup_freezer(pos);
+                struct freezer *child = css_freezer(pos);
                if ((child->state & CGROUP_FREEZER_ONLINE) &&
                    !(child->state & CGROUP_FROZEN))
@@ -283,9 +279,9 @@ static void update_if_frozen(struct cgroup *cgroup)
        }
        /* are all tasks frozen? */
-        cgroup_iter_start(cgroup, &it);
+        css_task_iter_start(css, &it);
-        while ((task = cgroup_iter_next(cgroup, &it))) {
+        while ((task = css_task_iter_next(&it))) {
                if (freezing(task)) {
                        /*
                         * freezer_should_skip() indicates that the task
@@ -300,52 +296,49 @@ static void update_if_frozen(struct cgroup *cgroup)
        freezer->state |= CGROUP_FROZEN;
 out_iter_end:
-        cgroup_iter_end(cgroup, &it);
+        css_task_iter_end(&it);
 out_unlock:
        spin_unlock_irq(&freezer->lock);
 }
-static int freezer_read(struct cgroup *cgroup, struct cftype *cft,
+static int freezer_read(struct cgroup_subsys_state *css, struct cftype *cft,
                        struct seq_file *m)
 {
-        struct cgroup *pos;
+        struct cgroup_subsys_state *pos;
        rcu_read_lock();
        /* update states bottom-up */
-        cgroup_for_each_descendant_post(pos, cgroup)
+        css_for_each_descendant_post(pos, css)
                update_if_frozen(pos);
-        update_if_frozen(cgroup);
        rcu_read_unlock();
-        seq_puts(m, freezer_state_strs(cgroup_freezer(cgroup)->state));
+        seq_puts(m, freezer_state_strs(css_freezer(css)->state));
        seq_putc(m, '\n');
        return 0;
 }
 static void freeze_cgroup(struct freezer *freezer)
 {
-        struct cgroup *cgroup = freezer->css.cgroup;
+        struct css_task_iter it;
-        struct cgroup_iter it;
        struct task_struct *task;
-        cgroup_iter_start(cgroup, &it);
+        css_task_iter_start(&freezer->css, &it);
-        while ((task = cgroup_iter_next(cgroup, &it)))
+        while ((task = css_task_iter_next(&it)))
                freeze_task(task);
-        cgroup_iter_end(cgroup, &it);
+        css_task_iter_end(&it);
 }
 static void unfreeze_cgroup(struct freezer *freezer)
 {
-        struct cgroup *cgroup = freezer->css.cgroup;
+        struct css_task_iter it;
-        struct cgroup_iter it;
        struct task_struct *task;
-        cgroup_iter_start(cgroup, &it);
+        css_task_iter_start(&freezer->css, &it);
-        while ((task = cgroup_iter_next(cgroup, &it)))
+        while ((task = css_task_iter_next(&it)))
                __thaw_task(task);
-        cgroup_iter_end(cgroup, &it);
+        css_task_iter_end(&it);
 }
 /**
@@ -395,12 +388,7 @@ static void freezer_apply_state(struct freezer *freezer, bool freeze,
 */
 static void freezer_change_state(struct freezer *freezer, bool freeze)
 {
-        struct cgroup *pos;
+        struct cgroup_subsys_state *pos;
-        /* update @freezer */
-        spin_lock_irq(&freezer->lock);
-        freezer_apply_state(freezer, freeze, CGROUP_FREEZING_SELF);
-        spin_unlock_irq(&freezer->lock);
        /*
         * Update all its descendants in pre-order traversal.  Each
@@ -408,24 +396,33 @@ static void freezer_change_state(struct freezer *freezer, bool freeze)
         * CGROUP_FREEZING_PARENT.
         */
        rcu_read_lock();
-        cgroup_for_each_descendant_pre(pos, freezer->css.cgroup) {
+        css_for_each_descendant_pre(pos, &freezer->css) {
-                struct freezer *pos_f = cgroup_freezer(pos);
+                struct freezer *pos_f = css_freezer(pos);
                struct freezer *parent = parent_freezer(pos_f);
-                /*
-                 * Our update to @parent->state is already visible which is
-                 * all we need.  No need to lock @parent.  For more info on
-                 * synchronization, see freezer_post_create().
-                 */
                spin_lock_irq(&pos_f->lock);
-                freezer_apply_state(pos_f, parent->state & CGROUP_FREEZING,
-                                    CGROUP_FREEZING_PARENT);
+                if (pos_f == freezer) {
+                        freezer_apply_state(pos_f, freeze,
+                                            CGROUP_FREEZING_SELF);
+                } else {
+                        /*
+                         * Our update to @parent->state is already visible
+                         * which is all we need.  No need to lock @parent.
+                         * For more info on synchronization, see
+                         * freezer_post_create().
+                         */
+                        freezer_apply_state(pos_f,
+                                            parent->state & CGROUP_FREEZING,
+                                            CGROUP_FREEZING_PARENT);
+                }
                spin_unlock_irq(&pos_f->lock);
        }
        rcu_read_unlock();
 }
-static int freezer_write(struct cgroup *cgroup, struct cftype *cft,
+static int freezer_write(struct cgroup_subsys_state *css, struct cftype *cft,
                         const char *buffer)
 {
        bool freeze;
@@ -437,20 +434,22 @@ static int freezer_write(struct cgroup *cgroup, struct cftype *cft,
        else
                return -EINVAL;
-        freezer_change_state(cgroup_freezer(cgroup), freeze);
+        freezer_change_state(css_freezer(css), freeze);
        return 0;
 }
-static u64 freezer_self_freezing_read(struct cgroup *cgroup, struct cftype *cft)
+static u64 freezer_self_freezing_read(struct cgroup_subsys_state *css,
+                                      struct cftype *cft)
 {
-        struct freezer *freezer = cgroup_freezer(cgroup);
+        struct freezer *freezer = css_freezer(css);
        return (bool)(freezer->state & CGROUP_FREEZING_SELF);
 }
-static u64 freezer_parent_freezing_read(struct cgroup *cgroup, struct cftype *cft)
+static u64 freezer_parent_freezing_read(struct cgroup_subsys_state *css,
+                                        struct cftype *cft)
 {
-        struct freezer *freezer = cgroup_freezer(cgroup);
+        struct freezer *freezer = css_freezer(css);
        return (bool)(freezer->state & CGROUP_FREEZING_PARENT);
 }
diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c
index 383f8231e436..247091bf0587 100644
--- a/kernel/context_tracking.c
+++ b/kernel/context_tracking.c
@@ -20,22 +20,33 @@
 #include <linux/hardirq.h>
 #include <linux/export.h>
-DEFINE_PER_CPU(struct context_tracking, context_tracking) = {
+#define CREATE_TRACE_POINTS
-#ifdef CONFIG_CONTEXT_TRACKING_FORCE
+#include <trace/events/context_tracking.h>
-        .active = true,
-#endif
+struct static_key context_tracking_enabled = STATIC_KEY_INIT_FALSE;
-};
+EXPORT_SYMBOL_GPL(context_tracking_enabled);
+DEFINE_PER_CPU(struct context_tracking, context_tracking);
+EXPORT_SYMBOL_GPL(context_tracking);
+void context_tracking_cpu_set(int cpu)
+{
+        if (!per_cpu(context_tracking.active, cpu)) {
+                per_cpu(context_tracking.active, cpu) = true;
+                static_key_slow_inc(&context_tracking_enabled);
+        }
+}
 /**
- * user_enter - Inform the context tracking that the CPU is going to
+ * context_tracking_user_enter - Inform the context tracking that the CPU is going to
- *              enter userspace mode.
+ *                               enter userspace mode.
 *
 * This function must be called right before we switch from the kernel
 * to userspace, when it's guaranteed the remaining kernel instructions
 * to execute won't use any RCU read side critical section because this
 * function sets RCU in extended quiescent state.
 */
-void user_enter(void)
+void context_tracking_user_enter(void)
 {
        unsigned long flags;
@@ -54,17 +65,32 @@ void user_enter(void)
        WARN_ON_ONCE(!current->mm);
        local_irq_save(flags);
-        if (__this_cpu_read(context_tracking.active) &&
+        if ( __this_cpu_read(context_tracking.state) != IN_USER) {
-            __this_cpu_read(context_tracking.state) != IN_USER) {
+                if (__this_cpu_read(context_tracking.active)) {
+                        trace_user_enter(0);
+                        /*
+                         * At this stage, only low level arch entry code remains and
+                         * then we'll run in userspace. We can assume there won't be
+                         * any RCU read-side critical section until the next call to
+                         * user_exit() or rcu_irq_enter(). Let's remove RCU's dependency
+                         * on the tick.
+                         */
+                        vtime_user_enter(current);
+                        rcu_user_enter();
+                }
                /*
-                 * At this stage, only low level arch entry code remains and
+                 * Even if context tracking is disabled on this CPU, because it's outside
-                 * then we'll run in userspace. We can assume there won't be
+                 * the full dynticks mask for example, we still have to keep track of the
-                 * any RCU read-side critical section until the next call to
+                 * context transitions and states to prevent inconsistency on those of
-                 * user_exit() or rcu_irq_enter(). Let's remove RCU's dependency
+                 * other CPUs.
-                 * on the tick.
+                 * If a task triggers an exception in userspace, sleep on the exception
+                 * handler and then migrate to another CPU, that new CPU must know where
+                 * the exception returns by the time we call exception_exit().
+                 * This information can only be provided by the previous CPU when it called
+                 * exception_enter().
+                 * OTOH we can spare the calls to vtime and RCU when context_tracking.active
+                 * is false because we know that CPU is not tickless.
                 */
-                vtime_user_enter(current);
-                rcu_user_enter();
                __this_cpu_write(context_tracking.state, IN_USER);
        }
        local_irq_restore(flags);
@@ -87,10 +113,9 @@ void user_enter(void)
 */
 void __sched notrace preempt_schedule_context(void)
 {
-        struct thread_info *ti = current_thread_info();
        enum ctx_state prev_ctx;
-        if (likely(ti->preempt_count || irqs_disabled()))
+        if (likely(!preemptible()))
                return;
        /*
@@ -112,8 +137,8 @@ EXPORT_SYMBOL_GPL(preempt_schedule_context);
 #endif /* CONFIG_PREEMPT */
 /**
- * user_exit - Inform the context tracking that the CPU is
+ * context_tracking_user_exit - Inform the context tracking that the CPU is
- *             exiting userspace mode and entering the kernel.
+ *                              exiting userspace mode and entering the kernel.
 *
 * This function must be called after we entered the kernel from userspace
 * before any use of RCU read side critical section. This potentially include
@@ -122,7 +147,7 @@ EXPORT_SYMBOL_GPL(preempt_schedule_context);
 * This call supports re-entrancy. This way it can be called from any exception
 * handler without needing to know if we came from userspace or not.
 */
-void user_exit(void)
+void context_tracking_user_exit(void)
 {
        unsigned long flags;
@@ -131,38 +156,22 @@ void user_exit(void)
        local_irq_save(flags);
        if (__this_cpu_read(context_tracking.state) == IN_USER) {
-                /*
+                if (__this_cpu_read(context_tracking.active)) {
-                 * We are going to run code that may use RCU. Inform
+                        /*
-                 * RCU core about that (ie: we may need the tick again).
+                         * We are going to run code that may use RCU. Inform
-                 */
+                         * RCU core about that (ie: we may need the tick again).
-                rcu_user_exit();
+                         */
-                vtime_user_exit(current);
+                        rcu_user_exit();
+                        vtime_user_exit(current);
+                        trace_user_exit(0);
+                }
                __this_cpu_write(context_tracking.state, IN_KERNEL);
        }
        local_irq_restore(flags);
 }
-void guest_enter(void)
-{
-        if (vtime_accounting_enabled())
-                vtime_guest_enter(current);
-        else
-                __guest_enter();
-}
-EXPORT_SYMBOL_GPL(guest_enter);
-void guest_exit(void)
-{
-        if (vtime_accounting_enabled())
-                vtime_guest_exit(current);
-        else
-                __guest_exit();
-}
-EXPORT_SYMBOL_GPL(guest_exit);
 /**
- * context_tracking_task_switch - context switch the syscall callbacks
+ * __context_tracking_task_switch - context switch the syscall callbacks
 * @prev: the task that is being switched out
 * @next: the task that is being switched in
 *
@@ -174,11 +183,19 @@ EXPORT_SYMBOL_GPL(guest_exit);
 * migrate to some CPU that doesn't do the context tracking. As such the TIF
 * flag may not be desired there.
 */
-void context_tracking_task_switch(struct task_struct *prev,
+void __context_tracking_task_switch(struct task_struct *prev,
-                             struct task_struct *next)
+                                    struct task_struct *next)
 {
-        if (__this_cpu_read(context_tracking.active)) {
+        clear_tsk_thread_flag(prev, TIF_NOHZ);
-                clear_tsk_thread_flag(prev, TIF_NOHZ);
+        set_tsk_thread_flag(next, TIF_NOHZ);
-                set_tsk_thread_flag(next, TIF_NOHZ);
-        }
 }
+#ifdef CONFIG_CONTEXT_TRACKING_FORCE
+void __init context_tracking_init(void)
+{
+        int cpu;
+        for_each_possible_cpu(cpu)
+                context_tracking_cpu_set(cpu);
+}
+#endif
diff --git a/kernel/cpu.c b/kernel/cpu.c
index b2b227b82123..d7f07a2da5a6 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -113,7 +113,7 @@ EXPORT_SYMBOL_GPL(put_online_cpus);
 * get_online_cpus() not an api which is called all that often.
 *
 */
-static void cpu_hotplug_begin(void)
+void cpu_hotplug_begin(void)
 {
        cpu_hotplug.active_writer = current;
@@ -127,7 +127,7 @@ static void cpu_hotplug_begin(void)
        }
 }
-static void cpu_hotplug_done(void)
+void cpu_hotplug_done(void)
 {
        cpu_hotplug.active_writer = NULL;
        mutex_unlock(&cpu_hotplug.lock);
@@ -154,10 +154,7 @@ void cpu_hotplug_enable(void)
        cpu_maps_update_done();
 }
-#else /* #if CONFIG_HOTPLUG_CPU */
+#endif  /* CONFIG_HOTPLUG_CPU */
-static void cpu_hotplug_begin(void) {}
-static void cpu_hotplug_done(void) {}
-#endif  /* #else #if CONFIG_HOTPLUG_CPU */
 /* Need to know about CPUs going up/down? */
 int __ref register_cpu_notifier(struct notifier_block *nb)
diff --git a/kernel/cpuset.c b/kernel/cpuset.c
index ea1966db34f2..6bf981e13c43 100644
--- a/kernel/cpuset.c
+++ b/kernel/cpuset.c
@@ -68,10 +68,6 @@
 */
 int number_of_cpusets __read_mostly;
-/* Forward declare cgroup structures */
-struct cgroup_subsys cpuset_subsys;
-struct cpuset;
 /* See "Frequency meter" comments, below. */
 struct fmeter {
@@ -115,27 +111,20 @@ struct cpuset {
        int relax_domain_level;
 };
-/* Retrieve the cpuset for a cgroup */
+static inline struct cpuset *css_cs(struct cgroup_subsys_state *css)
-static inline struct cpuset *cgroup_cs(struct cgroup *cgrp)
 {
-        return container_of(cgroup_subsys_state(cgrp, cpuset_subsys_id),
+        return css ? container_of(css, struct cpuset, css) : NULL;
-                            struct cpuset, css);
 }
 /* Retrieve the cpuset for a task */
 static inline struct cpuset *task_cs(struct task_struct *task)
 {
-        return container_of(task_subsys_state(task, cpuset_subsys_id),
+        return css_cs(task_css(task, cpuset_subsys_id));
-                            struct cpuset, css);
 }
-static inline struct cpuset *parent_cs(const struct cpuset *cs)
+static inline struct cpuset *parent_cs(struct cpuset *cs)
 {
-        struct cgroup *pcgrp = cs->css.cgroup->parent;
+        return css_cs(css_parent(&cs->css));
-        if (pcgrp)
-                return cgroup_cs(pcgrp);
-        return NULL;
 }
 #ifdef CONFIG_NUMA
@@ -212,29 +201,30 @@ static struct cpuset top_cpuset = {
 /**
 * cpuset_for_each_child - traverse online children of a cpuset
 * @child_cs: loop cursor pointing to the current child
- * @pos_cgrp: used for iteration
+ * @pos_css: used for iteration
 * @parent_cs: target cpuset to walk children of
 *
 * Walk @child_cs through the online children of @parent_cs.  Must be used
 * with RCU read locked.
 */
-#define cpuset_for_each_child(child_cs, pos_cgrp, parent_cs)            \
+#define cpuset_for_each_child(child_cs, pos_css, parent_cs)             \
-        cgroup_for_each_child((pos_cgrp), (parent_cs)->css.cgroup)      \
+        css_for_each_child((pos_css), &(parent_cs)->css)                \
-                if (is_cpuset_online(((child_cs) = cgroup_cs((pos_cgrp)))))
+                if (is_cpuset_online(((child_cs) = css_cs((pos_css)))))
 /**
 * cpuset_for_each_descendant_pre - pre-order walk of a cpuset's descendants
 * @des_cs: loop cursor pointing to the current descendant
- * @pos_cgrp: used for iteration
+ * @pos_css: used for iteration
 * @root_cs: target cpuset to walk ancestor of
 *
 * Walk @des_cs through the online descendants of @root_cs.  Must be used
- * with RCU read locked.  The caller may modify @pos_cgrp by calling
+ * with RCU read locked.  The caller may modify @pos_css by calling
- * cgroup_rightmost_descendant() to skip subtree.
+ * css_rightmost_descendant() to skip subtree.  @root_cs is included in the
+ * iteration and the first node to be visited.
 */
-#define cpuset_for_each_descendant_pre(des_cs, pos_cgrp, root_cs)       \
+#define cpuset_for_each_descendant_pre(des_cs, pos_css, root_cs)        \
-        cgroup_for_each_descendant_pre((pos_cgrp), (root_cs)->css.cgroup) \
+        css_for_each_descendant_pre((pos_css), &(root_cs)->css)         \
-                if (is_cpuset_online(((des_cs) = cgroup_cs((pos_cgrp)))))
+                if (is_cpuset_online(((des_cs) = css_cs((pos_css)))))
 /*
 * There are two global mutexes guarding cpuset structures - cpuset_mutex
@@ -320,8 +310,7 @@ static struct file_system_type cpuset_fs_type = {
 *
 * Call with callback_mutex held.
 */
-static void guarantee_online_cpus(const struct cpuset *cs,
+static void guarantee_online_cpus(struct cpuset *cs, struct cpumask *pmask)
-                                  struct cpumask *pmask)
 {
        while (!cpumask_intersects(cs->cpus_allowed, cpu_online_mask))
                cs = parent_cs(cs);
@@ -339,7 +328,7 @@ static void guarantee_online_cpus(const struct cpuset *cs,
 *
 * Call with callback_mutex held.
 */
-static void guarantee_online_mems(const struct cpuset *cs, nodemask_t *pmask)
+static void guarantee_online_mems(struct cpuset *cs, nodemask_t *pmask)
 {
        while (!nodes_intersects(cs->mems_allowed, node_states[N_MEMORY]))
                cs = parent_cs(cs);
@@ -384,7 +373,7 @@ static int is_cpuset_subset(const struct cpuset *p, const struct cpuset *q)
 * alloc_trial_cpuset - allocate a trial cpuset
 * @cs: the cpuset that the trial cpuset duplicates
 */
-static struct cpuset *alloc_trial_cpuset(const struct cpuset *cs)
+static struct cpuset *alloc_trial_cpuset(struct cpuset *cs)
 {
        struct cpuset *trial;
@@ -431,9 +420,9 @@ static void free_trial_cpuset(struct cpuset *trial)
 * Return 0 if valid, -errno if not.
 */
-static int validate_change(const struct cpuset *cur, const struct cpuset *trial)
+static int validate_change(struct cpuset *cur, struct cpuset *trial)
 {
-        struct cgroup *cgrp;
+        struct cgroup_subsys_state *css;
        struct cpuset *c, *par;
        int ret;
@@ -441,7 +430,7 @@ static int validate_change(const struct cpuset *cur, const struct cpuset *trial)
        /* Each of our child cpusets must be a subset of us */
        ret = -EBUSY;
-        cpuset_for_each_child(c, cgrp, cur)
+        cpuset_for_each_child(c, css, cur)
                if (!is_cpuset_subset(c, trial))
                        goto out;
@@ -462,7 +451,7 @@ static int validate_change(const struct cpuset *cur, const struct cpuset *trial)
         * overlap
         */
        ret = -EINVAL;
-        cpuset_for_each_child(c, cgrp, par) {
+        cpuset_for_each_child(c, css, par) {
                if ((is_cpu_exclusive(trial) || is_cpu_exclusive(c)) &&
                    c != cur &&
                    cpumask_intersects(trial->cpus_allowed, c->cpus_allowed))
@@ -515,13 +504,16 @@ static void update_domain_attr_tree(struct sched_domain_attr *dattr,
                                    struct cpuset *root_cs)
 {
        struct cpuset *cp;
-        struct cgroup *pos_cgrp;
+        struct cgroup_subsys_state *pos_css;
        rcu_read_lock();
-        cpuset_for_each_descendant_pre(cp, pos_cgrp, root_cs) {
+        cpuset_for_each_descendant_pre(cp, pos_css, root_cs) {
+                if (cp == root_cs)
+                        continue;
                /* skip the whole subtree if @cp doesn't have any CPU */
                if (cpumask_empty(cp->cpus_allowed)) {
-                        pos_cgrp = cgroup_rightmost_descendant(pos_cgrp);
+                        pos_css = css_rightmost_descendant(pos_css);
                        continue;
                }
@@ -596,7 +588,7 @@ static int generate_sched_domains(cpumask_var_t **domains,
        struct sched_domain_attr *dattr;  /* attributes for custom domains */
        int ndoms = 0;          /* number of sched domains in result */
        int nslot;              /* next empty doms[] struct cpumask slot */
-        struct cgroup *pos_cgrp;
+        struct cgroup_subsys_state *pos_css;
        doms = NULL;
        dattr = NULL;
@@ -625,7 +617,9 @@ static int generate_sched_domains(cpumask_var_t **domains,
        csn = 0;
        rcu_read_lock();
-        cpuset_for_each_descendant_pre(cp, pos_cgrp, &top_cpuset) {
+        cpuset_for_each_descendant_pre(cp, pos_css, &top_cpuset) {
+                if (cp == &top_cpuset)
+                        continue;
                /*
                 * Continue traversing beyond @cp iff @cp has some CPUs and
                 * isn't load balancing.  The former is obvious.  The
@@ -642,7 +636,7 @@ static int generate_sched_domains(cpumask_var_t **domains,
                        csa[csn++] = cp;
                /* skip @cp's subtree */
-                pos_cgrp = cgroup_rightmost_descendant(pos_cgrp);
+                pos_css = css_rightmost_descendant(pos_css);
        }
        rcu_read_unlock();
@@ -837,52 +831,45 @@ static struct cpuset *effective_nodemask_cpuset(struct cpuset *cs)
 /**
 * cpuset_change_cpumask - make a task's cpus_allowed the same as its cpuset's
 * @tsk: task to test
- * @scan: struct cgroup_scanner containing the cgroup of the task
+ * @data: cpuset to @tsk belongs to
 *
- * Called by cgroup_scan_tasks() for each task in a cgroup whose
+ * Called by css_scan_tasks() for each task in a cgroup whose cpus_allowed
- * cpus_allowed mask needs to be changed.
+ * mask needs to be changed.
 *
 * We don't need to re-check for the cgroup/cpuset membership, since we're
 * holding cpuset_mutex at this point.
 */
-static void cpuset_change_cpumask(struct task_struct *tsk,
+static void cpuset_change_cpumask(struct task_struct *tsk, void *data)
-                                  struct cgroup_scanner *scan)
 {
-        struct cpuset *cpus_cs;
+        struct cpuset *cs = data;
+        struct cpuset *cpus_cs = effective_cpumask_cpuset(cs);
-        cpus_cs = effective_cpumask_cpuset(cgroup_cs(scan->cg));
        set_cpus_allowed_ptr(tsk, cpus_cs->cpus_allowed);
 }
 /**
 * update_tasks_cpumask - Update the cpumasks of tasks in the cpuset.
 * @cs: the cpuset in which each task's cpus_allowed mask needs to be changed
- * @heap: if NULL, defer allocating heap memory to cgroup_scan_tasks()
+ * @heap: if NULL, defer allocating heap memory to css_scan_tasks()
 *
 * Called with cpuset_mutex held
 *
- * The cgroup_scan_tasks() function will scan all the tasks in a cgroup,
+ * The css_scan_tasks() function will scan all the tasks in a cgroup,
 * calling callback functions for each.
 *
- * No return value. It's guaranteed that cgroup_scan_tasks() always returns 0
+ * No return value. It's guaranteed that css_scan_tasks() always returns 0
 * if @heap != NULL.
 */
 static void update_tasks_cpumask(struct cpuset *cs, struct ptr_heap *heap)
 {
-        struct cgroup_scanner scan;
+        css_scan_tasks(&cs->css, NULL, cpuset_change_cpumask, cs, heap);
-        scan.cg = cs->css.cgroup;
-        scan.test_task = NULL;
-        scan.process_task = cpuset_change_cpumask;
-        scan.heap = heap;
-        cgroup_scan_tasks(&scan);
 }
 /*
 * update_tasks_cpumask_hier - Update the cpumasks of tasks in the hierarchy.
 * @root_cs: the root cpuset of the hierarchy
 * @update_root: update root cpuset or not?
- * @heap: the heap used by cgroup_scan_tasks()
+ * @heap: the heap used by css_scan_tasks()
 *
 * This will update cpumasks of tasks in @root_cs and all other empty cpusets
 * which take on cpumask of @root_cs.
@@ -893,17 +880,19 @@ static void update_tasks_cpumask_hier(struct cpuset *root_cs,
                                      bool update_root, struct ptr_heap *heap)
 {
        struct cpuset *cp;
-        struct cgroup *pos_cgrp;
+        struct cgroup_subsys_state *pos_css;
-        if (update_root)
-                update_tasks_cpumask(root_cs, heap);
        rcu_read_lock();
-        cpuset_for_each_descendant_pre(cp, pos_cgrp, root_cs) {
+        cpuset_for_each_descendant_pre(cp, pos_css, root_cs) {
-                /* skip the whole subtree if @cp have some CPU */
+                if (cp == root_cs) {
-                if (!cpumask_empty(cp->cpus_allowed)) {
+                        if (!update_root)
-                        pos_cgrp = cgroup_rightmost_descendant(pos_cgrp);
+                                continue;
-                        continue;
+                } else {
+                        /* skip the whole subtree if @cp have some CPU */
+                        if (!cpumask_empty(cp->cpus_allowed)) {
+                                pos_css = css_rightmost_descendant(pos_css);
+                                continue;
+                        }
                }
                if (!css_tryget(&cp->css))
                        continue;
@@ -1059,20 +1048,24 @@ static void cpuset_change_task_nodemask(struct task_struct *tsk,
        task_unlock(tsk);
 }
+struct cpuset_change_nodemask_arg {
+        struct cpuset           *cs;
+        nodemask_t              *newmems;
+};
 /*
 * Update task's mems_allowed and rebind its mempolicy and vmas' mempolicy
 * of it to cpuset's new mems_allowed, and migrate pages to new nodes if
 * memory_migrate flag is set. Called with cpuset_mutex held.
 */
-static void cpuset_change_nodemask(struct task_struct *p,
+static void cpuset_change_nodemask(struct task_struct *p, void *data)
-                                   struct cgroup_scanner *scan)
 {
-        struct cpuset *cs = cgroup_cs(scan->cg);
+        struct cpuset_change_nodemask_arg *arg = data;
+        struct cpuset *cs = arg->cs;
        struct mm_struct *mm;
        int migrate;
-        nodemask_t *newmems = scan->data;
-        cpuset_change_task_nodemask(p, newmems);
+        cpuset_change_task_nodemask(p, arg->newmems);
        mm = get_task_mm(p);
        if (!mm)
@@ -1082,7 +1075,7 @@ static void cpuset_change_nodemask(struct task_struct *p,
        mpol_rebind_mm(mm, &cs->mems_allowed);
        if (migrate)
-                cpuset_migrate_mm(mm, &cs->old_mems_allowed, newmems);
+                cpuset_migrate_mm(mm, &cs->old_mems_allowed, arg->newmems);
        mmput(mm);
 }
@@ -1091,28 +1084,22 @@ static void *cpuset_being_rebound;
 /**
 * update_tasks_nodemask - Update the nodemasks of tasks in the cpuset.
 * @cs: the cpuset in which each task's mems_allowed mask needs to be changed
- * @heap: if NULL, defer allocating heap memory to cgroup_scan_tasks()
+ * @heap: if NULL, defer allocating heap memory to css_scan_tasks()
 *
- * Called with cpuset_mutex held
+ * Called with cpuset_mutex held.  No return value. It's guaranteed that
- * No return value. It's guaranteed that cgroup_scan_tasks() always returns 0
+ * css_scan_tasks() always returns 0 if @heap != NULL.
- * if @heap != NULL.
 */
 static void update_tasks_nodemask(struct cpuset *cs, struct ptr_heap *heap)
 {
        static nodemask_t newmems;      /* protected by cpuset_mutex */
-        struct cgroup_scanner scan;
        struct cpuset *mems_cs = effective_nodemask_cpuset(cs);
+        struct cpuset_change_nodemask_arg arg = { .cs = cs,
+                                                  .newmems = &newmems };
        cpuset_being_rebound = cs;              /* causes mpol_dup() rebind */
        guarantee_online_mems(mems_cs, &newmems);
-        scan.cg = cs->css.cgroup;
-        scan.test_task = NULL;
-        scan.process_task = cpuset_change_nodemask;
-        scan.heap = heap;
-        scan.data = &newmems;
        /*
         * The mpol_rebind_mm() call takes mmap_sem, which we couldn't
         * take while holding tasklist_lock.  Forks can happen - the
@@ -1123,7 +1110,7 @@ static void update_tasks_nodemask(struct cpuset *cs, struct ptr_heap *heap)
         * It's ok if we rebind the same mm twice; mpol_rebind_mm()
         * is idempotent.  Also migrate pages in each mm to new nodes.
         */
-        cgroup_scan_tasks(&scan);
+        css_scan_tasks(&cs->css, NULL, cpuset_change_nodemask, &arg, heap);
        /*
         * All the tasks' nodemasks have been updated, update
@@ -1139,7 +1126,7 @@ static void update_tasks_nodemask(struct cpuset *cs, struct ptr_heap *heap)
 * update_tasks_nodemask_hier - Update the nodemasks of tasks in the hierarchy.
 * @cs: the root cpuset of the hierarchy
 * @update_root: update the root cpuset or not?
- * @heap: the heap used by cgroup_scan_tasks()
+ * @heap: the heap used by css_scan_tasks()
 *
 * This will update nodemasks of tasks in @root_cs and all other empty cpusets
 * which take on nodemask of @root_cs.
@@ -1150,17 +1137,19 @@ static void update_tasks_nodemask_hier(struct cpuset *root_cs,
                                       bool update_root, struct ptr_heap *heap)
 {
        struct cpuset *cp;
-        struct cgroup *pos_cgrp;
+        struct cgroup_subsys_state *pos_css;
-        if (update_root)
-                update_tasks_nodemask(root_cs, heap);
        rcu_read_lock();
-        cpuset_for_each_descendant_pre(cp, pos_cgrp, root_cs) {
+        cpuset_for_each_descendant_pre(cp, pos_css, root_cs) {
-                /* skip the whole subtree if @cp have some CPU */
+                if (cp == root_cs) {
-                if (!nodes_empty(cp->mems_allowed)) {
+                        if (!update_root)
-                        pos_cgrp = cgroup_rightmost_descendant(pos_cgrp);
+                                continue;
-                        continue;
+                } else {
+                        /* skip the whole subtree if @cp have some CPU */
+                        if (!nodes_empty(cp->mems_allowed)) {
+                                pos_css = css_rightmost_descendant(pos_css);
+                                continue;
+                        }
                }
                if (!css_tryget(&cp->css))
                        continue;
@@ -1267,44 +1256,39 @@ static int update_relax_domain_level(struct cpuset *cs, s64 val)
        return 0;
 }
-/*
+/**
 * cpuset_change_flag - make a task's spread flags the same as its cpuset's
 * @tsk: task to be updated
- * @scan: struct cgroup_scanner containing the cgroup of the task
+ * @data: cpuset to @tsk belongs to
 *
- * Called by cgroup_scan_tasks() for each task in a cgroup.
+ * Called by css_scan_tasks() for each task in a cgroup.
 *
 * We don't need to re-check for the cgroup/cpuset membership, since we're
 * holding cpuset_mutex at this point.
 */
-static void cpuset_change_flag(struct task_struct *tsk,
+static void cpuset_change_flag(struct task_struct *tsk, void *data)
-                                struct cgroup_scanner *scan)
 {
-        cpuset_update_task_spread_flag(cgroup_cs(scan->cg), tsk);
+        struct cpuset *cs = data;
+        cpuset_update_task_spread_flag(cs, tsk);
 }
-/*
+/**
 * update_tasks_flags - update the spread flags of tasks in the cpuset.
 * @cs: the cpuset in which each task's spread flags needs to be changed
- * @heap: if NULL, defer allocating heap memory to cgroup_scan_tasks()
+ * @heap: if NULL, defer allocating heap memory to css_scan_tasks()
 *
 * Called with cpuset_mutex held
 *
- * The cgroup_scan_tasks() function will scan all the tasks in a cgroup,
+ * The css_scan_tasks() function will scan all the tasks in a cgroup,
 * calling callback functions for each.
 *
- * No return value. It's guaranteed that cgroup_scan_tasks() always returns 0
+ * No return value. It's guaranteed that css_scan_tasks() always returns 0
 * if @heap != NULL.
 */
 static void update_tasks_flags(struct cpuset *cs, struct ptr_heap *heap)
 {
-        struct cgroup_scanner scan;
+        css_scan_tasks(&cs->css, NULL, cpuset_change_flag, cs, heap);
-        scan.cg = cs->css.cgroup;
-        scan.test_task = NULL;
-        scan.process_task = cpuset_change_flag;
-        scan.heap = heap;
-        cgroup_scan_tasks(&scan);
 }
 /*
@@ -1462,9 +1446,10 @@ static int fmeter_getrate(struct fmeter *fmp)
 }
 /* Called by cgroups to determine if a cpuset is usable; cpuset_mutex held */
-static int cpuset_can_attach(struct cgroup *cgrp, struct cgroup_taskset *tset)
+static int cpuset_can_attach(struct cgroup_subsys_state *css,
+                             struct cgroup_taskset *tset)
 {
-        struct cpuset *cs = cgroup_cs(cgrp);
+        struct cpuset *cs = css_cs(css);
        struct task_struct *task;
        int ret;
@@ -1475,11 +1460,11 @@ static int cpuset_can_attach(struct cgroup *cgrp, struct cgroup_taskset *tset)
         * flag is set.
         */
        ret = -ENOSPC;
-        if (!cgroup_sane_behavior(cgrp) &&
+        if (!cgroup_sane_behavior(css->cgroup) &&
            (cpumask_empty(cs->cpus_allowed) || nodes_empty(cs->mems_allowed)))
                goto out_unlock;
-        cgroup_taskset_for_each(task, cgrp, tset) {
+        cgroup_taskset_for_each(task, css, tset) {
                /*
                 * Kthreads which disallow setaffinity shouldn't be moved
                 * to a new cpuset; we don't want to change their cpu
@@ -1508,11 +1493,11 @@ out_unlock:
        return ret;
 }
-static void cpuset_cancel_attach(struct cgroup *cgrp,
+static void cpuset_cancel_attach(struct cgroup_subsys_state *css,
                                 struct cgroup_taskset *tset)
 {
        mutex_lock(&cpuset_mutex);
-        cgroup_cs(cgrp)->attach_in_progress--;
+        css_cs(css)->attach_in_progress--;
        mutex_unlock(&cpuset_mutex);
 }
@@ -1523,16 +1508,18 @@ static void cpuset_cancel_attach(struct cgroup *cgrp,
 */
 static cpumask_var_t cpus_attach;
-static void cpuset_attach(struct cgroup *cgrp, struct cgroup_taskset *tset)
+static void cpuset_attach(struct cgroup_subsys_state *css,
+                          struct cgroup_taskset *tset)
 {
        /* static buf protected by cpuset_mutex */
        static nodemask_t cpuset_attach_nodemask_to;
        struct mm_struct *mm;
        struct task_struct *task;
        struct task_struct *leader = cgroup_taskset_first(tset);
-        struct cgroup *oldcgrp = cgroup_taskset_cur_cgroup(tset);
+        struct cgroup_subsys_state *oldcss = cgroup_taskset_cur_css(tset,
-        struct cpuset *cs = cgroup_cs(cgrp);
+                                                        cpuset_subsys_id);
-        struct cpuset *oldcs = cgroup_cs(oldcgrp);
+        struct cpuset *cs = css_cs(css);
+        struct cpuset *oldcs = css_cs(oldcss);
        struct cpuset *cpus_cs = effective_cpumask_cpuset(cs);
        struct cpuset *mems_cs = effective_nodemask_cpuset(cs);
@@ -1546,7 +1533,7 @@ static void cpuset_attach(struct cgroup *cgrp, struct cgroup_taskset *tset)
        guarantee_online_mems(mems_cs, &cpuset_attach_nodemask_to);
-        cgroup_taskset_for_each(task, cgrp, tset) {
+        cgroup_taskset_for_each(task, css, tset) {
                /*
                 * can_attach beforehand should guarantee that this doesn't
                 * fail.  TODO: have a better way to handle failure here
@@ -1608,9 +1595,10 @@ typedef enum {
        FILE_SPREAD_SLAB,
 } cpuset_filetype_t;
-static int cpuset_write_u64(struct cgroup *cgrp, struct cftype *cft, u64 val)
+static int cpuset_write_u64(struct cgroup_subsys_state *css, struct cftype *cft,
+                            u64 val)
 {
-        struct cpuset *cs = cgroup_cs(cgrp);
+        struct cpuset *cs = css_cs(css);
        cpuset_filetype_t type = cft->private;
        int retval = 0;
@@ -1657,9 +1645,10 @@ out_unlock:
        return retval;
 }
-static int cpuset_write_s64(struct cgroup *cgrp, struct cftype *cft, s64 val)
+static int cpuset_write_s64(struct cgroup_subsys_state *css, struct cftype *cft,
+                            s64 val)
 {
-        struct cpuset *cs = cgroup_cs(cgrp);
+        struct cpuset *cs = css_cs(css);
        cpuset_filetype_t type = cft->private;
        int retval = -ENODEV;
@@ -1683,10 +1672,10 @@ out_unlock:
 /*
 * Common handling for a write to a "cpus" or "mems" file.
 */
-static int cpuset_write_resmask(struct cgroup *cgrp, struct cftype *cft,
+static int cpuset_write_resmask(struct cgroup_subsys_state *css,
-                                const char *buf)
+                                struct cftype *cft, const char *buf)
 {
-        struct cpuset *cs = cgroup_cs(cgrp);
+        struct cpuset *cs = css_cs(css);
        struct cpuset *trialcs;
        int retval = -ENODEV;
@@ -1765,13 +1754,12 @@ static size_t cpuset_sprintf_memlist(char *page, struct cpuset *cs)
        return count;
 }
-static ssize_t cpuset_common_file_read(struct cgroup *cgrp,
+static ssize_t cpuset_common_file_read(struct cgroup_subsys_state *css,
-                                       struct cftype *cft,
+                                       struct cftype *cft, struct file *file,
-                                       struct file *file,
+                                       char __user *buf, size_t nbytes,
-                                       char __user *buf,
+                                       loff_t *ppos)
-                                       size_t nbytes, loff_t *ppos)
 {
-        struct cpuset *cs = cgroup_cs(cgrp);
+        struct cpuset *cs = css_cs(css);
        cpuset_filetype_t type = cft->private;
        char *page;
        ssize_t retval = 0;
@@ -1801,9 +1789,9 @@ out:
        return retval;
 }
-static u64 cpuset_read_u64(struct cgroup *cgrp, struct cftype *cft)
+static u64 cpuset_read_u64(struct cgroup_subsys_state *css, struct cftype *cft)
 {
-        struct cpuset *cs = cgroup_cs(cgrp);
+        struct cpuset *cs = css_cs(css);
        cpuset_filetype_t type = cft->private;
        switch (type) {
        case FILE_CPU_EXCLUSIVE:
@@ -1832,9 +1820,9 @@ static u64 cpuset_read_u64(struct cgroup *cgrp, struct cftype *cft)
        return 0;
 }
-static s64 cpuset_read_s64(struct cgroup *cgrp, struct cftype *cft)
+static s64 cpuset_read_s64(struct cgroup_subsys_state *css, struct cftype *cft)
 {
-        struct cpuset *cs = cgroup_cs(cgrp);
+        struct cpuset *cs = css_cs(css);
        cpuset_filetype_t type = cft->private;
        switch (type) {
        case FILE_SCHED_RELAX_DOMAIN_LEVEL:
@@ -1949,11 +1937,12 @@ static struct cftype files[] = {
 *      cgrp:   control group that the new cpuset will be part of
 */
-static struct cgroup_subsys_state *cpuset_css_alloc(struct cgroup *cgrp)
+static struct cgroup_subsys_state *
+cpuset_css_alloc(struct cgroup_subsys_state *parent_css)
 {
        struct cpuset *cs;
-        if (!cgrp->parent)
+        if (!parent_css)
                return &top_cpuset.css;
        cs = kzalloc(sizeof(*cs), GFP_KERNEL);
@@ -1973,12 +1962,12 @@ static struct cgroup_subsys_state *cpuset_css_alloc(struct cgroup *cgrp)
        return &cs->css;
 }
-static int cpuset_css_online(struct cgroup *cgrp)
+static int cpuset_css_online(struct cgroup_subsys_state *css)
 {
-        struct cpuset *cs = cgroup_cs(cgrp);
+        struct cpuset *cs = css_cs(css);
        struct cpuset *parent = parent_cs(cs);
        struct cpuset *tmp_cs;
-        struct cgroup *pos_cg;
+        struct cgroup_subsys_state *pos_css;
        if (!parent)
                return 0;
@@ -1993,7 +1982,7 @@ static int cpuset_css_online(struct cgroup *cgrp)
        number_of_cpusets++;
-        if (!test_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags))
+        if (!test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags))
                goto out_unlock;
        /*
@@ -2010,7 +1999,7 @@ static int cpuset_css_online(struct cgroup *cgrp)
         * (and likewise for mems) to the new cgroup.
         */
        rcu_read_lock();
-        cpuset_for_each_child(tmp_cs, pos_cg, parent) {
+        cpuset_for_each_child(tmp_cs, pos_css, parent) {
                if (is_mem_exclusive(tmp_cs) || is_cpu_exclusive(tmp_cs)) {
                        rcu_read_unlock();
                        goto out_unlock;
@@ -2027,9 +2016,15 @@ out_unlock:
        return 0;
 }
-static void cpuset_css_offline(struct cgroup *cgrp)
+/*
+ * If the cpuset being removed has its flag 'sched_load_balance'
+ * enabled, then simulate turning sched_load_balance off, which
+ * will call rebuild_sched_domains_locked().
+ */
+static void cpuset_css_offline(struct cgroup_subsys_state *css)
 {
-        struct cpuset *cs = cgroup_cs(cgrp);
+        struct cpuset *cs = css_cs(css);
        mutex_lock(&cpuset_mutex);
@@ -2042,15 +2037,9 @@ static void cpuset_css_offline(struct cgroup *cgrp)
        mutex_unlock(&cpuset_mutex);
 }
-/*
+static void cpuset_css_free(struct cgroup_subsys_state *css)
- * If the cpuset being removed has its flag 'sched_load_balance'
- * enabled, then simulate turning sched_load_balance off, which
- * will call rebuild_sched_domains_locked().
- */
-static void cpuset_css_free(struct cgroup *cgrp)
 {
-        struct cpuset *cs = cgroup_cs(cgrp);
+        struct cpuset *cs = css_cs(css);
        free_cpumask_var(cs->cpus_allowed);
        kfree(cs);
@@ -2257,11 +2246,11 @@ static void cpuset_hotplug_workfn(struct work_struct *work)
        /* if cpus or mems changed, we need to propagate to descendants */
        if (cpus_updated || mems_updated) {
                struct cpuset *cs;
-                struct cgroup *pos_cgrp;
+                struct cgroup_subsys_state *pos_css;
                rcu_read_lock();
-                cpuset_for_each_descendant_pre(cs, pos_cgrp, &top_cpuset) {
+                cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) {
-                        if (!css_tryget(&cs->css))
+                        if (cs == &top_cpuset || !css_tryget(&cs->css))
                                continue;
                        rcu_read_unlock();
@@ -2350,7 +2339,7 @@ void cpuset_cpus_allowed(struct task_struct *tsk, struct cpumask *pmask)
 void cpuset_cpus_allowed_fallback(struct task_struct *tsk)
 {
-        const struct cpuset *cpus_cs;
+        struct cpuset *cpus_cs;
        rcu_read_lock();
        cpus_cs = effective_cpumask_cpuset(task_cs(tsk));
@@ -2423,7 +2412,7 @@ int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask)
 * callback_mutex.  If no ancestor is mem_exclusive or mem_hardwall
 * (an unusual configuration), then returns the root cpuset.
 */
-static const struct cpuset *nearest_hardwall_ancestor(const struct cpuset *cs)
+static struct cpuset *nearest_hardwall_ancestor(struct cpuset *cs)
 {
        while (!(is_mem_exclusive(cs) || is_mem_hardwall(cs)) && parent_cs(cs))
                cs = parent_cs(cs);
@@ -2493,7 +2482,7 @@ static const struct cpuset *nearest_hardwall_ancestor(const struct cpuset *cs)
 */
 int __cpuset_node_allowed_softwall(int node, gfp_t gfp_mask)
 {
-        const struct cpuset *cs;        /* current cpuset ancestors */
+        struct cpuset *cs;              /* current cpuset ancestors */
        int allowed;                    /* is allocation in zone z allowed? */
        if (in_interrupt() || (gfp_mask & __GFP_THISNODE))
@@ -2731,7 +2720,7 @@ int proc_cpuset_show(struct seq_file *m, void *unused_v)
                goto out_free;
        rcu_read_lock();
-        css = task_subsys_state(tsk, cpuset_subsys_id);
+        css = task_css(tsk, cpuset_subsys_id);
        retval = cgroup_path(css->cgroup, buf, PAGE_SIZE);
        rcu_read_unlock();
        if (retval < 0)
diff --git a/kernel/events/callchain.c b/kernel/events/callchain.c
index c77206184b8b..97b67df8fbfe 100644
--- a/kernel/events/callchain.c
+++ b/kernel/events/callchain.c
@@ -116,6 +116,9 @@ int get_callchain_buffers(void)
        err = alloc_callchain_buffers();
 exit:
+        if (err)
+                atomic_dec(&nr_callchain_events);
        mutex_unlock(&callchain_mutex);
        return err;
diff --git a/kernel/events/core.c b/kernel/events/core.c
index f86599e8c123..2207efc941d1 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -145,6 +145,7 @@ static DEFINE_PER_CPU(atomic_t, perf_branch_stack_events);
 static atomic_t nr_mmap_events __read_mostly;
 static atomic_t nr_comm_events __read_mostly;
 static atomic_t nr_task_events __read_mostly;
+static atomic_t nr_freq_events __read_mostly;
 static LIST_HEAD(pmus);
 static DEFINE_MUTEX(pmus_lock);
@@ -340,8 +341,8 @@ struct perf_cgroup {
 static inline struct perf_cgroup *
 perf_cgroup_from_task(struct task_struct *task)
 {
-        return container_of(task_subsys_state(task, perf_subsys_id),
+        return container_of(task_css(task, perf_subsys_id),
-                        struct perf_cgroup, css);
+                            struct perf_cgroup, css);
 }
 static inline bool
@@ -591,7 +592,9 @@ static inline int perf_cgroup_connect(int fd, struct perf_event *event,
        if (!f.file)
                return -EBADF;
-        css = cgroup_css_from_dir(f.file, perf_subsys_id);
+        rcu_read_lock();
+        css = css_from_dir(f.file->f_dentry, &perf_subsys);
        if (IS_ERR(css)) {
                ret = PTR_ERR(css);
                goto out;
@@ -617,6 +620,7 @@ static inline int perf_cgroup_connect(int fd, struct perf_event *event,
                ret = -EINVAL;
        }
 out:
+        rcu_read_unlock();
        fdput(f);
        return ret;
 }
@@ -869,12 +873,8 @@ static void perf_pmu_rotate_start(struct pmu *pmu)
        WARN_ON(!irqs_disabled());
-        if (list_empty(&cpuctx->rotation_list)) {
+        if (list_empty(&cpuctx->rotation_list))
-                int was_empty = list_empty(head);
                list_add(&cpuctx->rotation_list, head);
-                if (was_empty)
-                        tick_nohz_full_kick();
-        }
 }
 static void get_ctx(struct perf_event_context *ctx)
@@ -1216,6 +1216,9 @@ static void perf_event__id_header_size(struct perf_event *event)
        if (sample_type & PERF_SAMPLE_TIME)
                size += sizeof(data->time);
+        if (sample_type & PERF_SAMPLE_IDENTIFIER)
+                size += sizeof(data->id);
        if (sample_type & PERF_SAMPLE_ID)
                size += sizeof(data->id);
@@ -2712,7 +2715,7 @@ static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx,
                hwc = &event->hw;
-                if (needs_unthr && hwc->interrupts == MAX_INTERRUPTS) {
+                if (hwc->interrupts == MAX_INTERRUPTS) {
                        hwc->interrupts = 0;
                        perf_log_throttle(event, 1);
                        event->pmu->start(event, 0);
@@ -2811,10 +2814,11 @@ done:
 #ifdef CONFIG_NO_HZ_FULL
 bool perf_event_can_stop_tick(void)
 {
-        if (list_empty(&__get_cpu_var(rotation_list)))
+        if (atomic_read(&nr_freq_events) ||
-                return true;
+            __this_cpu_read(perf_throttled_count))
-        else
                return false;
+        else
+                return true;
 }
 #endif
@@ -3128,36 +3132,63 @@ static void free_event_rcu(struct rcu_head *head)
 static void ring_buffer_put(struct ring_buffer *rb);
 static void ring_buffer_detach(struct perf_event *event, struct ring_buffer *rb);
-static void free_event(struct perf_event *event)
+static void unaccount_event_cpu(struct perf_event *event, int cpu)
 {
-        irq_work_sync(&event->pending);
+        if (event->parent)
+                return;
+        if (has_branch_stack(event)) {
+                if (!(event->attach_state & PERF_ATTACH_TASK))
+                        atomic_dec(&per_cpu(perf_branch_stack_events, cpu));
+        }
+        if (is_cgroup_event(event))
+                atomic_dec(&per_cpu(perf_cgroup_events, cpu));
+}
+static void unaccount_event(struct perf_event *event)
+{
+        if (event->parent)
+                return;
+        if (event->attach_state & PERF_ATTACH_TASK)
+                static_key_slow_dec_deferred(&perf_sched_events);
+        if (event->attr.mmap || event->attr.mmap_data)
+                atomic_dec(&nr_mmap_events);
+        if (event->attr.comm)
+                atomic_dec(&nr_comm_events);
+        if (event->attr.task)
+                atomic_dec(&nr_task_events);
+        if (event->attr.freq)
+                atomic_dec(&nr_freq_events);
+        if (is_cgroup_event(event))
+                static_key_slow_dec_deferred(&perf_sched_events);
+        if (has_branch_stack(event))
+                static_key_slow_dec_deferred(&perf_sched_events);
+        unaccount_event_cpu(event, event->cpu);
+}
+static void __free_event(struct perf_event *event)
+{
        if (!event->parent) {
-                if (event->attach_state & PERF_ATTACH_TASK)
-                        static_key_slow_dec_deferred(&perf_sched_events);
-                if (event->attr.mmap || event->attr.mmap_data)
-                        atomic_dec(&nr_mmap_events);
-                if (event->attr.comm)
-                        atomic_dec(&nr_comm_events);
-                if (event->attr.task)
-                        atomic_dec(&nr_task_events);
                if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
                        put_callchain_buffers();
-                if (is_cgroup_event(event)) {
-                        atomic_dec(&per_cpu(perf_cgroup_events, event->cpu));
-                        static_key_slow_dec_deferred(&perf_sched_events);
-                }
-                if (has_branch_stack(event)) {
-                        static_key_slow_dec_deferred(&perf_sched_events);
-                        /* is system-wide event */
-                        if (!(event->attach_state & PERF_ATTACH_TASK)) {
-                                atomic_dec(&per_cpu(perf_branch_stack_events,
-                                                    event->cpu));
-                        }
-                }
        }
+        if (event->destroy)
+                event->destroy(event);
+        if (event->ctx)
+                put_ctx(event->ctx);
+        call_rcu(&event->rcu_head, free_event_rcu);
+}
+static void free_event(struct perf_event *event)
+{
+        irq_work_sync(&event->pending);
+        unaccount_event(event);
        if (event->rb) {
                struct ring_buffer *rb;
@@ -3180,13 +3211,8 @@ static void free_event(struct perf_event *event)
        if (is_cgroup_event(event))
                perf_detach_cgroup(event);
-        if (event->destroy)
-                event->destroy(event);
-        if (event->ctx)
-                put_ctx(event->ctx);
-        call_rcu(&event->rcu_head, free_event_rcu);
+        __free_event(event);
 }
 int perf_event_release_kernel(struct perf_event *event)
@@ -3544,6 +3570,15 @@ static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
        case PERF_EVENT_IOC_PERIOD:
                return perf_event_period(event, (u64 __user *)arg);
+        case PERF_EVENT_IOC_ID:
+        {
+                u64 id = primary_event_id(event);
+                if (copy_to_user((void __user *)arg, &id, sizeof(id)))
+                        return -EFAULT;
+                return 0;
+        }
        case PERF_EVENT_IOC_SET_OUTPUT:
        {
                int ret;
@@ -3641,6 +3676,10 @@ void perf_event_update_userpage(struct perf_event *event)
        u64 enabled, running, now;
        rcu_read_lock();
+        rb = rcu_dereference(event->rb);
+        if (!rb)
+                goto unlock;
        /*
         * compute total_time_enabled, total_time_running
         * based on snapshot values taken when the event
@@ -3651,12 +3690,8 @@ void perf_event_update_userpage(struct perf_event *event)
         * NMI context
         */
        calc_timer_values(event, &now, &enabled, &running);
-        rb = rcu_dereference(event->rb);
-        if (!rb)
-                goto unlock;
        userpg = rb->user_page;
        /*
         * Disable preemption so as to not let the corresponding user-space
         * spin too long if we get preempted.
@@ -4251,7 +4286,7 @@ static void __perf_event_header__init_id(struct perf_event_header *header,
        if (sample_type & PERF_SAMPLE_TIME)
                data->time = perf_clock();
-        if (sample_type & PERF_SAMPLE_ID)
+        if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER))
                data->id = primary_event_id(event);
        if (sample_type & PERF_SAMPLE_STREAM_ID)
@@ -4290,6 +4325,9 @@ static void __perf_event__output_id_sample(struct perf_output_handle *handle,
        if (sample_type & PERF_SAMPLE_CPU)
                perf_output_put(handle, data->cpu_entry);
+        if (sample_type & PERF_SAMPLE_IDENTIFIER)
+                perf_output_put(handle, data->id);
 }
 void perf_event__output_id_sample(struct perf_event *event,
@@ -4355,7 +4393,8 @@ static void perf_output_read_group(struct perf_output_handle *handle,
        list_for_each_entry(sub, &leader->sibling_list, group_entry) {
                n = 0;
-                if (sub != event)
+                if ((sub != event) &&
+                    (sub->state == PERF_EVENT_STATE_ACTIVE))
                        sub->pmu->read(sub);
                values[n++] = perf_event_count(sub);
@@ -4402,6 +4441,9 @@ void perf_output_sample(struct perf_output_handle *handle,
        perf_output_put(handle, *header);
+        if (sample_type & PERF_SAMPLE_IDENTIFIER)
+                perf_output_put(handle, data->id);
        if (sample_type & PERF_SAMPLE_IP)
                perf_output_put(handle, data->ip);
@@ -4462,20 +4504,6 @@ void perf_output_sample(struct perf_output_handle *handle,
                }
        }
-        if (!event->attr.watermark) {
-                int wakeup_events = event->attr.wakeup_events;
-                if (wakeup_events) {
-                        struct ring_buffer *rb = handle->rb;
-                        int events = local_inc_return(&rb->events);
-                        if (events >= wakeup_events) {
-                                local_sub(wakeup_events, &rb->events);
-                                local_inc(&rb->wakeup);
-                        }
-                }
-        }
        if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
                if (data->br_stack) {
                        size_t size;
@@ -4511,16 +4539,31 @@ void perf_output_sample(struct perf_output_handle *handle,
                }
        }
-        if (sample_type & PERF_SAMPLE_STACK_USER)
+        if (sample_type & PERF_SAMPLE_STACK_USER) {
                perf_output_sample_ustack(handle,
                                          data->stack_user_size,
                                          data->regs_user.regs);
+        }
        if (sample_type & PERF_SAMPLE_WEIGHT)
                perf_output_put(handle, data->weight);
        if (sample_type & PERF_SAMPLE_DATA_SRC)
                perf_output_put(handle, data->data_src.val);
+        if (!event->attr.watermark) {
+                int wakeup_events = event->attr.wakeup_events;
+                if (wakeup_events) {
+                        struct ring_buffer *rb = handle->rb;
+                        int events = local_inc_return(&rb->events);
+                        if (events >= wakeup_events) {
+                                local_sub(wakeup_events, &rb->events);
+                                local_inc(&rb->wakeup);
+                        }
+                }
+        }
 }
 void perf_prepare_sample(struct perf_event_header *header,
@@ -4680,12 +4723,10 @@ perf_event_read_event(struct perf_event *event,
        perf_output_end(&handle);
 }
-typedef int  (perf_event_aux_match_cb)(struct perf_event *event, void *data);
 typedef void (perf_event_aux_output_cb)(struct perf_event *event, void *data);
 static void
 perf_event_aux_ctx(struct perf_event_context *ctx,
-                   perf_event_aux_match_cb match,
                   perf_event_aux_output_cb output,
                   void *data)
 {
@@ -4696,15 +4737,12 @@ perf_event_aux_ctx(struct perf_event_context *ctx,
                        continue;
                if (!event_filter_match(event))
                        continue;
-                if (match(event, data))
+                output(event, data);
-                        output(event, data);
        }
 }
 static void
-perf_event_aux(perf_event_aux_match_cb match,
+perf_event_aux(perf_event_aux_output_cb output, void *data,
-               perf_event_aux_output_cb output,
-               void *data,
               struct perf_event_context *task_ctx)
 {
        struct perf_cpu_context *cpuctx;
@@ -4717,7 +4755,7 @@ perf_event_aux(perf_event_aux_match_cb match,
                cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
                if (cpuctx->unique_pmu != pmu)
                        goto next;
-                perf_event_aux_ctx(&cpuctx->ctx, match, output, data);
+                perf_event_aux_ctx(&cpuctx->ctx, output, data);
                if (task_ctx)
                        goto next;
                ctxn = pmu->task_ctx_nr;
@@ -4725,14 +4763,14 @@ perf_event_aux(perf_event_aux_match_cb match,
                        goto next;
                ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
                if (ctx)
-                        perf_event_aux_ctx(ctx, match, output, data);
+                        perf_event_aux_ctx(ctx, output, data);
 next:
                put_cpu_ptr(pmu->pmu_cpu_context);
        }
        if (task_ctx) {
                preempt_disable();
-                perf_event_aux_ctx(task_ctx, match, output, data);
+                perf_event_aux_ctx(task_ctx, output, data);
                preempt_enable();
        }
        rcu_read_unlock();
@@ -4741,7 +4779,7 @@ next:
 /*
 * task tracking -- fork/exit
 *
- * enabled by: attr.comm | attr.mmap | attr.mmap_data | attr.task
+ * enabled by: attr.comm | attr.mmap | attr.mmap2 | attr.mmap_data | attr.task
 */
 struct perf_task_event {
@@ -4759,6 +4797,13 @@ struct perf_task_event {
        } event_id;
 };
+static int perf_event_task_match(struct perf_event *event)
+{
+        return event->attr.comm  || event->attr.mmap ||
+               event->attr.mmap2 || event->attr.mmap_data ||
+               event->attr.task;
+}
 static void perf_event_task_output(struct perf_event *event,
                                   void *data)
 {
@@ -4768,6 +4813,9 @@ static void perf_event_task_output(struct perf_event *event,
        struct task_struct *task = task_event->task;
        int ret, size = task_event->event_id.header.size;
+        if (!perf_event_task_match(event))
+                return;
        perf_event_header__init_id(&task_event->event_id.header, &sample, event);
        ret = perf_output_begin(&handle, event,
@@ -4790,13 +4838,6 @@ out:
        task_event->event_id.header.size = size;
 }
-static int perf_event_task_match(struct perf_event *event,
-                                 void *data __maybe_unused)
-{
-        return event->attr.comm || event->attr.mmap ||
-               event->attr.mmap_data || event->attr.task;
-}
 static void perf_event_task(struct task_struct *task,
                              struct perf_event_context *task_ctx,
                              int new)
@@ -4825,8 +4866,7 @@ static void perf_event_task(struct task_struct *task,
                },
        };
-        perf_event_aux(perf_event_task_match,
+        perf_event_aux(perf_event_task_output,
-                       perf_event_task_output,
                       &task_event,
                       task_ctx);
 }
@@ -4853,6 +4893,11 @@ struct perf_comm_event {
        } event_id;
 };
+static int perf_event_comm_match(struct perf_event *event)
+{
+        return event->attr.comm;
+}
 static void perf_event_comm_output(struct perf_event *event,
                                   void *data)
 {
@@ -4862,6 +4907,9 @@ static void perf_event_comm_output(struct perf_event *event,
        int size = comm_event->event_id.header.size;
        int ret;
+        if (!perf_event_comm_match(event))
+                return;
        perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
        ret = perf_output_begin(&handle, event,
                                comm_event->event_id.header.size);
@@ -4883,12 +4931,6 @@ out:
        comm_event->event_id.header.size = size;
 }
-static int perf_event_comm_match(struct perf_event *event,
-                                 void *data __maybe_unused)
-{
-        return event->attr.comm;
-}
 static void perf_event_comm_event(struct perf_comm_event *comm_event)
 {
        char comm[TASK_COMM_LEN];
@@ -4903,8 +4945,7 @@ static void perf_event_comm_event(struct perf_comm_event *comm_event)
        comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
-        perf_event_aux(perf_event_comm_match,
+        perf_event_aux(perf_event_comm_output,
-                       perf_event_comm_output,
                       comm_event,
                       NULL);
 }
@@ -4955,6 +4996,9 @@ struct perf_mmap_event {
        const char              *file_name;
        int                     file_size;
+        int                     maj, min;
+        u64                     ino;
+        u64                     ino_generation;
        struct {
                struct perf_event_header        header;
@@ -4967,6 +5011,17 @@ struct perf_mmap_event {
        } event_id;
 };
+static int perf_event_mmap_match(struct perf_event *event,
+                                 void *data)
+{
+        struct perf_mmap_event *mmap_event = data;
+        struct vm_area_struct *vma = mmap_event->vma;
+        int executable = vma->vm_flags & VM_EXEC;
+        return (!executable && event->attr.mmap_data) ||
+               (executable && (event->attr.mmap || event->attr.mmap2));
+}
 static void perf_event_mmap_output(struct perf_event *event,
                                   void *data)
 {
@@ -4976,6 +5031,16 @@ static void perf_event_mmap_output(struct perf_event *event,
        int size = mmap_event->event_id.header.size;
        int ret;
+        if (!perf_event_mmap_match(event, data))
+                return;
+        if (event->attr.mmap2) {
+                mmap_event->event_id.header.type = PERF_RECORD_MMAP2;
+                mmap_event->event_id.header.size += sizeof(mmap_event->maj);
+                mmap_event->event_id.header.size += sizeof(mmap_event->min);
+                mmap_event->event_id.header.size += sizeof(mmap_event->ino);
+        }
        perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
        ret = perf_output_begin(&handle, event,
                                mmap_event->event_id.header.size);
@@ -4986,6 +5051,14 @@ static void perf_event_mmap_output(struct perf_event *event,
        mmap_event->event_id.tid = perf_event_tid(event, current);
        perf_output_put(&handle, mmap_event->event_id);
+        if (event->attr.mmap2) {
+                perf_output_put(&handle, mmap_event->maj);
+                perf_output_put(&handle, mmap_event->min);
+                perf_output_put(&handle, mmap_event->ino);
+                perf_output_put(&handle, mmap_event->ino_generation);
+        }
        __output_copy(&handle, mmap_event->file_name,
                                   mmap_event->file_size);
@@ -4996,21 +5069,12 @@ out:
        mmap_event->event_id.header.size = size;
 }
-static int perf_event_mmap_match(struct perf_event *event,
-                                 void *data)
-{
-        struct perf_mmap_event *mmap_event = data;
-        struct vm_area_struct *vma = mmap_event->vma;
-        int executable = vma->vm_flags & VM_EXEC;
-        return (!executable && event->attr.mmap_data) ||
-               (executable && event->attr.mmap);
-}
 static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
 {
        struct vm_area_struct *vma = mmap_event->vma;
        struct file *file = vma->vm_file;
+        int maj = 0, min = 0;
+        u64 ino = 0, gen = 0;
        unsigned int size;
        char tmp[16];
        char *buf = NULL;
@@ -5019,6 +5083,8 @@ static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
        memset(tmp, 0, sizeof(tmp));
        if (file) {
+                struct inode *inode;
+                dev_t dev;
                /*
                 * d_path works from the end of the rb backwards, so we
                 * need to add enough zero bytes after the string to handle
@@ -5034,6 +5100,13 @@ static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
                        name = strncpy(tmp, "//toolong", sizeof(tmp));
                        goto got_name;
                }
+                inode = file_inode(vma->vm_file);
+                dev = inode->i_sb->s_dev;
+                ino = inode->i_ino;
+                gen = inode->i_generation;
+                maj = MAJOR(dev);
+                min = MINOR(dev);
        } else {
                if (arch_vma_name(mmap_event->vma)) {
                        name = strncpy(tmp, arch_vma_name(mmap_event->vma),
@@ -5064,14 +5137,17 @@ got_name:
        mmap_event->file_name = name;
        mmap_event->file_size = size;
+        mmap_event->maj = maj;
+        mmap_event->min = min;
+        mmap_event->ino = ino;
+        mmap_event->ino_generation = gen;
        if (!(vma->vm_flags & VM_EXEC))
                mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA;
        mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
-        perf_event_aux(perf_event_mmap_match,
+        perf_event_aux(perf_event_mmap_output,
-                       perf_event_mmap_output,
                       mmap_event,
                       NULL);
@@ -5101,6 +5177,10 @@ void perf_event_mmap(struct vm_area_struct *vma)
                        .len    = vma->vm_end - vma->vm_start,
                        .pgoff  = (u64)vma->vm_pgoff << PAGE_SHIFT,
                },
+                /* .maj (attr_mmap2 only) */
+                /* .min (attr_mmap2 only) */
+                /* .ino (attr_mmap2 only) */
+                /* .ino_generation (attr_mmap2 only) */
        };
        perf_event_mmap_event(&mmap_event);
@@ -5178,6 +5258,7 @@ static int __perf_event_overflow(struct perf_event *event,
                        __this_cpu_inc(perf_throttled_count);
                        hwc->interrupts = MAX_INTERRUPTS;
                        perf_log_throttle(event, 0);
+                        tick_nohz_full_kick();
                        ret = 1;
                }
        }
@@ -6443,6 +6524,44 @@ unlock:
        return pmu;
 }
+static void account_event_cpu(struct perf_event *event, int cpu)
+{
+        if (event->parent)
+                return;
+        if (has_branch_stack(event)) {
+                if (!(event->attach_state & PERF_ATTACH_TASK))
+                        atomic_inc(&per_cpu(perf_branch_stack_events, cpu));
+        }
+        if (is_cgroup_event(event))
+                atomic_inc(&per_cpu(perf_cgroup_events, cpu));
+}
+static void account_event(struct perf_event *event)
+{
+        if (event->parent)
+                return;
+        if (event->attach_state & PERF_ATTACH_TASK)
+                static_key_slow_inc(&perf_sched_events.key);
+        if (event->attr.mmap || event->attr.mmap_data)
+                atomic_inc(&nr_mmap_events);
+        if (event->attr.comm)
+                atomic_inc(&nr_comm_events);
+        if (event->attr.task)
+                atomic_inc(&nr_task_events);
+        if (event->attr.freq) {
+                if (atomic_inc_return(&nr_freq_events) == 1)
+                        tick_nohz_full_kick_all();
+        }
+        if (has_branch_stack(event))
+                static_key_slow_inc(&perf_sched_events.key);
+        if (is_cgroup_event(event))
+                static_key_slow_inc(&perf_sched_events.key);
+        account_event_cpu(event, event->cpu);
+}
 /*
 * Allocate and initialize a event structure
 */
@@ -6457,7 +6576,7 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu,
        struct pmu *pmu;
        struct perf_event *event;
        struct hw_perf_event *hwc;
-        long err;
+        long err = -EINVAL;
        if ((unsigned)cpu >= nr_cpu_ids) {
                if (!task || cpu != -1)
@@ -6540,49 +6659,35 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu,
         * we currently do not support PERF_FORMAT_GROUP on inherited events
         */
        if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
-                goto done;
+                goto err_ns;
        pmu = perf_init_event(event);
-done:
-        err = 0;
        if (!pmu)
-                err = -EINVAL;
+                goto err_ns;
-        else if (IS_ERR(pmu))
+        else if (IS_ERR(pmu)) {
                err = PTR_ERR(pmu);
+                goto err_ns;
-        if (err) {
-                if (event->ns)
-                        put_pid_ns(event->ns);
-                kfree(event);
-                return ERR_PTR(err);
        }
        if (!event->parent) {
-                if (event->attach_state & PERF_ATTACH_TASK)
-                        static_key_slow_inc(&perf_sched_events.key);
-                if (event->attr.mmap || event->attr.mmap_data)
-                        atomic_inc(&nr_mmap_events);
-                if (event->attr.comm)
-                        atomic_inc(&nr_comm_events);
-                if (event->attr.task)
-                        atomic_inc(&nr_task_events);
                if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
                        err = get_callchain_buffers();
-                        if (err) {
+                        if (err)
-                                free_event(event);
+                                goto err_pmu;
-                                return ERR_PTR(err);
-                        }
-                }
-                if (has_branch_stack(event)) {
-                        static_key_slow_inc(&perf_sched_events.key);
-                        if (!(event->attach_state & PERF_ATTACH_TASK))
-                                atomic_inc(&per_cpu(perf_branch_stack_events,
-                                                    event->cpu));
                }
        }
        return event;
+err_pmu:
+        if (event->destroy)
+                event->destroy(event);
+err_ns:
+        if (event->ns)
+                put_pid_ns(event->ns);
+        kfree(event);
+        return ERR_PTR(err);
 }
 static int perf_copy_attr(struct perf_event_attr __user *uattr,
@@ -6864,17 +6969,14 @@ SYSCALL_DEFINE5(perf_event_open,
        if (flags & PERF_FLAG_PID_CGROUP) {
                err = perf_cgroup_connect(pid, event, &attr, group_leader);
-                if (err)
+                if (err) {
-                        goto err_alloc;
+                        __free_event(event);
-                /*
+                        goto err_task;
-                 * one more event:
+                }
-                 * - that has cgroup constraint on event->cpu
-                 * - that may need work on context switch
-                 */
-                atomic_inc(&per_cpu(perf_cgroup_events, event->cpu));
-                static_key_slow_inc(&perf_sched_events.key);
        }
+        account_event(event);
        /*
         * Special case software events and allow them to be part of
         * any hardware group.
@@ -7070,6 +7172,8 @@ perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
                goto err;
        }
+        account_event(event);
        ctx = find_get_context(event->pmu, task, cpu);
        if (IS_ERR(ctx)) {
                err = PTR_ERR(ctx);
@@ -7106,6 +7210,7 @@ void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu)
        list_for_each_entry_safe(event, tmp, &src_ctx->event_list,
                                 event_entry) {
                perf_remove_from_context(event);
+                unaccount_event_cpu(event, src_cpu);
                put_ctx(src_ctx);
                list_add(&event->event_entry, &events);
        }
@@ -7118,6 +7223,7 @@ void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu)
                list_del(&event->event_entry);
                if (event->state >= PERF_EVENT_STATE_OFF)
                        event->state = PERF_EVENT_STATE_INACTIVE;
+                account_event_cpu(event, dst_cpu);
                perf_install_in_context(dst_ctx, event, dst_cpu);
                get_ctx(dst_ctx);
        }
@@ -7798,7 +7904,8 @@ unlock:
 device_initcall(perf_event_sysfs_init);
 #ifdef CONFIG_CGROUP_PERF
-static struct cgroup_subsys_state *perf_cgroup_css_alloc(struct cgroup *cont)
+static struct cgroup_subsys_state *
+perf_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
 {
        struct perf_cgroup *jc;
@@ -7815,11 +7922,10 @@ static struct cgroup_subsys_state *perf_cgroup_css_alloc(struct cgroup *cont)
        return &jc->css;
 }
-static void perf_cgroup_css_free(struct cgroup *cont)
+static void perf_cgroup_css_free(struct cgroup_subsys_state *css)
 {
-        struct perf_cgroup *jc;
+        struct perf_cgroup *jc = container_of(css, struct perf_cgroup, css);
-        jc = container_of(cgroup_subsys_state(cont, perf_subsys_id),
-                          struct perf_cgroup, css);
        free_percpu(jc->info);
        kfree(jc);
 }
@@ -7831,15 +7937,17 @@ static int __perf_cgroup_move(void *info)
        return 0;
 }
-static void perf_cgroup_attach(struct cgroup *cgrp, struct cgroup_taskset *tset)
+static void perf_cgroup_attach(struct cgroup_subsys_state *css,
+                               struct cgroup_taskset *tset)
 {
        struct task_struct *task;
-        cgroup_taskset_for_each(task, cgrp, tset)
+        cgroup_taskset_for_each(task, css, tset)
                task_function_call(task, __perf_cgroup_move, task);
 }
-static void perf_cgroup_exit(struct cgroup *cgrp, struct cgroup *old_cgrp,
+static void perf_cgroup_exit(struct cgroup_subsys_state *css,
+                             struct cgroup_subsys_state *old_css,
                             struct task_struct *task)
 {
        /*
diff --git a/kernel/fork.c b/kernel/fork.c
index e23bb19e2a3e..bf46287c91a4 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -1177,7 +1177,8 @@ static struct task_struct *copy_process(unsigned long clone_flags,
         * don't allow the creation of threads.
         */
        if ((clone_flags & (CLONE_VM|CLONE_NEWPID)) &&
-            (task_active_pid_ns(current) != current->nsproxy->pid_ns))
+            (task_active_pid_ns(current) !=
+             current->nsproxy->pid_ns_for_children))
                return ERR_PTR(-EINVAL);
        retval = security_task_create(clone_flags);
@@ -1351,7 +1352,7 @@ static struct task_struct *copy_process(unsigned long clone_flags,
        if (pid != &init_struct_pid) {
                retval = -ENOMEM;
-                pid = alloc_pid(p->nsproxy->pid_ns);
+                pid = alloc_pid(p->nsproxy->pid_ns_for_children);
                if (!pid)
                        goto bad_fork_cleanup_io;
        }
diff --git a/kernel/hung_task.c b/kernel/hung_task.c
index 6df614912b9d..3e97fb126e6b 100644
--- a/kernel/hung_task.c
+++ b/kernel/hung_task.c
@@ -15,6 +15,7 @@
 #include <linux/lockdep.h>
 #include <linux/export.h>
 #include <linux/sysctl.h>
+#include <linux/utsname.h>
 /*
 * The number of tasks checked:
@@ -99,10 +100,14 @@ static void check_hung_task(struct task_struct *t, unsigned long timeout)
         * Ok, the task did not get scheduled for more than 2 minutes,
         * complain:
         */
-        printk(KERN_ERR "INFO: task %s:%d blocked for more than "
+        pr_err("INFO: task %s:%d blocked for more than %ld seconds.\n",
-                        "%ld seconds.\n", t->comm, t->pid, timeout);
+                t->comm, t->pid, timeout);
-        printk(KERN_ERR "\"echo 0 > /proc/sys/kernel/hung_task_timeout_secs\""
+        pr_err("      %s %s %.*s\n",
-                        " disables this message.\n");
+                print_tainted(), init_utsname()->release,
+                (int)strcspn(init_utsname()->version, " "),
+                init_utsname()->version);
+        pr_err("\"echo 0 > /proc/sys/kernel/hung_task_timeout_secs\""
+                " disables this message.\n");
        sched_show_task(t);
        debug_show_held_locks(t);
diff --git a/kernel/lglock.c b/kernel/lglock.c
index 6535a667a5a7..86ae2aebf004 100644
--- a/kernel/lglock.c
+++ b/kernel/lglock.c
@@ -21,7 +21,7 @@ void lg_local_lock(struct lglock *lg)
        arch_spinlock_t *lock;
        preempt_disable();
-        rwlock_acquire_read(&lg->lock_dep_map, 0, 0, _RET_IP_);
+        lock_acquire_shared(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_);
        lock = this_cpu_ptr(lg->lock);
        arch_spin_lock(lock);
 }
@@ -31,7 +31,7 @@ void lg_local_unlock(struct lglock *lg)
 {
        arch_spinlock_t *lock;
-        rwlock_release(&lg->lock_dep_map, 1, _RET_IP_);
+        lock_release(&lg->lock_dep_map, 1, _RET_IP_);
        lock = this_cpu_ptr(lg->lock);
        arch_spin_unlock(lock);
        preempt_enable();
@@ -43,7 +43,7 @@ void lg_local_lock_cpu(struct lglock *lg, int cpu)
        arch_spinlock_t *lock;
        preempt_disable();
-        rwlock_acquire_read(&lg->lock_dep_map, 0, 0, _RET_IP_);
+        lock_acquire_shared(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_);
        lock = per_cpu_ptr(lg->lock, cpu);
        arch_spin_lock(lock);
 }
@@ -53,7 +53,7 @@ void lg_local_unlock_cpu(struct lglock *lg, int cpu)
 {
        arch_spinlock_t *lock;
-        rwlock_release(&lg->lock_dep_map, 1, _RET_IP_);
+        lock_release(&lg->lock_dep_map, 1, _RET_IP_);
        lock = per_cpu_ptr(lg->lock, cpu);
        arch_spin_unlock(lock);
        preempt_enable();
@@ -65,7 +65,7 @@ void lg_global_lock(struct lglock *lg)
        int i;
        preempt_disable();
-        rwlock_acquire(&lg->lock_dep_map, 0, 0, _RET_IP_);
+        lock_acquire_exclusive(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_);
        for_each_possible_cpu(i) {
                arch_spinlock_t *lock;
                lock = per_cpu_ptr(lg->lock, i);
@@ -78,7 +78,7 @@ void lg_global_unlock(struct lglock *lg)
 {
        int i;
-        rwlock_release(&lg->lock_dep_map, 1, _RET_IP_);
+        lock_release(&lg->lock_dep_map, 1, _RET_IP_);
        for_each_possible_cpu(i) {
                arch_spinlock_t *lock;
                lock = per_cpu_ptr(lg->lock, i);
diff --git a/kernel/mutex.c b/kernel/mutex.c
index a52ee7bb830d..6d647aedffea 100644
--- a/kernel/mutex.c
+++ b/kernel/mutex.c
@@ -209,11 +209,13 @@ int mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner)
 */
 static inline int mutex_can_spin_on_owner(struct mutex *lock)
 {
+        struct task_struct *owner;
        int retval = 1;
        rcu_read_lock();
-        if (lock->owner)
+        owner = ACCESS_ONCE(lock->owner);
-                retval = lock->owner->on_cpu;
+        if (owner)
+                retval = owner->on_cpu;
        rcu_read_unlock();
        /*
         * if lock->owner is not set, the mutex owner may have just acquired
@@ -461,7 +463,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
                         * performed the optimistic spinning cannot be done.
                         */
                        if (ACCESS_ONCE(ww->ctx))
-                                break;
+                                goto slowpath;
                }
                /*
@@ -472,7 +474,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
                owner = ACCESS_ONCE(lock->owner);
                if (owner && !mutex_spin_on_owner(lock, owner)) {
                        mspin_unlock(MLOCK(lock), &node);
-                        break;
+                        goto slowpath;
                }
                if ((atomic_read(&lock->count) == 1) &&
@@ -499,7 +501,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
                 * the owner complete.
                 */
                if (!owner && (need_resched() || rt_task(task)))
-                        break;
+                        goto slowpath;
                /*
                 * The cpu_relax() call is a compiler barrier which forces
@@ -513,6 +515,10 @@ slowpath:
 #endif
        spin_lock_mutex(&lock->wait_lock, flags);
+        /* once more, can we acquire the lock? */
+        if (MUTEX_SHOW_NO_WAITER(lock) && (atomic_xchg(&lock->count, 0) == 1))
+                goto skip_wait;
        debug_mutex_lock_common(lock, &waiter);
        debug_mutex_add_waiter(lock, &waiter, task_thread_info(task));
@@ -520,9 +526,6 @@ slowpath:
        list_add_tail(&waiter.list, &lock->wait_list);
        waiter.task = task;
-        if (MUTEX_SHOW_NO_WAITER(lock) && (atomic_xchg(&lock->count, -1) == 1))
-                goto done;
        lock_contended(&lock->dep_map, ip);
        for (;;) {
@@ -536,7 +539,7 @@ slowpath:
                 * other waiters:
                 */
                if (MUTEX_SHOW_NO_WAITER(lock) &&
-                   (atomic_xchg(&lock->count, -1) == 1))
+                    (atomic_xchg(&lock->count, -1) == 1))
                        break;
                /*
@@ -561,24 +564,25 @@ slowpath:
                schedule_preempt_disabled();
                spin_lock_mutex(&lock->wait_lock, flags);
        }
+        mutex_remove_waiter(lock, &waiter, current_thread_info());
+        /* set it to 0 if there are no waiters left: */
+        if (likely(list_empty(&lock->wait_list)))
+                atomic_set(&lock->count, 0);
+        debug_mutex_free_waiter(&waiter);
-done:
+skip_wait:
+        /* got the lock - cleanup and rejoice! */
        lock_acquired(&lock->dep_map, ip);
-        /* got the lock - rejoice! */
-        mutex_remove_waiter(lock, &waiter, current_thread_info());
        mutex_set_owner(lock);
        if (!__builtin_constant_p(ww_ctx == NULL)) {
-                struct ww_mutex *ww = container_of(lock,
+                struct ww_mutex *ww = container_of(lock, struct ww_mutex, base);
-                                                      struct ww_mutex,
-                                                      base);
                struct mutex_waiter *cur;
                /*
                 * This branch gets optimized out for the common case,
                 * and is only important for ww_mutex_lock.
                 */
                ww_mutex_lock_acquired(ww, ww_ctx);
                ww->ctx = ww_ctx;
@@ -592,15 +596,8 @@ done:
                }
        }
-        /* set it to 0 if there are no waiters left: */
-        if (likely(list_empty(&lock->wait_list)))
-                atomic_set(&lock->count, 0);
        spin_unlock_mutex(&lock->wait_lock, flags);
-        debug_mutex_free_waiter(&waiter);
        preempt_enable();
        return 0;
 err:
diff --git a/kernel/nsproxy.c b/kernel/nsproxy.c
index 364ceab15f0c..997cbb951a3b 100644
--- a/kernel/nsproxy.c
+++ b/kernel/nsproxy.c
@@ -29,15 +29,15 @@
 static struct kmem_cache *nsproxy_cachep;
 struct nsproxy init_nsproxy = {
-        .count  = ATOMIC_INIT(1),
+        .count                  = ATOMIC_INIT(1),
-        .uts_ns = &init_uts_ns,
+        .uts_ns                 = &init_uts_ns,
 #if defined(CONFIG_POSIX_MQUEUE) || defined(CONFIG_SYSVIPC)
-        .ipc_ns = &init_ipc_ns,
+        .ipc_ns                 = &init_ipc_ns,
 #endif
-        .mnt_ns = NULL,
+        .mnt_ns                 = NULL,
-        .pid_ns = &init_pid_ns,
+        .pid_ns_for_children    = &init_pid_ns,
 #ifdef CONFIG_NET
-        .net_ns = &init_net,
+        .net_ns                 = &init_net,
 #endif
 };
@@ -85,9 +85,10 @@ static struct nsproxy *create_new_namespaces(unsigned long flags,
                goto out_ipc;
        }
-        new_nsp->pid_ns = copy_pid_ns(flags, user_ns, tsk->nsproxy->pid_ns);
+        new_nsp->pid_ns_for_children =
-        if (IS_ERR(new_nsp->pid_ns)) {
+                copy_pid_ns(flags, user_ns, tsk->nsproxy->pid_ns_for_children);
-                err = PTR_ERR(new_nsp->pid_ns);
+        if (IS_ERR(new_nsp->pid_ns_for_children)) {
+                err = PTR_ERR(new_nsp->pid_ns_for_children);
                goto out_pid;
        }
@@ -100,8 +101,8 @@ static struct nsproxy *create_new_namespaces(unsigned long flags,
        return new_nsp;
 out_net:
-        if (new_nsp->pid_ns)
+        if (new_nsp->pid_ns_for_children)
-                put_pid_ns(new_nsp->pid_ns);
+                put_pid_ns(new_nsp->pid_ns_for_children);
 out_pid:
        if (new_nsp->ipc_ns)
                put_ipc_ns(new_nsp->ipc_ns);
@@ -174,8 +175,8 @@ void free_nsproxy(struct nsproxy *ns)
                put_uts_ns(ns->uts_ns);
        if (ns->ipc_ns)
                put_ipc_ns(ns->ipc_ns);
-        if (ns->pid_ns)
+        if (ns->pid_ns_for_children)
-                put_pid_ns(ns->pid_ns);
+                put_pid_ns(ns->pid_ns_for_children);
        put_net(ns->net_ns);
        kmem_cache_free(nsproxy_cachep, ns);
 }
diff --git a/kernel/pid_namespace.c b/kernel/pid_namespace.c
index 6917e8edb48e..601bb361c235 100644
--- a/kernel/pid_namespace.c
+++ b/kernel/pid_namespace.c
@@ -349,8 +349,8 @@ static int pidns_install(struct nsproxy *nsproxy, void *ns)
        if (ancestor != active)
                return -EINVAL;
-        put_pid_ns(nsproxy->pid_ns);
+        put_pid_ns(nsproxy->pid_ns_for_children);
-        nsproxy->pid_ns = get_pid_ns(new);
+        nsproxy->pid_ns_for_children = get_pid_ns(new);
        return 0;
 }
diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c
index b26f5f1e773e..3085e62a80a5 100644
--- a/kernel/power/hibernate.c
+++ b/kernel/power/hibernate.c
@@ -39,7 +39,7 @@ static int resume_delay;
 static char resume_file[256] = CONFIG_PM_STD_PARTITION;
 dev_t swsusp_resume_device;
 sector_t swsusp_resume_block;
-int in_suspend __nosavedata;
+__visible int in_suspend __nosavedata;
 enum {
        HIBERNATION_INVALID,
diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c
index ece04223bb1e..62ee437b5c7e 100644
--- a/kernel/power/suspend.c
+++ b/kernel/power/suspend.c
@@ -210,6 +210,7 @@ static int suspend_enter(suspend_state_t state, bool *wakeup)
                goto Platform_wake;
        }
+        ftrace_stop();
        error = disable_nonboot_cpus();
        if (error || suspend_test(TEST_CPUS))
                goto Enable_cpus;
@@ -232,6 +233,7 @@ static int suspend_enter(suspend_state_t state, bool *wakeup)
 Enable_cpus:
        enable_nonboot_cpus();
+        ftrace_start();
 Platform_wake:
        if (need_suspend_ops(state) && suspend_ops->wake)
@@ -265,7 +267,6 @@ int suspend_devices_and_enter(suspend_state_t state)
                        goto Close;
        }
        suspend_console();
-        ftrace_stop();
        suspend_test_start();
        error = dpm_suspend_start(PMSG_SUSPEND);
        if (error) {
@@ -285,7 +286,6 @@ int suspend_devices_and_enter(suspend_state_t state)
        suspend_test_start();
        dpm_resume_end(PMSG_RESUME);
        suspend_test_finish("resume devices");
-        ftrace_start();
        resume_console();
 Close:
        if (need_suspend_ops(state) && suspend_ops->end)
diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c
index 5b5a7080e2a5..b4e8500afdb3 100644
--- a/kernel/printk/printk.c
+++ b/kernel/printk/printk.c
@@ -2226,6 +2226,13 @@ void register_console(struct console *newcon)
        struct console *bcon = NULL;
        struct console_cmdline *c;
+        if (console_drivers)
+                for_each_console(bcon)
+                        if (WARN(bcon == newcon,
+                                        "console '%s%d' already registered\n",
+                                        bcon->name, bcon->index))
+                                return;
        /*
         * before we register a new CON_BOOT console, make sure we don't
         * already have a valid console
diff --git a/kernel/rcu.h b/kernel/rcu.h
index 7f8e7590e3e5..77131966c4ad 100644
--- a/kernel/rcu.h
+++ b/kernel/rcu.h
@@ -67,12 +67,15 @@
 extern struct debug_obj_descr rcuhead_debug_descr;
-static inline void debug_rcu_head_queue(struct rcu_head *head)
+static inline int debug_rcu_head_queue(struct rcu_head *head)
 {
-        debug_object_activate(head, &rcuhead_debug_descr);
+        int r1;
+        r1 = debug_object_activate(head, &rcuhead_debug_descr);
        debug_object_active_state(head, &rcuhead_debug_descr,
                                  STATE_RCU_HEAD_READY,
                                  STATE_RCU_HEAD_QUEUED);
+        return r1;
 }
 static inline void debug_rcu_head_unqueue(struct rcu_head *head)
@@ -83,8 +86,9 @@ static inline void debug_rcu_head_unqueue(struct rcu_head *head)
        debug_object_deactivate(head, &rcuhead_debug_descr);
 }
 #else   /* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
-static inline void debug_rcu_head_queue(struct rcu_head *head)
+static inline int debug_rcu_head_queue(struct rcu_head *head)
 {
+        return 0;
 }
 static inline void debug_rcu_head_unqueue(struct rcu_head *head)
@@ -94,7 +98,7 @@ static inline void debug_rcu_head_unqueue(struct rcu_head *head)
 extern void kfree(const void *);
-static inline bool __rcu_reclaim(char *rn, struct rcu_head *head)
+static inline bool __rcu_reclaim(const char *rn, struct rcu_head *head)
 {
        unsigned long offset = (unsigned long)head->func;
diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c
index cce6ba8bbace..33eb4620aa17 100644
--- a/kernel/rcupdate.c
+++ b/kernel/rcupdate.c
@@ -212,43 +212,6 @@ static inline void debug_rcu_head_free(struct rcu_head *head)
 }
 /*
- * fixup_init is called when:
- * - an active object is initialized
- */
-static int rcuhead_fixup_init(void *addr, enum debug_obj_state state)
-{
-        struct rcu_head *head = addr;
-        switch (state) {
-        case ODEBUG_STATE_ACTIVE:
-                /*
-                 * Ensure that queued callbacks are all executed.
-                 * If we detect that we are nested in a RCU read-side critical
-                 * section, we should simply fail, otherwise we would deadlock.
-                 * In !PREEMPT configurations, there is no way to tell if we are
-                 * in a RCU read-side critical section or not, so we never
-                 * attempt any fixup and just print a warning.
-                 */
-#ifndef CONFIG_PREEMPT
-                WARN_ON_ONCE(1);
-                return 0;
-#endif
-                if (rcu_preempt_depth() != 0 || preempt_count() != 0 ||
-                    irqs_disabled()) {
-                        WARN_ON_ONCE(1);
-                        return 0;
-                }
-                rcu_barrier();
-                rcu_barrier_sched();
-                rcu_barrier_bh();
-                debug_object_init(head, &rcuhead_debug_descr);
-                return 1;
-        default:
-                return 0;
-        }
-}
-/*
 * fixup_activate is called when:
 * - an active object is activated
 * - an unknown object is activated (might be a statically initialized object)
@@ -268,69 +231,8 @@ static int rcuhead_fixup_activate(void *addr, enum debug_obj_state state)
                debug_object_init(head, &rcuhead_debug_descr);
                debug_object_activate(head, &rcuhead_debug_descr);
                return 0;
-        case ODEBUG_STATE_ACTIVE:
-                /*
-                 * Ensure that queued callbacks are all executed.
-                 * If we detect that we are nested in a RCU read-side critical
-                 * section, we should simply fail, otherwise we would deadlock.
-                 * In !PREEMPT configurations, there is no way to tell if we are
-                 * in a RCU read-side critical section or not, so we never
-                 * attempt any fixup and just print a warning.
-                 */
-#ifndef CONFIG_PREEMPT
-                WARN_ON_ONCE(1);
-                return 0;
-#endif
-                if (rcu_preempt_depth() != 0 || preempt_count() != 0 ||
-                    irqs_disabled()) {
-                        WARN_ON_ONCE(1);
-                        return 0;
-                }
-                rcu_barrier();
-                rcu_barrier_sched();
-                rcu_barrier_bh();
-                debug_object_activate(head, &rcuhead_debug_descr);
-                return 1;
        default:
-                return 0;
-        }
-}
-/*
- * fixup_free is called when:
- * - an active object is freed
- */
-static int rcuhead_fixup_free(void *addr, enum debug_obj_state state)
-{
-        struct rcu_head *head = addr;
-        switch (state) {
-        case ODEBUG_STATE_ACTIVE:
-                /*
-                 * Ensure that queued callbacks are all executed.
-                 * If we detect that we are nested in a RCU read-side critical
-                 * section, we should simply fail, otherwise we would deadlock.
-                 * In !PREEMPT configurations, there is no way to tell if we are
-                 * in a RCU read-side critical section or not, so we never
-                 * attempt any fixup and just print a warning.
-                 */
-#ifndef CONFIG_PREEMPT
-                WARN_ON_ONCE(1);
-                return 0;
-#endif
-                if (rcu_preempt_depth() != 0 || preempt_count() != 0 ||
-                    irqs_disabled()) {
-                        WARN_ON_ONCE(1);
-                        return 0;
-                }
-                rcu_barrier();
-                rcu_barrier_sched();
-                rcu_barrier_bh();
-                debug_object_free(head, &rcuhead_debug_descr);
                return 1;
-        default:
-                return 0;
        }
 }
@@ -369,15 +271,13 @@ EXPORT_SYMBOL_GPL(destroy_rcu_head_on_stack);
 struct debug_obj_descr rcuhead_debug_descr = {
        .name = "rcu_head",
-        .fixup_init = rcuhead_fixup_init,
        .fixup_activate = rcuhead_fixup_activate,
-        .fixup_free = rcuhead_fixup_free,
 };
 EXPORT_SYMBOL_GPL(rcuhead_debug_descr);
 #endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
 #if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU) || defined(CONFIG_RCU_TRACE)
-void do_trace_rcu_torture_read(char *rcutorturename, struct rcu_head *rhp,
+void do_trace_rcu_torture_read(const char *rcutorturename, struct rcu_head *rhp,
                               unsigned long secs,
                               unsigned long c_old, unsigned long c)
 {
diff --git a/kernel/rcutiny.c b/kernel/rcutiny.c
index aa344111de3e..9ed6075dc562 100644
--- a/kernel/rcutiny.c
+++ b/kernel/rcutiny.c
@@ -264,7 +264,7 @@ void rcu_check_callbacks(int cpu, int user)
 */
 static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp)
 {
-        char *rn = NULL;
+        const char *rn = NULL;
        struct rcu_head *next, *list;
        unsigned long flags;
        RCU_TRACE(int cb_count = 0);
diff --git a/kernel/rcutiny_plugin.h b/kernel/rcutiny_plugin.h
index 0cd385acccfa..280d06cae352 100644
--- a/kernel/rcutiny_plugin.h
+++ b/kernel/rcutiny_plugin.h
@@ -36,7 +36,7 @@ struct rcu_ctrlblk {
        RCU_TRACE(unsigned long gp_start); /* Start time for stalls. */
        RCU_TRACE(unsigned long ticks_this_gp); /* Statistic for stalls. */
        RCU_TRACE(unsigned long jiffies_stall); /* Jiffies at next stall. */
-        RCU_TRACE(char *name);          /* Name of RCU type. */
+        RCU_TRACE(const char *name);    /* Name of RCU type. */
 };
 /* Definition for rcupdate control block. */
diff --git a/kernel/rcutorture.c b/kernel/rcutorture.c
index f4871e52c546..be63101c6175 100644
--- a/kernel/rcutorture.c
+++ b/kernel/rcutorture.c
@@ -52,72 +52,78 @@
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and Josh Triplett <josh@freedesktop.org>");
-static int nreaders = -1;       /* # reader threads, defaults to 2*ncpus */
+static int fqs_duration;
-static int nfakewriters = 4;    /* # fake writer threads */
-static int stat_interval = 60;  /* Interval between stats, in seconds. */
-                                /*  Zero means "only at end of test". */
-static bool verbose;            /* Print more debug info. */
-static bool test_no_idle_hz = true;
-                                /* Test RCU support for tickless idle CPUs. */
-static int shuffle_interval = 3; /* Interval between shuffles (in sec)*/
-static int stutter = 5;         /* Start/stop testing interval (in sec) */
-static int irqreader = 1;       /* RCU readers from irq (timers). */
-static int fqs_duration;        /* Duration of bursts (us), 0 to disable. */
-static int fqs_holdoff;         /* Hold time within burst (us). */
-static int fqs_stutter = 3;     /* Wait time between bursts (s). */
-static int n_barrier_cbs;       /* Number of callbacks to test RCU barriers. */
-static int onoff_interval;      /* Wait time between CPU hotplugs, 0=disable. */
-static int onoff_holdoff;       /* Seconds after boot before CPU hotplugs. */
-static int shutdown_secs;       /* Shutdown time (s).  <=0 for no shutdown. */
-static int stall_cpu;           /* CPU-stall duration (s).  0 for no stall. */
-static int stall_cpu_holdoff = 10; /* Time to wait until stall (s).  */
-static int test_boost = 1;      /* Test RCU prio boost: 0=no, 1=maybe, 2=yes. */
-static int test_boost_interval = 7; /* Interval between boost tests, seconds. */
-static int test_boost_duration = 4; /* Duration of each boost test, seconds. */
-static char *torture_type = "rcu"; /* What RCU implementation to torture. */
-module_param(nreaders, int, 0444);
-MODULE_PARM_DESC(nreaders, "Number of RCU reader threads");
-module_param(nfakewriters, int, 0444);
-MODULE_PARM_DESC(nfakewriters, "Number of RCU fake writer threads");
-module_param(stat_interval, int, 0644);
-MODULE_PARM_DESC(stat_interval, "Number of seconds between stats printk()s");
-module_param(verbose, bool, 0444);
-MODULE_PARM_DESC(verbose, "Enable verbose debugging printk()s");
-module_param(test_no_idle_hz, bool, 0444);
-MODULE_PARM_DESC(test_no_idle_hz, "Test support for tickless idle CPUs");
-module_param(shuffle_interval, int, 0444);
-MODULE_PARM_DESC(shuffle_interval, "Number of seconds between shuffles");
-module_param(stutter, int, 0444);
-MODULE_PARM_DESC(stutter, "Number of seconds to run/halt test");
-module_param(irqreader, int, 0444);
-MODULE_PARM_DESC(irqreader, "Allow RCU readers from irq handlers");
 module_param(fqs_duration, int, 0444);
-MODULE_PARM_DESC(fqs_duration, "Duration of fqs bursts (us)");
+MODULE_PARM_DESC(fqs_duration, "Duration of fqs bursts (us), 0 to disable");
+static int fqs_holdoff;
 module_param(fqs_holdoff, int, 0444);
 MODULE_PARM_DESC(fqs_holdoff, "Holdoff time within fqs bursts (us)");
+static int fqs_stutter = 3;
 module_param(fqs_stutter, int, 0444);
 MODULE_PARM_DESC(fqs_stutter, "Wait time between fqs bursts (s)");
+static bool gp_exp;
+module_param(gp_exp, bool, 0444);
+MODULE_PARM_DESC(gp_exp, "Use expedited GP wait primitives");
+static bool gp_normal;
+module_param(gp_normal, bool, 0444);
+MODULE_PARM_DESC(gp_normal, "Use normal (non-expedited) GP wait primitives");
+static int irqreader = 1;
+module_param(irqreader, int, 0444);
+MODULE_PARM_DESC(irqreader, "Allow RCU readers from irq handlers");
+static int n_barrier_cbs;
 module_param(n_barrier_cbs, int, 0444);
 MODULE_PARM_DESC(n_barrier_cbs, "# of callbacks/kthreads for barrier testing");
-module_param(onoff_interval, int, 0444);
+static int nfakewriters = 4;
-MODULE_PARM_DESC(onoff_interval, "Time between CPU hotplugs (s), 0=disable");
+module_param(nfakewriters, int, 0444);
+MODULE_PARM_DESC(nfakewriters, "Number of RCU fake writer threads");
+static int nreaders = -1;
+module_param(nreaders, int, 0444);
+MODULE_PARM_DESC(nreaders, "Number of RCU reader threads");
+static int object_debug;
+module_param(object_debug, int, 0444);
+MODULE_PARM_DESC(object_debug, "Enable debug-object double call_rcu() testing");
+static int onoff_holdoff;
 module_param(onoff_holdoff, int, 0444);
 MODULE_PARM_DESC(onoff_holdoff, "Time after boot before CPU hotplugs (s)");
+static int onoff_interval;
+module_param(onoff_interval, int, 0444);
+MODULE_PARM_DESC(onoff_interval, "Time between CPU hotplugs (s), 0=disable");
+static int shuffle_interval = 3;
+module_param(shuffle_interval, int, 0444);
+MODULE_PARM_DESC(shuffle_interval, "Number of seconds between shuffles");
+static int shutdown_secs;
 module_param(shutdown_secs, int, 0444);
-MODULE_PARM_DESC(shutdown_secs, "Shutdown time (s), zero to disable.");
+MODULE_PARM_DESC(shutdown_secs, "Shutdown time (s), <= zero to disable.");
+static int stall_cpu;
 module_param(stall_cpu, int, 0444);
 MODULE_PARM_DESC(stall_cpu, "Stall duration (s), zero to disable.");
+static int stall_cpu_holdoff = 10;
 module_param(stall_cpu_holdoff, int, 0444);
 MODULE_PARM_DESC(stall_cpu_holdoff, "Time to wait before starting stall (s).");
+static int stat_interval = 60;
+module_param(stat_interval, int, 0644);
+MODULE_PARM_DESC(stat_interval, "Number of seconds between stats printk()s");
+static int stutter = 5;
+module_param(stutter, int, 0444);
+MODULE_PARM_DESC(stutter, "Number of seconds to run/halt test");
+static int test_boost = 1;
 module_param(test_boost, int, 0444);
 MODULE_PARM_DESC(test_boost, "Test RCU prio boost: 0=no, 1=maybe, 2=yes.");
-module_param(test_boost_interval, int, 0444);
+static int test_boost_duration = 4;
-MODULE_PARM_DESC(test_boost_interval, "Interval between boost tests, seconds.");
 module_param(test_boost_duration, int, 0444);
 MODULE_PARM_DESC(test_boost_duration, "Duration of each boost test, seconds.");
+static int test_boost_interval = 7;
+module_param(test_boost_interval, int, 0444);
+MODULE_PARM_DESC(test_boost_interval, "Interval between boost tests, seconds.");
+static bool test_no_idle_hz = true;
+module_param(test_no_idle_hz, bool, 0444);
+MODULE_PARM_DESC(test_no_idle_hz, "Test support for tickless idle CPUs");
+static char *torture_type = "rcu";
 module_param(torture_type, charp, 0444);
-MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, srcu)");
+MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, ...)");
+static bool verbose;
+module_param(verbose, bool, 0444);
+MODULE_PARM_DESC(verbose, "Enable verbose debugging printk()s");
 #define TORTURE_FLAG "-torture:"
 #define PRINTK_STRING(s) \
@@ -267,7 +273,7 @@ rcutorture_shutdown_notify(struct notifier_block *unused1,
 * Absorb kthreads into a kernel function that won't return, so that
 * they won't ever access module text or data again.
 */
-static void rcutorture_shutdown_absorb(char *title)
+static void rcutorture_shutdown_absorb(const char *title)
 {
        if (ACCESS_ONCE(fullstop) == FULLSTOP_SHUTDOWN) {
                pr_notice(
@@ -337,7 +343,7 @@ rcu_random(struct rcu_random_state *rrsp)
 }
 static void
-rcu_stutter_wait(char *title)
+rcu_stutter_wait(const char *title)
 {
        while (stutter_pause_test || !rcutorture_runnable) {
                if (rcutorture_runnable)
@@ -360,13 +366,14 @@ struct rcu_torture_ops {
        int (*completed)(void);
        void (*deferred_free)(struct rcu_torture *p);
        void (*sync)(void);
+        void (*exp_sync)(void);
        void (*call)(struct rcu_head *head, void (*func)(struct rcu_head *rcu));
        void (*cb_barrier)(void);
        void (*fqs)(void);
        int (*stats)(char *page);
        int irq_capable;
        int can_boost;
-        char *name;
+        const char *name;
 };
 static struct rcu_torture_ops *cur_ops;
@@ -443,81 +450,27 @@ static void rcu_torture_deferred_free(struct rcu_torture *p)
        call_rcu(&p->rtort_rcu, rcu_torture_cb);
 }
-static struct rcu_torture_ops rcu_ops = {
-        .init           = NULL,
-        .readlock       = rcu_torture_read_lock,
-        .read_delay     = rcu_read_delay,
-        .readunlock     = rcu_torture_read_unlock,
-        .completed      = rcu_torture_completed,
-        .deferred_free  = rcu_torture_deferred_free,
-        .sync           = synchronize_rcu,
-        .call           = call_rcu,
-        .cb_barrier     = rcu_barrier,
-        .fqs            = rcu_force_quiescent_state,
-        .stats          = NULL,
-        .irq_capable    = 1,
-        .can_boost      = rcu_can_boost(),
-        .name           = "rcu"
-};
-static void rcu_sync_torture_deferred_free(struct rcu_torture *p)
-{
-        int i;
-        struct rcu_torture *rp;
-        struct rcu_torture *rp1;
-        cur_ops->sync();
-        list_add(&p->rtort_free, &rcu_torture_removed);
-        list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) {
-                i = rp->rtort_pipe_count;
-                if (i > RCU_TORTURE_PIPE_LEN)
-                        i = RCU_TORTURE_PIPE_LEN;
-                atomic_inc(&rcu_torture_wcount[i]);
-                if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
-                        rp->rtort_mbtest = 0;
-                        list_del(&rp->rtort_free);
-                        rcu_torture_free(rp);
-                }
-        }
-}
 static void rcu_sync_torture_init(void)
 {
        INIT_LIST_HEAD(&rcu_torture_removed);
 }
-static struct rcu_torture_ops rcu_sync_ops = {
+static struct rcu_torture_ops rcu_ops = {
        .init           = rcu_sync_torture_init,
        .readlock       = rcu_torture_read_lock,
        .read_delay     = rcu_read_delay,
        .readunlock     = rcu_torture_read_unlock,
        .completed      = rcu_torture_completed,
-        .deferred_free  = rcu_sync_torture_deferred_free,
+        .deferred_free  = rcu_torture_deferred_free,
        .sync           = synchronize_rcu,
-        .call           = NULL,
+        .exp_sync       = synchronize_rcu_expedited,
-        .cb_barrier     = NULL,
+        .call           = call_rcu,
-        .fqs            = rcu_force_quiescent_state,
+        .cb_barrier     = rcu_barrier,
-        .stats          = NULL,
-        .irq_capable    = 1,
-        .can_boost      = rcu_can_boost(),
-        .name           = "rcu_sync"
-};
-static struct rcu_torture_ops rcu_expedited_ops = {
-        .init           = rcu_sync_torture_init,
-        .readlock       = rcu_torture_read_lock,
-        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
-        .readunlock     = rcu_torture_read_unlock,
-        .completed      = rcu_no_completed,
-        .deferred_free  = rcu_sync_torture_deferred_free,
-        .sync           = synchronize_rcu_expedited,
-        .call           = NULL,
-        .cb_barrier     = NULL,
        .fqs            = rcu_force_quiescent_state,
        .stats          = NULL,
        .irq_capable    = 1,
        .can_boost      = rcu_can_boost(),
-        .name           = "rcu_expedited"
+        .name           = "rcu"
 };
 /*
@@ -546,13 +499,14 @@ static void rcu_bh_torture_deferred_free(struct rcu_torture *p)
 }
 static struct rcu_torture_ops rcu_bh_ops = {
-        .init           = NULL,
+        .init           = rcu_sync_torture_init,
        .readlock       = rcu_bh_torture_read_lock,
        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
        .readunlock     = rcu_bh_torture_read_unlock,
        .completed      = rcu_bh_torture_completed,
        .deferred_free  = rcu_bh_torture_deferred_free,
        .sync           = synchronize_rcu_bh,
+        .exp_sync       = synchronize_rcu_bh_expedited,
        .call           = call_rcu_bh,
        .cb_barrier     = rcu_barrier_bh,
        .fqs            = rcu_bh_force_quiescent_state,
@@ -561,38 +515,6 @@ static struct rcu_torture_ops rcu_bh_ops = {
        .name           = "rcu_bh"
 };
-static struct rcu_torture_ops rcu_bh_sync_ops = {
-        .init           = rcu_sync_torture_init,
-        .readlock       = rcu_bh_torture_read_lock,
-        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
-        .readunlock     = rcu_bh_torture_read_unlock,
-        .completed      = rcu_bh_torture_completed,
-        .deferred_free  = rcu_sync_torture_deferred_free,
-        .sync           = synchronize_rcu_bh,
-        .call           = NULL,
-        .cb_barrier     = NULL,
-        .fqs            = rcu_bh_force_quiescent_state,
-        .stats          = NULL,
-        .irq_capable    = 1,
-        .name           = "rcu_bh_sync"
-};
-static struct rcu_torture_ops rcu_bh_expedited_ops = {
-        .init           = rcu_sync_torture_init,
-        .readlock       = rcu_bh_torture_read_lock,
-        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
-        .readunlock     = rcu_bh_torture_read_unlock,
-        .completed      = rcu_bh_torture_completed,
-        .deferred_free  = rcu_sync_torture_deferred_free,
-        .sync           = synchronize_rcu_bh_expedited,
-        .call           = NULL,
-        .cb_barrier     = NULL,
-        .fqs            = rcu_bh_force_quiescent_state,
-        .stats          = NULL,
-        .irq_capable    = 1,
-        .name           = "rcu_bh_expedited"
-};
 /*
 * Definitions for srcu torture testing.
 */
@@ -667,6 +589,11 @@ static int srcu_torture_stats(char *page)
        return cnt;
 }
+static void srcu_torture_synchronize_expedited(void)
+{
+        synchronize_srcu_expedited(&srcu_ctl);
+}
 static struct rcu_torture_ops srcu_ops = {
        .init           = rcu_sync_torture_init,
        .readlock       = srcu_torture_read_lock,
@@ -675,45 +602,13 @@ static struct rcu_torture_ops srcu_ops = {
        .completed      = srcu_torture_completed,
        .deferred_free  = srcu_torture_deferred_free,
        .sync           = srcu_torture_synchronize,
+        .exp_sync       = srcu_torture_synchronize_expedited,
        .call           = srcu_torture_call,
        .cb_barrier     = srcu_torture_barrier,
        .stats          = srcu_torture_stats,
        .name           = "srcu"
 };
-static struct rcu_torture_ops srcu_sync_ops = {
-        .init           = rcu_sync_torture_init,
-        .readlock       = srcu_torture_read_lock,
-        .read_delay     = srcu_read_delay,
-        .readunlock     = srcu_torture_read_unlock,
-        .completed      = srcu_torture_completed,
-        .deferred_free  = rcu_sync_torture_deferred_free,
-        .sync           = srcu_torture_synchronize,
-        .call           = NULL,
-        .cb_barrier     = NULL,
-        .stats          = srcu_torture_stats,
-        .name           = "srcu_sync"
-};
-static void srcu_torture_synchronize_expedited(void)
-{
-        synchronize_srcu_expedited(&srcu_ctl);
-}
-static struct rcu_torture_ops srcu_expedited_ops = {
-        .init           = rcu_sync_torture_init,
-        .readlock       = srcu_torture_read_lock,
-        .read_delay     = srcu_read_delay,
-        .readunlock     = srcu_torture_read_unlock,
-        .completed      = srcu_torture_completed,
-        .deferred_free  = rcu_sync_torture_deferred_free,
-        .sync           = srcu_torture_synchronize_expedited,
-        .call           = NULL,
-        .cb_barrier     = NULL,
-        .stats          = srcu_torture_stats,
-        .name           = "srcu_expedited"
-};
 /*
 * Definitions for sched torture testing.
 */
@@ -742,6 +637,8 @@ static struct rcu_torture_ops sched_ops = {
        .completed      = rcu_no_completed,
        .deferred_free  = rcu_sched_torture_deferred_free,
        .sync           = synchronize_sched,
+        .exp_sync       = synchronize_sched_expedited,
+        .call           = call_rcu_sched,
        .cb_barrier     = rcu_barrier_sched,
        .fqs            = rcu_sched_force_quiescent_state,
        .stats          = NULL,
@@ -749,35 +646,6 @@ static struct rcu_torture_ops sched_ops = {
        .name           = "sched"
 };
-static struct rcu_torture_ops sched_sync_ops = {
-        .init           = rcu_sync_torture_init,
-        .readlock       = sched_torture_read_lock,
-        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
-        .readunlock     = sched_torture_read_unlock,
-        .completed      = rcu_no_completed,
-        .deferred_free  = rcu_sync_torture_deferred_free,
-        .sync           = synchronize_sched,
-        .cb_barrier     = NULL,
-        .fqs            = rcu_sched_force_quiescent_state,
-        .stats          = NULL,
-        .name           = "sched_sync"
-};
-static struct rcu_torture_ops sched_expedited_ops = {
-        .init           = rcu_sync_torture_init,
-        .readlock       = sched_torture_read_lock,
-        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
-        .readunlock     = sched_torture_read_unlock,
-        .completed      = rcu_no_completed,
-        .deferred_free  = rcu_sync_torture_deferred_free,
-        .sync           = synchronize_sched_expedited,
-        .cb_barrier     = NULL,
-        .fqs            = rcu_sched_force_quiescent_state,
-        .stats          = NULL,
-        .irq_capable    = 1,
-        .name           = "sched_expedited"
-};
 /*
 * RCU torture priority-boost testing.  Runs one real-time thread per
 * CPU for moderate bursts, repeatedly registering RCU callbacks and
@@ -927,9 +795,10 @@ rcu_torture_fqs(void *arg)
 static int
 rcu_torture_writer(void *arg)
 {
+        bool exp;
        int i;
-        long oldbatch = rcu_batches_completed();
        struct rcu_torture *rp;
+        struct rcu_torture *rp1;
        struct rcu_torture *old_rp;
        static DEFINE_RCU_RANDOM(rand);
@@ -954,10 +823,33 @@ rcu_torture_writer(void *arg)
                                i = RCU_TORTURE_PIPE_LEN;
                        atomic_inc(&rcu_torture_wcount[i]);
                        old_rp->rtort_pipe_count++;
-                        cur_ops->deferred_free(old_rp);
+                        if (gp_normal == gp_exp)
+                                exp = !!(rcu_random(&rand) & 0x80);
+                        else
+                                exp = gp_exp;
+                        if (!exp) {
+                                cur_ops->deferred_free(old_rp);
+                        } else {
+                                cur_ops->exp_sync();
+                                list_add(&old_rp->rtort_free,
+                                         &rcu_torture_removed);
+                                list_for_each_entry_safe(rp, rp1,
+                                                         &rcu_torture_removed,
+                                                         rtort_free) {
+                                        i = rp->rtort_pipe_count;
+                                        if (i > RCU_TORTURE_PIPE_LEN)
+                                                i = RCU_TORTURE_PIPE_LEN;
+                                        atomic_inc(&rcu_torture_wcount[i]);
+                                        if (++rp->rtort_pipe_count >=
+                                            RCU_TORTURE_PIPE_LEN) {
+                                                rp->rtort_mbtest = 0;
+                                                list_del(&rp->rtort_free);
+                                                rcu_torture_free(rp);
+                                        }
+                                 }
+                        }
                }
                rcutorture_record_progress(++rcu_torture_current_version);
-                oldbatch = cur_ops->completed();
                rcu_stutter_wait("rcu_torture_writer");
        } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
        VERBOSE_PRINTK_STRING("rcu_torture_writer task stopping");
@@ -983,10 +875,18 @@ rcu_torture_fakewriter(void *arg)
                schedule_timeout_uninterruptible(1 + rcu_random(&rand)%10);
                udelay(rcu_random(&rand) & 0x3ff);
                if (cur_ops->cb_barrier != NULL &&
-                    rcu_random(&rand) % (nfakewriters * 8) == 0)
+                    rcu_random(&rand) % (nfakewriters * 8) == 0) {
                        cur_ops->cb_barrier();
-                else
+                } else if (gp_normal == gp_exp) {
+                        if (rcu_random(&rand) & 0x80)
+                                cur_ops->sync();
+                        else
+                                cur_ops->exp_sync();
+                } else if (gp_normal) {
                        cur_ops->sync();
+                } else {
+                        cur_ops->exp_sync();
+                }
                rcu_stutter_wait("rcu_torture_fakewriter");
        } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
@@ -1364,7 +1264,7 @@ rcu_torture_stutter(void *arg)
 }
 static inline void
-rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, char *tag)
+rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag)
 {
        pr_alert("%s" TORTURE_FLAG
                 "--- %s: nreaders=%d nfakewriters=%d "
@@ -1534,7 +1434,13 @@ rcu_torture_onoff(void *arg)
                                         torture_type, cpu);
                        starttime = jiffies;
                        n_online_attempts++;
-                        if (cpu_up(cpu) == 0) {
+                        ret = cpu_up(cpu);
+                        if (ret) {
+                                if (verbose)
+                                        pr_alert("%s" TORTURE_FLAG
+                                                 "rcu_torture_onoff task: online %d failed: errno %d\n",
+                                                 torture_type, cpu, ret);
+                        } else {
                                if (verbose)
                                        pr_alert("%s" TORTURE_FLAG
                                                 "rcu_torture_onoff task: onlined %d\n",
@@ -1934,6 +1840,62 @@ rcu_torture_cleanup(void)
                rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS");
 }
+#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
+static void rcu_torture_leak_cb(struct rcu_head *rhp)
+{
+}
+static void rcu_torture_err_cb(struct rcu_head *rhp)
+{
+        /*
+         * This -might- happen due to race conditions, but is unlikely.
+         * The scenario that leads to this happening is that the
+         * first of the pair of duplicate callbacks is queued,
+         * someone else starts a grace period that includes that
+         * callback, then the second of the pair must wait for the
+         * next grace period.  Unlikely, but can happen.  If it
+         * does happen, the debug-objects subsystem won't have splatted.
+         */
+        pr_alert("rcutorture: duplicated callback was invoked.\n");
+}
+#endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
+/*
+ * Verify that double-free causes debug-objects to complain, but only
+ * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y.  Otherwise, say that the test
+ * cannot be carried out.
+ */
+static void rcu_test_debug_objects(void)
+{
+#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
+        struct rcu_head rh1;
+        struct rcu_head rh2;
+        init_rcu_head_on_stack(&rh1);
+        init_rcu_head_on_stack(&rh2);
+        pr_alert("rcutorture: WARN: Duplicate call_rcu() test starting.\n");
+        /* Try to queue the rh2 pair of callbacks for the same grace period. */
+        preempt_disable(); /* Prevent preemption from interrupting test. */
+        rcu_read_lock(); /* Make it impossible to finish a grace period. */
+        call_rcu(&rh1, rcu_torture_leak_cb); /* Start grace period. */
+        local_irq_disable(); /* Make it harder to start a new grace period. */
+        call_rcu(&rh2, rcu_torture_leak_cb);
+        call_rcu(&rh2, rcu_torture_err_cb); /* Duplicate callback. */
+        local_irq_enable();
+        rcu_read_unlock();
+        preempt_enable();
+        /* Wait for them all to get done so we can safely return. */
+        rcu_barrier();
+        pr_alert("rcutorture: WARN: Duplicate call_rcu() test complete.\n");
+        destroy_rcu_head_on_stack(&rh1);
+        destroy_rcu_head_on_stack(&rh2);
+#else /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
+        pr_alert("rcutorture: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_rcu()\n");
+#endif /* #else #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
+}
 static int __init
 rcu_torture_init(void)
 {
@@ -1941,11 +1903,9 @@ rcu_torture_init(void)
        int cpu;
        int firsterr = 0;
        int retval;
-        static struct rcu_torture_ops *torture_ops[] =
+        static struct rcu_torture_ops *torture_ops[] = {
-                { &rcu_ops, &rcu_sync_ops, &rcu_expedited_ops,
+                &rcu_ops, &rcu_bh_ops, &srcu_ops, &sched_ops,
-                  &rcu_bh_ops, &rcu_bh_sync_ops, &rcu_bh_expedited_ops,
+        };
-                  &srcu_ops, &srcu_sync_ops, &srcu_expedited_ops,
-                  &sched_ops, &sched_sync_ops, &sched_expedited_ops, };
        mutex_lock(&fullstop_mutex);
@@ -2163,6 +2123,8 @@ rcu_torture_init(void)
                firsterr = retval;
                goto unwind;
        }
+        if (object_debug)
+                rcu_test_debug_objects();
        rcutorture_record_test_transition();
        mutex_unlock(&fullstop_mutex);
        return 0;
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index 068de3a93606..32618b3fe4e6 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -53,18 +53,38 @@
 #include <linux/delay.h>
 #include <linux/stop_machine.h>
 #include <linux/random.h>
+#include <linux/ftrace_event.h>
+#include <linux/suspend.h>
 #include "rcutree.h"
 #include <trace/events/rcu.h>
 #include "rcu.h"
+/*
+ * Strings used in tracepoints need to be exported via the
+ * tracing system such that tools like perf and trace-cmd can
+ * translate the string address pointers to actual text.
+ */
+#define TPS(x)  tracepoint_string(x)
 /* Data structures. */
 static struct lock_class_key rcu_node_class[RCU_NUM_LVLS];
 static struct lock_class_key rcu_fqs_class[RCU_NUM_LVLS];
-#define RCU_STATE_INITIALIZER(sname, sabbr, cr) { \
+/*
+ * In order to export the rcu_state name to the tracing tools, it
+ * needs to be added in the __tracepoint_string section.
+ * This requires defining a separate variable tp_<sname>_varname
+ * that points to the string being used, and this will allow
+ * the tracing userspace tools to be able to decipher the string
+ * address to the matching string.
+ */
+#define RCU_STATE_INITIALIZER(sname, sabbr, cr) \
+static char sname##_varname[] = #sname; \
+static const char *tp_##sname##_varname __used __tracepoint_string = sname##_varname; \
+struct rcu_state sname##_state = { \
        .level = { &sname##_state.node[0] }, \
        .call = cr, \
        .fqs_state = RCU_GP_IDLE, \
@@ -75,16 +95,13 @@ static struct lock_class_key rcu_fqs_class[RCU_NUM_LVLS];
        .orphan_donetail = &sname##_state.orphan_donelist, \
        .barrier_mutex = __MUTEX_INITIALIZER(sname##_state.barrier_mutex), \
        .onoff_mutex = __MUTEX_INITIALIZER(sname##_state.onoff_mutex), \
-        .name = #sname, \
+        .name = sname##_varname, \
        .abbr = sabbr, \
-}
+}; \
+DEFINE_PER_CPU(struct rcu_data, sname##_data)
-struct rcu_state rcu_sched_state =
-        RCU_STATE_INITIALIZER(rcu_sched, 's', call_rcu_sched);
-DEFINE_PER_CPU(struct rcu_data, rcu_sched_data);
-struct rcu_state rcu_bh_state = RCU_STATE_INITIALIZER(rcu_bh, 'b', call_rcu_bh);
+RCU_STATE_INITIALIZER(rcu_sched, 's', call_rcu_sched);
-DEFINE_PER_CPU(struct rcu_data, rcu_bh_data);
+RCU_STATE_INITIALIZER(rcu_bh, 'b', call_rcu_bh);
 static struct rcu_state *rcu_state;
 LIST_HEAD(rcu_struct_flavors);
@@ -178,7 +195,7 @@ void rcu_sched_qs(int cpu)
        struct rcu_data *rdp = &per_cpu(rcu_sched_data, cpu);
        if (rdp->passed_quiesce == 0)
-                trace_rcu_grace_period("rcu_sched", rdp->gpnum, "cpuqs");
+                trace_rcu_grace_period(TPS("rcu_sched"), rdp->gpnum, TPS("cpuqs"));
        rdp->passed_quiesce = 1;
 }
@@ -187,7 +204,7 @@ void rcu_bh_qs(int cpu)
        struct rcu_data *rdp = &per_cpu(rcu_bh_data, cpu);
        if (rdp->passed_quiesce == 0)
-                trace_rcu_grace_period("rcu_bh", rdp->gpnum, "cpuqs");
+                trace_rcu_grace_period(TPS("rcu_bh"), rdp->gpnum, TPS("cpuqs"));
        rdp->passed_quiesce = 1;
 }
@@ -198,16 +215,20 @@ void rcu_bh_qs(int cpu)
 */
 void rcu_note_context_switch(int cpu)
 {
-        trace_rcu_utilization("Start context switch");
+        trace_rcu_utilization(TPS("Start context switch"));
        rcu_sched_qs(cpu);
        rcu_preempt_note_context_switch(cpu);
-        trace_rcu_utilization("End context switch");
+        trace_rcu_utilization(TPS("End context switch"));
 }
 EXPORT_SYMBOL_GPL(rcu_note_context_switch);
 DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
        .dynticks_nesting = DYNTICK_TASK_EXIT_IDLE,
        .dynticks = ATOMIC_INIT(1),
+#ifdef CONFIG_NO_HZ_FULL_SYSIDLE
+        .dynticks_idle_nesting = DYNTICK_TASK_NEST_VALUE,
+        .dynticks_idle = ATOMIC_INIT(1),
+#endif /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
 };
 static long blimit = 10;        /* Maximum callbacks per rcu_do_batch. */
@@ -226,7 +247,10 @@ module_param(jiffies_till_next_fqs, ulong, 0644);
 static void rcu_start_gp_advanced(struct rcu_state *rsp, struct rcu_node *rnp,
                                  struct rcu_data *rdp);
-static void force_qs_rnp(struct rcu_state *rsp, int (*f)(struct rcu_data *));
+static void force_qs_rnp(struct rcu_state *rsp,
+                         int (*f)(struct rcu_data *rsp, bool *isidle,
+                                  unsigned long *maxj),
+                         bool *isidle, unsigned long *maxj);
 static void force_quiescent_state(struct rcu_state *rsp);
 static int rcu_pending(int cpu);
@@ -345,11 +369,11 @@ static struct rcu_node *rcu_get_root(struct rcu_state *rsp)
 static void rcu_eqs_enter_common(struct rcu_dynticks *rdtp, long long oldval,
                                bool user)
 {
-        trace_rcu_dyntick("Start", oldval, rdtp->dynticks_nesting);
+        trace_rcu_dyntick(TPS("Start"), oldval, rdtp->dynticks_nesting);
        if (!user && !is_idle_task(current)) {
                struct task_struct *idle = idle_task(smp_processor_id());
-                trace_rcu_dyntick("Error on entry: not idle task", oldval, 0);
+                trace_rcu_dyntick(TPS("Error on entry: not idle task"), oldval, 0);
                ftrace_dump(DUMP_ORIG);
                WARN_ONCE(1, "Current pid: %d comm: %s / Idle pid: %d comm: %s",
                          current->pid, current->comm,
@@ -411,6 +435,7 @@ void rcu_idle_enter(void)
        local_irq_save(flags);
        rcu_eqs_enter(false);
+        rcu_sysidle_enter(&__get_cpu_var(rcu_dynticks), 0);
        local_irq_restore(flags);
 }
 EXPORT_SYMBOL_GPL(rcu_idle_enter);
@@ -428,27 +453,6 @@ void rcu_user_enter(void)
 {
        rcu_eqs_enter(1);
 }
-/**
- * rcu_user_enter_after_irq - inform RCU that we are going to resume userspace
- * after the current irq returns.
- *
- * This is similar to rcu_user_enter() but in the context of a non-nesting
- * irq. After this call, RCU enters into idle mode when the interrupt
- * returns.
- */
-void rcu_user_enter_after_irq(void)
-{
-        unsigned long flags;
-        struct rcu_dynticks *rdtp;
-        local_irq_save(flags);
-        rdtp = &__get_cpu_var(rcu_dynticks);
-        /* Ensure this irq is interrupting a non-idle RCU state.  */
-        WARN_ON_ONCE(!(rdtp->dynticks_nesting & DYNTICK_TASK_MASK));
-        rdtp->dynticks_nesting = 1;
-        local_irq_restore(flags);
-}
 #endif /* CONFIG_RCU_USER_QS */
 /**
@@ -479,9 +483,10 @@ void rcu_irq_exit(void)
        rdtp->dynticks_nesting--;
        WARN_ON_ONCE(rdtp->dynticks_nesting < 0);
        if (rdtp->dynticks_nesting)
-                trace_rcu_dyntick("--=", oldval, rdtp->dynticks_nesting);
+                trace_rcu_dyntick(TPS("--="), oldval, rdtp->dynticks_nesting);
        else
                rcu_eqs_enter_common(rdtp, oldval, true);
+        rcu_sysidle_enter(rdtp, 1);
        local_irq_restore(flags);
 }
@@ -501,11 +506,11 @@ static void rcu_eqs_exit_common(struct rcu_dynticks *rdtp, long long oldval,
        smp_mb__after_atomic_inc();  /* See above. */
        WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
        rcu_cleanup_after_idle(smp_processor_id());
-        trace_rcu_dyntick("End", oldval, rdtp->dynticks_nesting);
+        trace_rcu_dyntick(TPS("End"), oldval, rdtp->dynticks_nesting);
        if (!user && !is_idle_task(current)) {
                struct task_struct *idle = idle_task(smp_processor_id());
-                trace_rcu_dyntick("Error on exit: not idle task",
+                trace_rcu_dyntick(TPS("Error on exit: not idle task"),
                                  oldval, rdtp->dynticks_nesting);
                ftrace_dump(DUMP_ORIG);
                WARN_ONCE(1, "Current pid: %d comm: %s / Idle pid: %d comm: %s",
@@ -550,6 +555,7 @@ void rcu_idle_exit(void)
        local_irq_save(flags);
        rcu_eqs_exit(false);
+        rcu_sysidle_exit(&__get_cpu_var(rcu_dynticks), 0);
        local_irq_restore(flags);
 }
 EXPORT_SYMBOL_GPL(rcu_idle_exit);
@@ -565,28 +571,6 @@ void rcu_user_exit(void)
 {
        rcu_eqs_exit(1);
 }
-/**
- * rcu_user_exit_after_irq - inform RCU that we won't resume to userspace
- * idle mode after the current non-nesting irq returns.
- *
- * This is similar to rcu_user_exit() but in the context of an irq.
- * This is called when the irq has interrupted a userspace RCU idle mode
- * context. When the current non-nesting interrupt returns after this call,
- * the CPU won't restore the RCU idle mode.
- */
-void rcu_user_exit_after_irq(void)
-{
-        unsigned long flags;
-        struct rcu_dynticks *rdtp;
-        local_irq_save(flags);
-        rdtp = &__get_cpu_var(rcu_dynticks);
-        /* Ensure we are interrupting an RCU idle mode. */
-        WARN_ON_ONCE(rdtp->dynticks_nesting & DYNTICK_TASK_NEST_MASK);
-        rdtp->dynticks_nesting += DYNTICK_TASK_EXIT_IDLE;
-        local_irq_restore(flags);
-}
 #endif /* CONFIG_RCU_USER_QS */
 /**
@@ -620,9 +604,10 @@ void rcu_irq_enter(void)
        rdtp->dynticks_nesting++;
        WARN_ON_ONCE(rdtp->dynticks_nesting == 0);
        if (oldval)
-                trace_rcu_dyntick("++=", oldval, rdtp->dynticks_nesting);
+                trace_rcu_dyntick(TPS("++="), oldval, rdtp->dynticks_nesting);
        else
                rcu_eqs_exit_common(rdtp, oldval, true);
+        rcu_sysidle_exit(rdtp, 1);
        local_irq_restore(flags);
 }
@@ -746,9 +731,11 @@ static int rcu_is_cpu_rrupt_from_idle(void)
 * credit them with an implicit quiescent state.  Return 1 if this CPU
 * is in dynticks idle mode, which is an extended quiescent state.
 */
-static int dyntick_save_progress_counter(struct rcu_data *rdp)
+static int dyntick_save_progress_counter(struct rcu_data *rdp,
+                                         bool *isidle, unsigned long *maxj)
 {
        rdp->dynticks_snap = atomic_add_return(0, &rdp->dynticks->dynticks);
+        rcu_sysidle_check_cpu(rdp, isidle, maxj);
        return (rdp->dynticks_snap & 0x1) == 0;
 }
@@ -758,7 +745,8 @@ static int dyntick_save_progress_counter(struct rcu_data *rdp)
 * idle state since the last call to dyntick_save_progress_counter()
 * for this same CPU, or by virtue of having been offline.
 */
-static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)
+static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
+                                    bool *isidle, unsigned long *maxj)
 {
        unsigned int curr;
        unsigned int snap;
@@ -775,7 +763,7 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)
         * of the current RCU grace period.
         */
        if ((curr & 0x1) == 0 || UINT_CMP_GE(curr, snap + 2)) {
-                trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, "dti");
+                trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti"));
                rdp->dynticks_fqs++;
                return 1;
        }
@@ -795,7 +783,7 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)
                return 0;  /* Grace period is not old enough. */
        barrier();
        if (cpu_is_offline(rdp->cpu)) {
-                trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, "ofl");
+                trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("ofl"));
                rdp->offline_fqs++;
                return 1;
        }
@@ -1032,7 +1020,7 @@ static unsigned long rcu_cbs_completed(struct rcu_state *rsp,
 * rcu_nocb_wait_gp().
 */
 static void trace_rcu_future_gp(struct rcu_node *rnp, struct rcu_data *rdp,
-                                unsigned long c, char *s)
+                                unsigned long c, const char *s)
 {
        trace_rcu_future_grace_period(rdp->rsp->name, rnp->gpnum,
                                      rnp->completed, c, rnp->level,
@@ -1058,9 +1046,9 @@ rcu_start_future_gp(struct rcu_node *rnp, struct rcu_data *rdp)
         * grace period is already marked as needed, return to the caller.
         */
        c = rcu_cbs_completed(rdp->rsp, rnp);
-        trace_rcu_future_gp(rnp, rdp, c, "Startleaf");
+        trace_rcu_future_gp(rnp, rdp, c, TPS("Startleaf"));
        if (rnp->need_future_gp[c & 0x1]) {
-                trace_rcu_future_gp(rnp, rdp, c, "Prestartleaf");
+                trace_rcu_future_gp(rnp, rdp, c, TPS("Prestartleaf"));
                return c;
        }
@@ -1074,7 +1062,7 @@ rcu_start_future_gp(struct rcu_node *rnp, struct rcu_data *rdp)
        if (rnp->gpnum != rnp->completed ||
            ACCESS_ONCE(rnp->gpnum) != ACCESS_ONCE(rnp->completed)) {
                rnp->need_future_gp[c & 0x1]++;
-                trace_rcu_future_gp(rnp, rdp, c, "Startedleaf");
+                trace_rcu_future_gp(rnp, rdp, c, TPS("Startedleaf"));
                return c;
        }
@@ -1102,7 +1090,7 @@ rcu_start_future_gp(struct rcu_node *rnp, struct rcu_data *rdp)
         * recorded, trace and leave.
         */
        if (rnp_root->need_future_gp[c & 0x1]) {
-                trace_rcu_future_gp(rnp, rdp, c, "Prestartedroot");
+                trace_rcu_future_gp(rnp, rdp, c, TPS("Prestartedroot"));
                goto unlock_out;
        }
@@ -1111,9 +1099,9 @@ rcu_start_future_gp(struct rcu_node *rnp, struct rcu_data *rdp)
        /* If a grace period is not already in progress, start one. */
        if (rnp_root->gpnum != rnp_root->completed) {
-                trace_rcu_future_gp(rnp, rdp, c, "Startedleafroot");
+                trace_rcu_future_gp(rnp, rdp, c, TPS("Startedleafroot"));
        } else {
-                trace_rcu_future_gp(rnp, rdp, c, "Startedroot");
+                trace_rcu_future_gp(rnp, rdp, c, TPS("Startedroot"));
                rcu_start_gp_advanced(rdp->rsp, rnp_root, rdp);
        }
 unlock_out:
@@ -1137,7 +1125,8 @@ static int rcu_future_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp)
        rcu_nocb_gp_cleanup(rsp, rnp);
        rnp->need_future_gp[c & 0x1] = 0;
        needmore = rnp->need_future_gp[(c + 1) & 0x1];
-        trace_rcu_future_gp(rnp, rdp, c, needmore ? "CleanupMore" : "Cleanup");
+        trace_rcu_future_gp(rnp, rdp, c,
+                            needmore ? TPS("CleanupMore") : TPS("Cleanup"));
        return needmore;
 }
@@ -1205,9 +1194,9 @@ static void rcu_accelerate_cbs(struct rcu_state *rsp, struct rcu_node *rnp,
        /* Trace depending on how much we were able to accelerate. */
        if (!*rdp->nxttail[RCU_WAIT_TAIL])
-                trace_rcu_grace_period(rsp->name, rdp->gpnum, "AccWaitCB");
+                trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("AccWaitCB"));
        else
-                trace_rcu_grace_period(rsp->name, rdp->gpnum, "AccReadyCB");
+                trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("AccReadyCB"));
 }
 /*
@@ -1273,7 +1262,7 @@ static void __note_gp_changes(struct rcu_state *rsp, struct rcu_node *rnp, struc
                /* Remember that we saw this grace-period completion. */
                rdp->completed = rnp->completed;
-                trace_rcu_grace_period(rsp->name, rdp->gpnum, "cpuend");
+                trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuend"));
        }
        if (rdp->gpnum != rnp->gpnum) {
@@ -1283,7 +1272,7 @@ static void __note_gp_changes(struct rcu_state *rsp, struct rcu_node *rnp, struc
                 * go looking for one.
                 */
                rdp->gpnum = rnp->gpnum;
-                trace_rcu_grace_period(rsp->name, rdp->gpnum, "cpustart");
+                trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpustart"));
                rdp->passed_quiesce = 0;
                rdp->qs_pending = !!(rnp->qsmask & rdp->grpmask);
                zero_cpu_stall_ticks(rdp);
@@ -1315,6 +1304,7 @@ static int rcu_gp_init(struct rcu_state *rsp)
        struct rcu_data *rdp;
        struct rcu_node *rnp = rcu_get_root(rsp);
+        rcu_bind_gp_kthread();
        raw_spin_lock_irq(&rnp->lock);
        rsp->gp_flags = 0; /* Clear all flags: New grace period. */
@@ -1326,7 +1316,7 @@ static int rcu_gp_init(struct rcu_state *rsp)
        /* Advance to a new grace period and initialize state. */
        rsp->gpnum++;
-        trace_rcu_grace_period(rsp->name, rsp->gpnum, "start");
+        trace_rcu_grace_period(rsp->name, rsp->gpnum, TPS("start"));
        record_gp_stall_check_time(rsp);
        raw_spin_unlock_irq(&rnp->lock);
@@ -1379,16 +1369,25 @@ static int rcu_gp_init(struct rcu_state *rsp)
 int rcu_gp_fqs(struct rcu_state *rsp, int fqs_state_in)
 {
        int fqs_state = fqs_state_in;
+        bool isidle = false;
+        unsigned long maxj;
        struct rcu_node *rnp = rcu_get_root(rsp);
        rsp->n_force_qs++;
        if (fqs_state == RCU_SAVE_DYNTICK) {
                /* Collect dyntick-idle snapshots. */
-                force_qs_rnp(rsp, dyntick_save_progress_counter);
+                if (is_sysidle_rcu_state(rsp)) {
+                        isidle = 1;
+                        maxj = jiffies - ULONG_MAX / 4;
+                }
+                force_qs_rnp(rsp, dyntick_save_progress_counter,
+                             &isidle, &maxj);
+                rcu_sysidle_report_gp(rsp, isidle, maxj);
                fqs_state = RCU_FORCE_QS;
        } else {
                /* Handle dyntick-idle and offline CPUs. */
-                force_qs_rnp(rsp, rcu_implicit_dynticks_qs);
+                isidle = 0;
+                force_qs_rnp(rsp, rcu_implicit_dynticks_qs, &isidle, &maxj);
        }
        /* Clear flag to prevent immediate re-entry. */
        if (ACCESS_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) {
@@ -1448,7 +1447,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
        rcu_nocb_gp_set(rnp, nocb);
        rsp->completed = rsp->gpnum; /* Declare grace period done. */
-        trace_rcu_grace_period(rsp->name, rsp->completed, "end");
+        trace_rcu_grace_period(rsp->name, rsp->completed, TPS("end"));
        rsp->fqs_state = RCU_GP_IDLE;
        rdp = this_cpu_ptr(rsp->rda);
        rcu_advance_cbs(rsp, rnp, rdp);  /* Reduce false positives below. */
@@ -1558,10 +1557,12 @@ rcu_start_gp_advanced(struct rcu_state *rsp, struct rcu_node *rnp,
        /*
         * We can't do wakeups while holding the rnp->lock, as that
-         * could cause possible deadlocks with the rq->lock. Deter
+         * could cause possible deadlocks with the rq->lock. Defer
-         * the wakeup to interrupt context.
+         * the wakeup to interrupt context.  And don't bother waking
+         * up the running kthread.
         */
-        irq_work_queue(&rsp->wakeup_work);
+        if (current != rsp->gp_kthread)
+                irq_work_queue(&rsp->wakeup_work);
 }
 /*
@@ -1857,7 +1858,7 @@ static void rcu_cleanup_dying_cpu(struct rcu_state *rsp)
        RCU_TRACE(mask = rdp->grpmask);
        trace_rcu_grace_period(rsp->name,
                               rnp->gpnum + 1 - !!(rnp->qsmask & mask),
-                               "cpuofl");
+                               TPS("cpuofl"));
 }
 /*
@@ -2044,7 +2045,7 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp)
 */
 void rcu_check_callbacks(int cpu, int user)
 {
-        trace_rcu_utilization("Start scheduler-tick");
+        trace_rcu_utilization(TPS("Start scheduler-tick"));
        increment_cpu_stall_ticks();
        if (user || rcu_is_cpu_rrupt_from_idle()) {
@@ -2077,7 +2078,7 @@ void rcu_check_callbacks(int cpu, int user)
        rcu_preempt_check_callbacks(cpu);
        if (rcu_pending(cpu))
                invoke_rcu_core();
-        trace_rcu_utilization("End scheduler-tick");
+        trace_rcu_utilization(TPS("End scheduler-tick"));
 }
 /*
@@ -2087,7 +2088,10 @@ void rcu_check_callbacks(int cpu, int user)
 *
 * The caller must have suppressed start of new grace periods.
 */
-static void force_qs_rnp(struct rcu_state *rsp, int (*f)(struct rcu_data *))
+static void force_qs_rnp(struct rcu_state *rsp,
+                         int (*f)(struct rcu_data *rsp, bool *isidle,
+                                  unsigned long *maxj),
+                         bool *isidle, unsigned long *maxj)
 {
        unsigned long bit;
        int cpu;
@@ -2110,9 +2114,12 @@ static void force_qs_rnp(struct rcu_state *rsp, int (*f)(struct rcu_data *))
                cpu = rnp->grplo;
                bit = 1;
                for (; cpu <= rnp->grphi; cpu++, bit <<= 1) {
-                        if ((rnp->qsmask & bit) != 0 &&
+                        if ((rnp->qsmask & bit) != 0) {
-                            f(per_cpu_ptr(rsp->rda, cpu)))
+                                if ((rnp->qsmaskinit & bit) != 0)
-                                mask |= bit;
+                                        *isidle = 0;
+                                if (f(per_cpu_ptr(rsp->rda, cpu), isidle, maxj))
+                                        mask |= bit;
+                        }
                }
                if (mask != 0) {
@@ -2208,10 +2215,10 @@ static void rcu_process_callbacks(struct softirq_action *unused)
        if (cpu_is_offline(smp_processor_id()))
                return;
-        trace_rcu_utilization("Start RCU core");
+        trace_rcu_utilization(TPS("Start RCU core"));
        for_each_rcu_flavor(rsp)
                __rcu_process_callbacks(rsp);
-        trace_rcu_utilization("End RCU core");
+        trace_rcu_utilization(TPS("End RCU core"));
 }
 /*
@@ -2287,6 +2294,13 @@ static void __call_rcu_core(struct rcu_state *rsp, struct rcu_data *rdp,
 }
 /*
+ * RCU callback function to leak a callback.
+ */
+static void rcu_leak_callback(struct rcu_head *rhp)
+{
+}
+/*
 * Helper function for call_rcu() and friends.  The cpu argument will
 * normally be -1, indicating "currently running CPU".  It may specify
 * a CPU only if that CPU is a no-CBs CPU.  Currently, only _rcu_barrier()
@@ -2300,7 +2314,12 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
        struct rcu_data *rdp;
        WARN_ON_ONCE((unsigned long)head & 0x3); /* Misaligned rcu_head! */
-        debug_rcu_head_queue(head);
+        if (debug_rcu_head_queue(head)) {
+                /* Probable double call_rcu(), so leak the callback. */
+                ACCESS_ONCE(head->func) = rcu_leak_callback;
+                WARN_ONCE(1, "__call_rcu(): Leaked duplicate callback\n");
+                return;
+        }
        head->func = func;
        head->next = NULL;
@@ -2720,7 +2739,7 @@ static int rcu_cpu_has_callbacks(int cpu, bool *all_lazy)
 * Helper function for _rcu_barrier() tracing.  If tracing is disabled,
 * the compiler is expected to optimize this away.
 */
-static void _rcu_barrier_trace(struct rcu_state *rsp, char *s,
+static void _rcu_barrier_trace(struct rcu_state *rsp, const char *s,
                               int cpu, unsigned long done)
 {
        trace_rcu_barrier(rsp->name, s, cpu,
@@ -2785,9 +2804,20 @@ static void _rcu_barrier(struct rcu_state *rsp)
         * transition.  The "if" expression below therefore rounds the old
         * value up to the next even number and adds two before comparing.
         */
-        snap_done = ACCESS_ONCE(rsp->n_barrier_done);
+        snap_done = rsp->n_barrier_done;
        _rcu_barrier_trace(rsp, "Check", -1, snap_done);
-        if (ULONG_CMP_GE(snap_done, ((snap + 1) & ~0x1) + 2)) {
+        /*
+         * If the value in snap is odd, we needed to wait for the current
+         * rcu_barrier() to complete, then wait for the next one, in other
+         * words, we need the value of snap_done to be three larger than
+         * the value of snap.  On the other hand, if the value in snap is
+         * even, we only had to wait for the next rcu_barrier() to complete,
+         * in other words, we need the value of snap_done to be only two
+         * greater than the value of snap.  The "(snap + 3) & ~0x1" computes
+         * this for us (thank you, Linus!).
+         */
+        if (ULONG_CMP_GE(snap_done, (snap + 3) & ~0x1)) {
                _rcu_barrier_trace(rsp, "EarlyExit", -1, snap_done);
                smp_mb(); /* caller's subsequent code after above check. */
                mutex_unlock(&rsp->barrier_mutex);
@@ -2930,6 +2960,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp, int preemptible)
        rdp->blimit = blimit;
        init_callback_list(rdp);  /* Re-enable callbacks on this CPU. */
        rdp->dynticks->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
+        rcu_sysidle_init_percpu_data(rdp->dynticks);
        atomic_set(&rdp->dynticks->dynticks,
                   (atomic_read(&rdp->dynticks->dynticks) & ~0x1) + 1);
        raw_spin_unlock(&rnp->lock);            /* irqs remain disabled. */
@@ -2952,7 +2983,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp, int preemptible)
                        rdp->completed = rnp->completed;
                        rdp->passed_quiesce = 0;
                        rdp->qs_pending = 0;
-                        trace_rcu_grace_period(rsp->name, rdp->gpnum, "cpuonl");
+                        trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuonl"));
                }
                raw_spin_unlock(&rnp->lock); /* irqs already disabled. */
                rnp = rnp->parent;
@@ -2982,7 +3013,7 @@ static int rcu_cpu_notify(struct notifier_block *self,
        struct rcu_node *rnp = rdp->mynode;
        struct rcu_state *rsp;
-        trace_rcu_utilization("Start CPU hotplug");
+        trace_rcu_utilization(TPS("Start CPU hotplug"));
        switch (action) {
        case CPU_UP_PREPARE:
        case CPU_UP_PREPARE_FROZEN:
@@ -3011,7 +3042,26 @@ static int rcu_cpu_notify(struct notifier_block *self,
        default:
                break;
        }
-        trace_rcu_utilization("End CPU hotplug");
+        trace_rcu_utilization(TPS("End CPU hotplug"));
+        return NOTIFY_OK;
+}
+static int rcu_pm_notify(struct notifier_block *self,
+                         unsigned long action, void *hcpu)
+{
+        switch (action) {
+        case PM_HIBERNATION_PREPARE:
+        case PM_SUSPEND_PREPARE:
+                if (nr_cpu_ids <= 256) /* Expediting bad for large systems. */
+                        rcu_expedited = 1;
+                break;
+        case PM_POST_HIBERNATION:
+        case PM_POST_SUSPEND:
+                rcu_expedited = 0;
+                break;
+        default:
+                break;
+        }
        return NOTIFY_OK;
 }
@@ -3256,6 +3306,7 @@ void __init rcu_init(void)
         * or the scheduler are operational.
         */
        cpu_notifier(rcu_cpu_notify, 0);
+        pm_notifier(rcu_pm_notify, 0);
        for_each_online_cpu(cpu)
                rcu_cpu_notify(NULL, CPU_UP_PREPARE, (void *)(long)cpu);
 }
diff --git a/kernel/rcutree.h b/kernel/rcutree.h
index b3832581043c..5f97eab602cd 100644
--- a/kernel/rcutree.h
+++ b/kernel/rcutree.h
@@ -88,6 +88,14 @@ struct rcu_dynticks {
                                    /* Process level is worth LLONG_MAX/2. */
        int dynticks_nmi_nesting;   /* Track NMI nesting level. */
        atomic_t dynticks;          /* Even value for idle, else odd. */
+#ifdef CONFIG_NO_HZ_FULL_SYSIDLE
+        long long dynticks_idle_nesting;
+                                    /* irq/process nesting level from idle. */
+        atomic_t dynticks_idle;     /* Even value for idle, else odd. */
+                                    /*  "Idle" excludes userspace execution. */
+        unsigned long dynticks_idle_jiffies;
+                                    /* End of last non-NMI non-idle period. */
+#endif /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
 #ifdef CONFIG_RCU_FAST_NO_HZ
        bool all_lazy;              /* Are all CPU's CBs lazy? */
        unsigned long nonlazy_posted;
@@ -445,7 +453,7 @@ struct rcu_state {
                                                /*  for CPU stalls. */
        unsigned long gp_max;                   /* Maximum GP duration in */
                                                /*  jiffies. */
-        char *name;                             /* Name of structure. */
+        const char *name;                       /* Name of structure. */
        char abbr;                              /* Abbreviated name. */
        struct list_head flavors;               /* List of RCU flavors. */
        struct irq_work wakeup_work;            /* Postponed wakeups */
@@ -545,6 +553,15 @@ static void rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp);
 static void rcu_spawn_nocb_kthreads(struct rcu_state *rsp);
 static void rcu_kick_nohz_cpu(int cpu);
 static bool init_nocb_callback_list(struct rcu_data *rdp);
+static void rcu_sysidle_enter(struct rcu_dynticks *rdtp, int irq);
+static void rcu_sysidle_exit(struct rcu_dynticks *rdtp, int irq);
+static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
+                                  unsigned long *maxj);
+static bool is_sysidle_rcu_state(struct rcu_state *rsp);
+static void rcu_sysidle_report_gp(struct rcu_state *rsp, int isidle,
+                                  unsigned long maxj);
+static void rcu_bind_gp_kthread(void);
+static void rcu_sysidle_init_percpu_data(struct rcu_dynticks *rdtp);
 #endif /* #ifndef RCU_TREE_NONCORE */
diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h
index 769e12e3151b..130c97b027f2 100644
--- a/kernel/rcutree_plugin.h
+++ b/kernel/rcutree_plugin.h
@@ -28,7 +28,7 @@
 #include <linux/gfp.h>
 #include <linux/oom.h>
 #include <linux/smpboot.h>
-#include <linux/tick.h>
+#include "time/tick-internal.h"
 #define RCU_KTHREAD_PRIO 1
@@ -110,9 +110,7 @@ static void __init rcu_bootup_announce_oddness(void)
 #ifdef CONFIG_TREE_PREEMPT_RCU
-struct rcu_state rcu_preempt_state =
+RCU_STATE_INITIALIZER(rcu_preempt, 'p', call_rcu);
-        RCU_STATE_INITIALIZER(rcu_preempt, 'p', call_rcu);
-DEFINE_PER_CPU(struct rcu_data, rcu_preempt_data);
 static struct rcu_state *rcu_state = &rcu_preempt_state;
 static int rcu_preempted_readers_exp(struct rcu_node *rnp);
@@ -169,7 +167,7 @@ static void rcu_preempt_qs(int cpu)
        struct rcu_data *rdp = &per_cpu(rcu_preempt_data, cpu);
        if (rdp->passed_quiesce == 0)
-                trace_rcu_grace_period("rcu_preempt", rdp->gpnum, "cpuqs");
+                trace_rcu_grace_period(TPS("rcu_preempt"), rdp->gpnum, TPS("cpuqs"));
        rdp->passed_quiesce = 1;
        current->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_NEED_QS;
 }
@@ -388,7 +386,7 @@ void rcu_read_unlock_special(struct task_struct *t)
                np = rcu_next_node_entry(t, rnp);
                list_del_init(&t->rcu_node_entry);
                t->rcu_blocked_node = NULL;
-                trace_rcu_unlock_preempted_task("rcu_preempt",
+                trace_rcu_unlock_preempted_task(TPS("rcu_preempt"),
                                                rnp->gpnum, t->pid);
                if (&t->rcu_node_entry == rnp->gp_tasks)
                        rnp->gp_tasks = np;
@@ -412,7 +410,7 @@ void rcu_read_unlock_special(struct task_struct *t)
                 */
                empty_exp_now = !rcu_preempted_readers_exp(rnp);
                if (!empty && !rcu_preempt_blocked_readers_cgp(rnp)) {
-                        trace_rcu_quiescent_state_report("preempt_rcu",
+                        trace_rcu_quiescent_state_report(TPS("preempt_rcu"),
                                                         rnp->gpnum,
                                                         0, rnp->qsmask,
                                                         rnp->level,
@@ -1250,12 +1248,12 @@ static int rcu_boost_kthread(void *arg)
        int spincnt = 0;
        int more2boost;
-        trace_rcu_utilization("Start boost kthread@init");
+        trace_rcu_utilization(TPS("Start boost kthread@init"));
        for (;;) {
                rnp->boost_kthread_status = RCU_KTHREAD_WAITING;
-                trace_rcu_utilization("End boost kthread@rcu_wait");
+                trace_rcu_utilization(TPS("End boost kthread@rcu_wait"));
                rcu_wait(rnp->boost_tasks || rnp->exp_tasks);
-                trace_rcu_utilization("Start boost kthread@rcu_wait");
+                trace_rcu_utilization(TPS("Start boost kthread@rcu_wait"));
                rnp->boost_kthread_status = RCU_KTHREAD_RUNNING;
                more2boost = rcu_boost(rnp);
                if (more2boost)
@@ -1264,14 +1262,14 @@ static int rcu_boost_kthread(void *arg)
                        spincnt = 0;
                if (spincnt > 10) {
                        rnp->boost_kthread_status = RCU_KTHREAD_YIELDING;
-                        trace_rcu_utilization("End boost kthread@rcu_yield");
+                        trace_rcu_utilization(TPS("End boost kthread@rcu_yield"));
                        schedule_timeout_interruptible(2);
-                        trace_rcu_utilization("Start boost kthread@rcu_yield");
+                        trace_rcu_utilization(TPS("Start boost kthread@rcu_yield"));
                        spincnt = 0;
                }
        }
        /* NOTREACHED */
-        trace_rcu_utilization("End boost kthread@notreached");
+        trace_rcu_utilization(TPS("End boost kthread@notreached"));
        return 0;
 }
@@ -1419,7 +1417,7 @@ static void rcu_cpu_kthread(unsigned int cpu)
        int spincnt;
        for (spincnt = 0; spincnt < 10; spincnt++) {
-                trace_rcu_utilization("Start CPU kthread@rcu_wait");
+                trace_rcu_utilization(TPS("Start CPU kthread@rcu_wait"));
                local_bh_disable();
                *statusp = RCU_KTHREAD_RUNNING;
                this_cpu_inc(rcu_cpu_kthread_loops);
@@ -1431,15 +1429,15 @@ static void rcu_cpu_kthread(unsigned int cpu)
                        rcu_kthread_do_work();
                local_bh_enable();
                if (*workp == 0) {
-                        trace_rcu_utilization("End CPU kthread@rcu_wait");
+                        trace_rcu_utilization(TPS("End CPU kthread@rcu_wait"));
                        *statusp = RCU_KTHREAD_WAITING;
                        return;
                }
        }
        *statusp = RCU_KTHREAD_YIELDING;
-        trace_rcu_utilization("Start CPU kthread@rcu_yield");
+        trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield"));
        schedule_timeout_interruptible(2);
-        trace_rcu_utilization("End CPU kthread@rcu_yield");
+        trace_rcu_utilization(TPS("End CPU kthread@rcu_yield"));
        *statusp = RCU_KTHREAD_WAITING;
 }
@@ -2202,7 +2200,7 @@ static void rcu_nocb_wait_gp(struct rcu_data *rdp)
         * Wait for the grace period.  Do so interruptibly to avoid messing
         * up the load average.
         */
-        trace_rcu_future_gp(rnp, rdp, c, "StartWait");
+        trace_rcu_future_gp(rnp, rdp, c, TPS("StartWait"));
        for (;;) {
                wait_event_interruptible(
                        rnp->nocb_gp_wq[c & 0x1],
@@ -2210,9 +2208,9 @@ static void rcu_nocb_wait_gp(struct rcu_data *rdp)
                if (likely(d))
                        break;
                flush_signals(current);
-                trace_rcu_future_gp(rnp, rdp, c, "ResumeWait");
+                trace_rcu_future_gp(rnp, rdp, c, TPS("ResumeWait"));
        }
-        trace_rcu_future_gp(rnp, rdp, c, "EndWait");
+        trace_rcu_future_gp(rnp, rdp, c, TPS("EndWait"));
        smp_mb(); /* Ensure that CB invocation happens after GP end. */
 }
@@ -2375,3 +2373,425 @@ static void rcu_kick_nohz_cpu(int cpu)
                smp_send_reschedule(cpu);
 #endif /* #ifdef CONFIG_NO_HZ_FULL */
 }
+#ifdef CONFIG_NO_HZ_FULL_SYSIDLE
+/*
+ * Define RCU flavor that holds sysidle state.  This needs to be the
+ * most active flavor of RCU.
+ */
+#ifdef CONFIG_PREEMPT_RCU
+static struct rcu_state *rcu_sysidle_state = &rcu_preempt_state;
+#else /* #ifdef CONFIG_PREEMPT_RCU */
+static struct rcu_state *rcu_sysidle_state = &rcu_sched_state;
+#endif /* #else #ifdef CONFIG_PREEMPT_RCU */
+static int full_sysidle_state;          /* Current system-idle state. */
+#define RCU_SYSIDLE_NOT         0       /* Some CPU is not idle. */
+#define RCU_SYSIDLE_SHORT       1       /* All CPUs idle for brief period. */
+#define RCU_SYSIDLE_LONG        2       /* All CPUs idle for long enough. */
+#define RCU_SYSIDLE_FULL        3       /* All CPUs idle, ready for sysidle. */
+#define RCU_SYSIDLE_FULL_NOTED  4       /* Actually entered sysidle state. */
+/*
+ * Invoked to note exit from irq or task transition to idle.  Note that
+ * usermode execution does -not- count as idle here!  After all, we want
+ * to detect full-system idle states, not RCU quiescent states and grace
+ * periods.  The caller must have disabled interrupts.
+ */
+static void rcu_sysidle_enter(struct rcu_dynticks *rdtp, int irq)
+{
+        unsigned long j;
+        /* Adjust nesting, check for fully idle. */
+        if (irq) {
+                rdtp->dynticks_idle_nesting--;
+                WARN_ON_ONCE(rdtp->dynticks_idle_nesting < 0);
+                if (rdtp->dynticks_idle_nesting != 0)
+                        return;  /* Still not fully idle. */
+        } else {
+                if ((rdtp->dynticks_idle_nesting & DYNTICK_TASK_NEST_MASK) ==
+                    DYNTICK_TASK_NEST_VALUE) {
+                        rdtp->dynticks_idle_nesting = 0;
+                } else {
+                        rdtp->dynticks_idle_nesting -= DYNTICK_TASK_NEST_VALUE;
+                        WARN_ON_ONCE(rdtp->dynticks_idle_nesting < 0);
+                        return;  /* Still not fully idle. */
+                }
+        }
+        /* Record start of fully idle period. */
+        j = jiffies;
+        ACCESS_ONCE(rdtp->dynticks_idle_jiffies) = j;
+        smp_mb__before_atomic_inc();
+        atomic_inc(&rdtp->dynticks_idle);
+        smp_mb__after_atomic_inc();
+        WARN_ON_ONCE(atomic_read(&rdtp->dynticks_idle) & 0x1);
+}
+/*
+ * Unconditionally force exit from full system-idle state.  This is
+ * invoked when a normal CPU exits idle, but must be called separately
+ * for the timekeeping CPU (tick_do_timer_cpu).  The reason for this
+ * is that the timekeeping CPU is permitted to take scheduling-clock
+ * interrupts while the system is in system-idle state, and of course
+ * rcu_sysidle_exit() has no way of distinguishing a scheduling-clock
+ * interrupt from any other type of interrupt.
+ */
+void rcu_sysidle_force_exit(void)
+{
+        int oldstate = ACCESS_ONCE(full_sysidle_state);
+        int newoldstate;
+        /*
+         * Each pass through the following loop attempts to exit full
+         * system-idle state.  If contention proves to be a problem,
+         * a trylock-based contention tree could be used here.
+         */
+        while (oldstate > RCU_SYSIDLE_SHORT) {
+                newoldstate = cmpxchg(&full_sysidle_state,
+                                      oldstate, RCU_SYSIDLE_NOT);
+                if (oldstate == newoldstate &&
+                    oldstate == RCU_SYSIDLE_FULL_NOTED) {
+                        rcu_kick_nohz_cpu(tick_do_timer_cpu);
+                        return; /* We cleared it, done! */
+                }
+                oldstate = newoldstate;
+        }
+        smp_mb(); /* Order initial oldstate fetch vs. later non-idle work. */
+}
+/*
+ * Invoked to note entry to irq or task transition from idle.  Note that
+ * usermode execution does -not- count as idle here!  The caller must
+ * have disabled interrupts.
+ */
+static void rcu_sysidle_exit(struct rcu_dynticks *rdtp, int irq)
+{
+        /* Adjust nesting, check for already non-idle. */
+        if (irq) {
+                rdtp->dynticks_idle_nesting++;
+                WARN_ON_ONCE(rdtp->dynticks_idle_nesting <= 0);
+                if (rdtp->dynticks_idle_nesting != 1)
+                        return; /* Already non-idle. */
+        } else {
+                /*
+                 * Allow for irq misnesting.  Yes, it really is possible
+                 * to enter an irq handler then never leave it, and maybe
+                 * also vice versa.  Handle both possibilities.
+                 */
+                if (rdtp->dynticks_idle_nesting & DYNTICK_TASK_NEST_MASK) {
+                        rdtp->dynticks_idle_nesting += DYNTICK_TASK_NEST_VALUE;
+                        WARN_ON_ONCE(rdtp->dynticks_idle_nesting <= 0);
+                        return; /* Already non-idle. */
+                } else {
+                        rdtp->dynticks_idle_nesting = DYNTICK_TASK_EXIT_IDLE;
+                }
+        }
+        /* Record end of idle period. */
+        smp_mb__before_atomic_inc();
+        atomic_inc(&rdtp->dynticks_idle);
+        smp_mb__after_atomic_inc();
+        WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks_idle) & 0x1));
+        /*
+         * If we are the timekeeping CPU, we are permitted to be non-idle
+         * during a system-idle state.  This must be the case, because
+         * the timekeeping CPU has to take scheduling-clock interrupts
+         * during the time that the system is transitioning to full
+         * system-idle state.  This means that the timekeeping CPU must
+         * invoke rcu_sysidle_force_exit() directly if it does anything
+         * more than take a scheduling-clock interrupt.
+         */
+        if (smp_processor_id() == tick_do_timer_cpu)
+                return;
+        /* Update system-idle state: We are clearly no longer fully idle! */
+        rcu_sysidle_force_exit();
+}
+/*
+ * Check to see if the current CPU is idle.  Note that usermode execution
+ * does not count as idle.  The caller must have disabled interrupts.
+ */
+static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
+                                  unsigned long *maxj)
+{
+        int cur;
+        unsigned long j;
+        struct rcu_dynticks *rdtp = rdp->dynticks;
+        /*
+         * If some other CPU has already reported non-idle, if this is
+         * not the flavor of RCU that tracks sysidle state, or if this
+         * is an offline or the timekeeping CPU, nothing to do.
+         */
+        if (!*isidle || rdp->rsp != rcu_sysidle_state ||
+            cpu_is_offline(rdp->cpu) || rdp->cpu == tick_do_timer_cpu)
+                return;
+        if (rcu_gp_in_progress(rdp->rsp))
+                WARN_ON_ONCE(smp_processor_id() != tick_do_timer_cpu);
+        /* Pick up current idle and NMI-nesting counter and check. */
+        cur = atomic_read(&rdtp->dynticks_idle);
+        if (cur & 0x1) {
+                *isidle = false; /* We are not idle! */
+                return;
+        }
+        smp_mb(); /* Read counters before timestamps. */
+        /* Pick up timestamps. */
+        j = ACCESS_ONCE(rdtp->dynticks_idle_jiffies);
+        /* If this CPU entered idle more recently, update maxj timestamp. */
+        if (ULONG_CMP_LT(*maxj, j))
+                *maxj = j;
+}
+/*
+ * Is this the flavor of RCU that is handling full-system idle?
+ */
+static bool is_sysidle_rcu_state(struct rcu_state *rsp)
+{
+        return rsp == rcu_sysidle_state;
+}
+/*
+ * Bind the grace-period kthread for the sysidle flavor of RCU to the
+ * timekeeping CPU.
+ */
+static void rcu_bind_gp_kthread(void)
+{
+        int cpu = ACCESS_ONCE(tick_do_timer_cpu);
+        if (cpu < 0 || cpu >= nr_cpu_ids)
+                return;
+        if (raw_smp_processor_id() != cpu)
+                set_cpus_allowed_ptr(current, cpumask_of(cpu));
+}
+/*
+ * Return a delay in jiffies based on the number of CPUs, rcu_node
+ * leaf fanout, and jiffies tick rate.  The idea is to allow larger
+ * systems more time to transition to full-idle state in order to
+ * avoid the cache thrashing that otherwise occur on the state variable.
+ * Really small systems (less than a couple of tens of CPUs) should
+ * instead use a single global atomically incremented counter, and later
+ * versions of this will automatically reconfigure themselves accordingly.
+ */
+static unsigned long rcu_sysidle_delay(void)
+{
+        if (nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL)
+                return 0;
+        return DIV_ROUND_UP(nr_cpu_ids * HZ, rcu_fanout_leaf * 1000);
+}
+/*
+ * Advance the full-system-idle state.  This is invoked when all of
+ * the non-timekeeping CPUs are idle.
+ */
+static void rcu_sysidle(unsigned long j)
+{
+        /* Check the current state. */
+        switch (ACCESS_ONCE(full_sysidle_state)) {
+        case RCU_SYSIDLE_NOT:
+                /* First time all are idle, so note a short idle period. */
+                ACCESS_ONCE(full_sysidle_state) = RCU_SYSIDLE_SHORT;
+                break;
+        case RCU_SYSIDLE_SHORT:
+                /*
+                 * Idle for a bit, time to advance to next state?
+                 * cmpxchg failure means race with non-idle, let them win.
+                 */
+                if (ULONG_CMP_GE(jiffies, j + rcu_sysidle_delay()))
+                        (void)cmpxchg(&full_sysidle_state,
+                                      RCU_SYSIDLE_SHORT, RCU_SYSIDLE_LONG);
+                break;
+        case RCU_SYSIDLE_LONG:
+                /*
+                 * Do an additional check pass before advancing to full.
+                 * cmpxchg failure means race with non-idle, let them win.
+                 */
+                if (ULONG_CMP_GE(jiffies, j + rcu_sysidle_delay()))
+                        (void)cmpxchg(&full_sysidle_state,
+                                      RCU_SYSIDLE_LONG, RCU_SYSIDLE_FULL);
+                break;
+        default:
+                break;
+        }
+}
+/*
+ * Found a non-idle non-timekeeping CPU, so kick the system-idle state
+ * back to the beginning.
+ */
+static void rcu_sysidle_cancel(void)
+{
+        smp_mb();
+        ACCESS_ONCE(full_sysidle_state) = RCU_SYSIDLE_NOT;
+}
+/*
+ * Update the sysidle state based on the results of a force-quiescent-state
+ * scan of the CPUs' dyntick-idle state.
+ */
+static void rcu_sysidle_report(struct rcu_state *rsp, int isidle,
+                               unsigned long maxj, bool gpkt)
+{
+        if (rsp != rcu_sysidle_state)
+                return;  /* Wrong flavor, ignore. */
+        if (gpkt && nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL)
+                return;  /* Running state machine from timekeeping CPU. */
+        if (isidle)
+                rcu_sysidle(maxj);    /* More idle! */
+        else
+                rcu_sysidle_cancel(); /* Idle is over. */
+}
+/*
+ * Wrapper for rcu_sysidle_report() when called from the grace-period
+ * kthread's context.
+ */
+static void rcu_sysidle_report_gp(struct rcu_state *rsp, int isidle,
+                                  unsigned long maxj)
+{
+        rcu_sysidle_report(rsp, isidle, maxj, true);
+}
+/* Callback and function for forcing an RCU grace period. */
+struct rcu_sysidle_head {
+        struct rcu_head rh;
+        int inuse;
+};
+static void rcu_sysidle_cb(struct rcu_head *rhp)
+{
+        struct rcu_sysidle_head *rshp;
+        /*
+         * The following memory barrier is needed to replace the
+         * memory barriers that would normally be in the memory
+         * allocator.
+         */
+        smp_mb();  /* grace period precedes setting inuse. */
+        rshp = container_of(rhp, struct rcu_sysidle_head, rh);
+        ACCESS_ONCE(rshp->inuse) = 0;
+}
+/*
+ * Check to see if the system is fully idle, other than the timekeeping CPU.
+ * The caller must have disabled interrupts.
+ */
+bool rcu_sys_is_idle(void)
+{
+        static struct rcu_sysidle_head rsh;
+        int rss = ACCESS_ONCE(full_sysidle_state);
+        if (WARN_ON_ONCE(smp_processor_id() != tick_do_timer_cpu))
+                return false;
+        /* Handle small-system case by doing a full scan of CPUs. */
+        if (nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL) {
+                int oldrss = rss - 1;
+                /*
+                 * One pass to advance to each state up to _FULL.
+                 * Give up if any pass fails to advance the state.
+                 */
+                while (rss < RCU_SYSIDLE_FULL && oldrss < rss) {
+                        int cpu;
+                        bool isidle = true;
+                        unsigned long maxj = jiffies - ULONG_MAX / 4;
+                        struct rcu_data *rdp;
+                        /* Scan all the CPUs looking for nonidle CPUs. */
+                        for_each_possible_cpu(cpu) {
+                                rdp = per_cpu_ptr(rcu_sysidle_state->rda, cpu);
+                                rcu_sysidle_check_cpu(rdp, &isidle, &maxj);
+                                if (!isidle)
+                                        break;
+                        }
+                        rcu_sysidle_report(rcu_sysidle_state,
+                                           isidle, maxj, false);
+                        oldrss = rss;
+                        rss = ACCESS_ONCE(full_sysidle_state);
+                }
+        }
+        /* If this is the first observation of an idle period, record it. */
+        if (rss == RCU_SYSIDLE_FULL) {
+                rss = cmpxchg(&full_sysidle_state,
+                              RCU_SYSIDLE_FULL, RCU_SYSIDLE_FULL_NOTED);
+                return rss == RCU_SYSIDLE_FULL;
+        }
+        smp_mb(); /* ensure rss load happens before later caller actions. */
+        /* If already fully idle, tell the caller (in case of races). */
+        if (rss == RCU_SYSIDLE_FULL_NOTED)
+                return true;
+        /*
+         * If we aren't there yet, and a grace period is not in flight,
+         * initiate a grace period.  Either way, tell the caller that
+         * we are not there yet.  We use an xchg() rather than an assignment
+         * to make up for the memory barriers that would otherwise be
+         * provided by the memory allocator.
+         */
+        if (nr_cpu_ids > CONFIG_NO_HZ_FULL_SYSIDLE_SMALL &&
+            !rcu_gp_in_progress(rcu_sysidle_state) &&
+            !rsh.inuse && xchg(&rsh.inuse, 1) == 0)
+                call_rcu(&rsh.rh, rcu_sysidle_cb);
+        return false;
+}
+/*
+ * Initialize dynticks sysidle state for CPUs coming online.
+ */
+static void rcu_sysidle_init_percpu_data(struct rcu_dynticks *rdtp)
+{
+        rdtp->dynticks_idle_nesting = DYNTICK_TASK_NEST_VALUE;
+}
+#else /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
+static void rcu_sysidle_enter(struct rcu_dynticks *rdtp, int irq)
+{
+}
+static void rcu_sysidle_exit(struct rcu_dynticks *rdtp, int irq)
+{
+}
+static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
+                                  unsigned long *maxj)
+{
+}
+static bool is_sysidle_rcu_state(struct rcu_state *rsp)
+{
+        return false;
+}
+static void rcu_bind_gp_kthread(void)
+{
+}
+static void rcu_sysidle_report_gp(struct rcu_state *rsp, int isidle,
+                                  unsigned long maxj)
+{
+}
+static void rcu_sysidle_init_percpu_data(struct rcu_dynticks *rdtp)
+{
+}
+#endif /* #else #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 05c39f030314..725aa067ad63 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2527,13 +2527,11 @@ void __sched schedule_preempt_disabled(void)
 */
 asmlinkage void __sched notrace preempt_schedule(void)
 {
-        struct thread_info *ti = current_thread_info();
        /*
         * If there is a non-zero preempt_count or interrupts are disabled,
         * we do not want to preempt the current task. Just return..
         */
-        if (likely(ti->preempt_count || irqs_disabled()))
+        if (likely(!preemptible()))
                return;
        do {
@@ -2677,7 +2675,7 @@ void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode,
        if (unlikely(!q))
                return;
-        if (unlikely(!nr_exclusive))
+        if (unlikely(nr_exclusive != 1))
                wake_flags = 0;
        spin_lock_irqsave(&q->lock, flags);
@@ -4964,7 +4962,8 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
                                SD_BALANCE_FORK |
                                SD_BALANCE_EXEC |
                                SD_SHARE_CPUPOWER |
-                                SD_SHARE_PKG_RESOURCES);
+                                SD_SHARE_PKG_RESOURCES |
+                                SD_PREFER_SIBLING);
                if (nr_node_ids == 1)
                        pflags &= ~SD_SERIALIZE;
        }
@@ -5133,18 +5132,23 @@ static void destroy_sched_domains(struct sched_domain *sd, int cpu)
 * two cpus are in the same cache domain, see cpus_share_cache().
 */
 DEFINE_PER_CPU(struct sched_domain *, sd_llc);
+DEFINE_PER_CPU(int, sd_llc_size);
 DEFINE_PER_CPU(int, sd_llc_id);
 static void update_top_cache_domain(int cpu)
 {
        struct sched_domain *sd;
        int id = cpu;
+        int size = 1;
        sd = highest_flag_domain(cpu, SD_SHARE_PKG_RESOURCES);
-        if (sd)
+        if (sd) {
                id = cpumask_first(sched_domain_span(sd));
+                size = cpumask_weight(sched_domain_span(sd));
+        }
        rcu_assign_pointer(per_cpu(sd_llc, cpu), sd);
+        per_cpu(sd_llc_size, cpu) = size;
        per_cpu(sd_llc_id, cpu) = id;
 }
@@ -5168,6 +5172,13 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
                        tmp->parent = parent->parent;
                        if (parent->parent)
                                parent->parent->child = tmp;
+                        /*
+                         * Transfer SD_PREFER_SIBLING down in case of a
+                         * degenerate parent; the spans match for this
+                         * so the property transfers.
+                         */
+                        if (parent->flags & SD_PREFER_SIBLING)
+                                tmp->flags |= SD_PREFER_SIBLING;
                        destroy_sched_domain(parent, cpu);
                } else
                        tmp = tmp->parent;
@@ -6234,8 +6245,9 @@ match1:
                ;
        }
+        n = ndoms_cur;
        if (doms_new == NULL) {
-                ndoms_cur = 0;
+                n = 0;
                doms_new = &fallback_doms;
                cpumask_andnot(doms_new[0], cpu_active_mask, cpu_isolated_map);
                WARN_ON_ONCE(dattr_new);
@@ -6243,7 +6255,7 @@ match1:
        /* Build new domains */
        for (i = 0; i < ndoms_new; i++) {
-                for (j = 0; j < ndoms_cur && !new_topology; j++) {
+                for (j = 0; j < n && !new_topology; j++) {
                        if (cpumask_equal(doms_new[i], doms_cur[j])
                            && dattrs_equal(dattr_new, i, dattr_cur, j))
                                goto match2;
@@ -6815,7 +6827,7 @@ void sched_move_task(struct task_struct *tsk)
        if (unlikely(running))
                tsk->sched_class->put_prev_task(rq, tsk);
-        tg = container_of(task_subsys_state_check(tsk, cpu_cgroup_subsys_id,
+        tg = container_of(task_css_check(tsk, cpu_cgroup_subsys_id,
                                lockdep_is_held(&tsk->sighand->siglock)),
                          struct task_group, css);
        tg = autogroup_task_group(tsk, tg);
@@ -7137,23 +7149,22 @@ int sched_rt_handler(struct ctl_table *table, int write,
 #ifdef CONFIG_CGROUP_SCHED
-/* return corresponding task_group object of a cgroup */
+static inline struct task_group *css_tg(struct cgroup_subsys_state *css)
-static inline struct task_group *cgroup_tg(struct cgroup *cgrp)
 {
-        return container_of(cgroup_subsys_state(cgrp, cpu_cgroup_subsys_id),
+        return css ? container_of(css, struct task_group, css) : NULL;
-                            struct task_group, css);
 }
-static struct cgroup_subsys_state *cpu_cgroup_css_alloc(struct cgroup *cgrp)
+static struct cgroup_subsys_state *
+cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
 {
-        struct task_group *tg, *parent;
+        struct task_group *parent = css_tg(parent_css);
+        struct task_group *tg;
-        if (!cgrp->parent) {
+        if (!parent) {
                /* This is early initialization for the top cgroup */
                return &root_task_group.css;
        }
-        parent = cgroup_tg(cgrp->parent);
        tg = sched_create_group(parent);
        if (IS_ERR(tg))
                return ERR_PTR(-ENOMEM);
@@ -7161,41 +7172,38 @@ static struct cgroup_subsys_state *cpu_cgroup_css_alloc(struct cgroup *cgrp)
        return &tg->css;
 }
-static int cpu_cgroup_css_online(struct cgroup *cgrp)
+static int cpu_cgroup_css_online(struct cgroup_subsys_state *css)
 {
-        struct task_group *tg = cgroup_tg(cgrp);
+        struct task_group *tg = css_tg(css);
-        struct task_group *parent;
+        struct task_group *parent = css_tg(css_parent(css));
-        if (!cgrp->parent)
-                return 0;
-        parent = cgroup_tg(cgrp->parent);
+        if (parent)
-        sched_online_group(tg, parent);
+                sched_online_group(tg, parent);
        return 0;
 }
-static void cpu_cgroup_css_free(struct cgroup *cgrp)
+static void cpu_cgroup_css_free(struct cgroup_subsys_state *css)
 {
-        struct task_group *tg = cgroup_tg(cgrp);
+        struct task_group *tg = css_tg(css);
        sched_destroy_group(tg);
 }
-static void cpu_cgroup_css_offline(struct cgroup *cgrp)
+static void cpu_cgroup_css_offline(struct cgroup_subsys_state *css)
 {
-        struct task_group *tg = cgroup_tg(cgrp);
+        struct task_group *tg = css_tg(css);
        sched_offline_group(tg);
 }
-static int cpu_cgroup_can_attach(struct cgroup *cgrp,
+static int cpu_cgroup_can_attach(struct cgroup_subsys_state *css,
                                 struct cgroup_taskset *tset)
 {
        struct task_struct *task;
-        cgroup_taskset_for_each(task, cgrp, tset) {
+        cgroup_taskset_for_each(task, css, tset) {
 #ifdef CONFIG_RT_GROUP_SCHED
-                if (!sched_rt_can_attach(cgroup_tg(cgrp), task))
+                if (!sched_rt_can_attach(css_tg(css), task))
                        return -EINVAL;
 #else
                /* We don't support RT-tasks being in separate groups */
@@ -7206,18 +7214,18 @@ static int cpu_cgroup_can_attach(struct cgroup *cgrp,
        return 0;
 }
-static void cpu_cgroup_attach(struct cgroup *cgrp,
+static void cpu_cgroup_attach(struct cgroup_subsys_state *css,
                              struct cgroup_taskset *tset)
 {
        struct task_struct *task;
-        cgroup_taskset_for_each(task, cgrp, tset)
+        cgroup_taskset_for_each(task, css, tset)
                sched_move_task(task);
 }
-static void
+static void cpu_cgroup_exit(struct cgroup_subsys_state *css,
-cpu_cgroup_exit(struct cgroup *cgrp, struct cgroup *old_cgrp,
+                            struct cgroup_subsys_state *old_css,
-                struct task_struct *task)
+                            struct task_struct *task)
 {
        /*
         * cgroup_exit() is called in the copy_process() failure path.
@@ -7231,15 +7239,16 @@ cpu_cgroup_exit(struct cgroup *cgrp, struct cgroup *old_cgrp,
 }
 #ifdef CONFIG_FAIR_GROUP_SCHED
-static int cpu_shares_write_u64(struct cgroup *cgrp, struct cftype *cftype,
+static int cpu_shares_write_u64(struct cgroup_subsys_state *css,
-                                u64 shareval)
+                                struct cftype *cftype, u64 shareval)
 {
-        return sched_group_set_shares(cgroup_tg(cgrp), scale_load(shareval));
+        return sched_group_set_shares(css_tg(css), scale_load(shareval));
 }
-static u64 cpu_shares_read_u64(struct cgroup *cgrp, struct cftype *cft)
+static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css,
+                               struct cftype *cft)
 {
-        struct task_group *tg = cgroup_tg(cgrp);
+        struct task_group *tg = css_tg(css);
        return (u64) scale_load_down(tg->shares);
 }
@@ -7361,26 +7370,28 @@ long tg_get_cfs_period(struct task_group *tg)
        return cfs_period_us;
 }
-static s64 cpu_cfs_quota_read_s64(struct cgroup *cgrp, struct cftype *cft)
+static s64 cpu_cfs_quota_read_s64(struct cgroup_subsys_state *css,
+                                  struct cftype *cft)
 {
-        return tg_get_cfs_quota(cgroup_tg(cgrp));
+        return tg_get_cfs_quota(css_tg(css));
 }
-static int cpu_cfs_quota_write_s64(struct cgroup *cgrp, struct cftype *cftype,
+static int cpu_cfs_quota_write_s64(struct cgroup_subsys_state *css,
-                                s64 cfs_quota_us)
+                                   struct cftype *cftype, s64 cfs_quota_us)
 {
-        return tg_set_cfs_quota(cgroup_tg(cgrp), cfs_quota_us);
+        return tg_set_cfs_quota(css_tg(css), cfs_quota_us);
 }
-static u64 cpu_cfs_period_read_u64(struct cgroup *cgrp, struct cftype *cft)
+static u64 cpu_cfs_period_read_u64(struct cgroup_subsys_state *css,
+                                   struct cftype *cft)
 {
-        return tg_get_cfs_period(cgroup_tg(cgrp));
+        return tg_get_cfs_period(css_tg(css));
 }
-static int cpu_cfs_period_write_u64(struct cgroup *cgrp, struct cftype *cftype,
+static int cpu_cfs_period_write_u64(struct cgroup_subsys_state *css,
-                                u64 cfs_period_us)
+                                    struct cftype *cftype, u64 cfs_period_us)
 {
-        return tg_set_cfs_period(cgroup_tg(cgrp), cfs_period_us);
+        return tg_set_cfs_period(css_tg(css), cfs_period_us);
 }
 struct cfs_schedulable_data {
@@ -7461,10 +7472,10 @@ static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota)
        return ret;
 }
-static int cpu_stats_show(struct cgroup *cgrp, struct cftype *cft,
+static int cpu_stats_show(struct cgroup_subsys_state *css, struct cftype *cft,
                struct cgroup_map_cb *cb)
 {
-        struct task_group *tg = cgroup_tg(cgrp);
+        struct task_group *tg = css_tg(css);
        struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
        cb->fill(cb, "nr_periods", cfs_b->nr_periods);
@@ -7477,26 +7488,28 @@ static int cpu_stats_show(struct cgroup *cgrp, struct cftype *cft,
 #endif /* CONFIG_FAIR_GROUP_SCHED */
 #ifdef CONFIG_RT_GROUP_SCHED
-static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft,
+static int cpu_rt_runtime_write(struct cgroup_subsys_state *css,
-                                s64 val)
+                                struct cftype *cft, s64 val)
 {
-        return sched_group_set_rt_runtime(cgroup_tg(cgrp), val);
+        return sched_group_set_rt_runtime(css_tg(css), val);
 }
-static s64 cpu_rt_runtime_read(struct cgroup *cgrp, struct cftype *cft)
+static s64 cpu_rt_runtime_read(struct cgroup_subsys_state *css,
+                               struct cftype *cft)
 {
-        return sched_group_rt_runtime(cgroup_tg(cgrp));
+        return sched_group_rt_runtime(css_tg(css));
 }
-static int cpu_rt_period_write_uint(struct cgroup *cgrp, struct cftype *cftype,
+static int cpu_rt_period_write_uint(struct cgroup_subsys_state *css,
-                u64 rt_period_us)
+                                    struct cftype *cftype, u64 rt_period_us)
 {
-        return sched_group_set_rt_period(cgroup_tg(cgrp), rt_period_us);
+        return sched_group_set_rt_period(css_tg(css), rt_period_us);
 }
-static u64 cpu_rt_period_read_uint(struct cgroup *cgrp, struct cftype *cft)
+static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css,
+                                   struct cftype *cft)
 {
-        return sched_group_rt_period(cgroup_tg(cgrp));
+        return sched_group_rt_period(css_tg(css));
 }
 #endif /* CONFIG_RT_GROUP_SCHED */
diff --git a/kernel/sched/cpuacct.c b/kernel/sched/cpuacct.c
index dbb7e2cd95eb..f64722ff0299 100644
--- a/kernel/sched/cpuacct.c
+++ b/kernel/sched/cpuacct.c
@@ -33,30 +33,20 @@ struct cpuacct {
        struct kernel_cpustat __percpu *cpustat;
 };
-/* return cpu accounting group corresponding to this container */
+static inline struct cpuacct *css_ca(struct cgroup_subsys_state *css)
-static inline struct cpuacct *cgroup_ca(struct cgroup *cgrp)
 {
-        return container_of(cgroup_subsys_state(cgrp, cpuacct_subsys_id),
+        return css ? container_of(css, struct cpuacct, css) : NULL;
-                            struct cpuacct, css);
 }
 /* return cpu accounting group to which this task belongs */
 static inline struct cpuacct *task_ca(struct task_struct *tsk)
 {
-        return container_of(task_subsys_state(tsk, cpuacct_subsys_id),
+        return css_ca(task_css(tsk, cpuacct_subsys_id));
-                            struct cpuacct, css);
-}
-static inline struct cpuacct *__parent_ca(struct cpuacct *ca)
-{
-        return cgroup_ca(ca->css.cgroup->parent);
 }
 static inline struct cpuacct *parent_ca(struct cpuacct *ca)
 {
-        if (!ca->css.cgroup->parent)
+        return css_ca(css_parent(&ca->css));
-                return NULL;
-        return cgroup_ca(ca->css.cgroup->parent);
 }
 static DEFINE_PER_CPU(u64, root_cpuacct_cpuusage);
@@ -66,11 +56,12 @@ static struct cpuacct root_cpuacct = {
 };
 /* create a new cpu accounting group */
-static struct cgroup_subsys_state *cpuacct_css_alloc(struct cgroup *cgrp)
+static struct cgroup_subsys_state *
+cpuacct_css_alloc(struct cgroup_subsys_state *parent_css)
 {
        struct cpuacct *ca;
-        if (!cgrp->parent)
+        if (!parent_css)
                return &root_cpuacct.css;
        ca = kzalloc(sizeof(*ca), GFP_KERNEL);
@@ -96,9 +87,9 @@ out:
 }
 /* destroy an existing cpu accounting group */
-static void cpuacct_css_free(struct cgroup *cgrp)
+static void cpuacct_css_free(struct cgroup_subsys_state *css)
 {
-        struct cpuacct *ca = cgroup_ca(cgrp);
+        struct cpuacct *ca = css_ca(css);
        free_percpu(ca->cpustat);
        free_percpu(ca->cpuusage);
@@ -141,9 +132,9 @@ static void cpuacct_cpuusage_write(struct cpuacct *ca, int cpu, u64 val)
 }
 /* return total cpu usage (in nanoseconds) of a group */
-static u64 cpuusage_read(struct cgroup *cgrp, struct cftype *cft)
+static u64 cpuusage_read(struct cgroup_subsys_state *css, struct cftype *cft)
 {
-        struct cpuacct *ca = cgroup_ca(cgrp);
+        struct cpuacct *ca = css_ca(css);
        u64 totalcpuusage = 0;
        int i;
@@ -153,10 +144,10 @@ static u64 cpuusage_read(struct cgroup *cgrp, struct cftype *cft)
        return totalcpuusage;
 }
-static int cpuusage_write(struct cgroup *cgrp, struct cftype *cftype,
+static int cpuusage_write(struct cgroup_subsys_state *css, struct cftype *cft,
-                                                                u64 reset)
+                          u64 reset)
 {
-        struct cpuacct *ca = cgroup_ca(cgrp);
+        struct cpuacct *ca = css_ca(css);
        int err = 0;
        int i;
@@ -172,10 +163,10 @@ out:
        return err;
 }
-static int cpuacct_percpu_seq_read(struct cgroup *cgroup, struct cftype *cft,
+static int cpuacct_percpu_seq_read(struct cgroup_subsys_state *css,
-                                   struct seq_file *m)
+                                   struct cftype *cft, struct seq_file *m)
 {
-        struct cpuacct *ca = cgroup_ca(cgroup);
+        struct cpuacct *ca = css_ca(css);
        u64 percpu;
        int i;
@@ -192,10 +183,10 @@ static const char * const cpuacct_stat_desc[] = {
        [CPUACCT_STAT_SYSTEM] = "system",
 };
-static int cpuacct_stats_show(struct cgroup *cgrp, struct cftype *cft,
+static int cpuacct_stats_show(struct cgroup_subsys_state *css,
-                              struct cgroup_map_cb *cb)
+                              struct cftype *cft, struct cgroup_map_cb *cb)
 {
-        struct cpuacct *ca = cgroup_ca(cgrp);
+        struct cpuacct *ca = css_ca(css);
        int cpu;
        s64 val = 0;
@@ -281,7 +272,7 @@ void cpuacct_account_field(struct task_struct *p, int index, u64 val)
        while (ca != &root_cpuacct) {
                kcpustat = this_cpu_ptr(ca->cpustat);
                kcpustat->cpustat[index] += val;
-                ca = __parent_ca(ca);
+                ca = parent_ca(ca);
        }
        rcu_read_unlock();
 }
diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c
index a7959e05a9d5..ace34f95e200 100644
--- a/kernel/sched/cputime.c
+++ b/kernel/sched/cputime.c
@@ -121,7 +121,7 @@ static inline void task_group_account_field(struct task_struct *p, int index,
         * is the only cgroup, then nothing else should be necessary.
         *
         */
-        __get_cpu_var(kernel_cpustat).cpustat[index] += tmp;
+        __this_cpu_add(kernel_cpustat.cpustat[index], tmp);
        cpuacct_account_field(p, index, tmp);
 }
@@ -378,11 +378,8 @@ static inline void irqtime_account_process_tick(struct task_struct *p, int user_
 #ifdef CONFIG_VIRT_CPU_ACCOUNTING
 #ifndef __ARCH_HAS_VTIME_TASK_SWITCH
-void vtime_task_switch(struct task_struct *prev)
+void vtime_common_task_switch(struct task_struct *prev)
 {
-        if (!vtime_accounting_enabled())
-                return;
        if (is_idle_task(prev))
                vtime_account_idle(prev);
        else
@@ -404,11 +401,8 @@ void vtime_task_switch(struct task_struct *prev)
 * vtime_account().
 */
 #ifndef __ARCH_HAS_VTIME_ACCOUNT
-void vtime_account_irq_enter(struct task_struct *tsk)
+void vtime_common_account_irq_enter(struct task_struct *tsk)
 {
-        if (!vtime_accounting_enabled())
-                return;
        if (!in_interrupt()) {
                /*
                 * If we interrupted user, context_tracking_in_user()
@@ -428,7 +422,7 @@ void vtime_account_irq_enter(struct task_struct *tsk)
        }
        vtime_account_system(tsk);
 }
-EXPORT_SYMBOL_GPL(vtime_account_irq_enter);
+EXPORT_SYMBOL_GPL(vtime_common_account_irq_enter);
 #endif /* __ARCH_HAS_VTIME_ACCOUNT */
 #endif /* CONFIG_VIRT_CPU_ACCOUNTING */
@@ -559,12 +553,6 @@ static void cputime_adjust(struct task_cputime *curr,
 {
        cputime_t rtime, stime, utime, total;
-        if (vtime_accounting_enabled()) {
-                *ut = curr->utime;
-                *st = curr->stime;
-                return;
-        }
        stime = curr->stime;
        total = stime + curr->utime;
@@ -664,23 +652,17 @@ static void __vtime_account_system(struct task_struct *tsk)
 void vtime_account_system(struct task_struct *tsk)
 {
-        if (!vtime_accounting_enabled())
-                return;
        write_seqlock(&tsk->vtime_seqlock);
        __vtime_account_system(tsk);
        write_sequnlock(&tsk->vtime_seqlock);
 }
-void vtime_account_irq_exit(struct task_struct *tsk)
+void vtime_gen_account_irq_exit(struct task_struct *tsk)
 {
-        if (!vtime_accounting_enabled())
-                return;
        write_seqlock(&tsk->vtime_seqlock);
+        __vtime_account_system(tsk);
        if (context_tracking_in_user())
                tsk->vtime_snap_whence = VTIME_USER;
-        __vtime_account_system(tsk);
        write_sequnlock(&tsk->vtime_seqlock);
 }
@@ -688,12 +670,8 @@ void vtime_account_user(struct task_struct *tsk)
 {
        cputime_t delta_cpu;
-        if (!vtime_accounting_enabled())
-                return;
-        delta_cpu = get_vtime_delta(tsk);
        write_seqlock(&tsk->vtime_seqlock);
+        delta_cpu = get_vtime_delta(tsk);
        tsk->vtime_snap_whence = VTIME_SYS;
        account_user_time(tsk, delta_cpu, cputime_to_scaled(delta_cpu));
        write_sequnlock(&tsk->vtime_seqlock);
@@ -701,22 +679,27 @@ void vtime_account_user(struct task_struct *tsk)
 void vtime_user_enter(struct task_struct *tsk)
 {
-        if (!vtime_accounting_enabled())
-                return;
        write_seqlock(&tsk->vtime_seqlock);
-        tsk->vtime_snap_whence = VTIME_USER;
        __vtime_account_system(tsk);
+        tsk->vtime_snap_whence = VTIME_USER;
        write_sequnlock(&tsk->vtime_seqlock);
 }
 void vtime_guest_enter(struct task_struct *tsk)
 {
+        /*
+         * The flags must be updated under the lock with
+         * the vtime_snap flush and update.
+         * That enforces a right ordering and update sequence
+         * synchronization against the reader (task_gtime())
+         * that can thus safely catch up with a tickless delta.
+         */
        write_seqlock(&tsk->vtime_seqlock);
        __vtime_account_system(tsk);
        current->flags |= PF_VCPU;
        write_sequnlock(&tsk->vtime_seqlock);
 }
+EXPORT_SYMBOL_GPL(vtime_guest_enter);
 void vtime_guest_exit(struct task_struct *tsk)
 {
@@ -725,6 +708,7 @@ void vtime_guest_exit(struct task_struct *tsk)
        current->flags &= ~PF_VCPU;
        write_sequnlock(&tsk->vtime_seqlock);
 }
+EXPORT_SYMBOL_GPL(vtime_guest_exit);
 void vtime_account_idle(struct task_struct *tsk)
 {
@@ -733,11 +717,6 @@ void vtime_account_idle(struct task_struct *tsk)
        account_idle_time(delta_cpu);
 }
-bool vtime_accounting_enabled(void)
-{
-        return context_tracking_active();
-}
 void arch_vtime_task_switch(struct task_struct *prev)
 {
        write_seqlock(&prev->vtime_seqlock);
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 68f1609ca149..7f0a5e6cdae0 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -3018,6 +3018,23 @@ static unsigned long cpu_avg_load_per_task(int cpu)
        return 0;
 }
+static void record_wakee(struct task_struct *p)
+{
+        /*
+         * Rough decay (wiping) for cost saving, don't worry
+         * about the boundary, really active task won't care
+         * about the loss.
+         */
+        if (jiffies > current->wakee_flip_decay_ts + HZ) {
+                current->wakee_flips = 0;
+                current->wakee_flip_decay_ts = jiffies;
+        }
+        if (current->last_wakee != p) {
+                current->last_wakee = p;
+                current->wakee_flips++;
+        }
+}
 static void task_waking_fair(struct task_struct *p)
 {
@@ -3038,6 +3055,7 @@ static void task_waking_fair(struct task_struct *p)
 #endif
        se->vruntime -= min_vruntime;
+        record_wakee(p);
 }
 #ifdef CONFIG_FAIR_GROUP_SCHED
@@ -3156,6 +3174,28 @@ static inline unsigned long effective_load(struct task_group *tg, int cpu,
 #endif
+static int wake_wide(struct task_struct *p)
+{
+        int factor = this_cpu_read(sd_llc_size);
+        /*
+         * Yeah, it's the switching-frequency, could means many wakee or
+         * rapidly switch, use factor here will just help to automatically
+         * adjust the loose-degree, so bigger node will lead to more pull.
+         */
+        if (p->wakee_flips > factor) {
+                /*
+                 * wakee is somewhat hot, it needs certain amount of cpu
+                 * resource, so if waker is far more hot, prefer to leave
+                 * it alone.
+                 */
+                if (current->wakee_flips > (factor * p->wakee_flips))
+                        return 1;
+        }
+        return 0;
+}
 static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync)
 {
        s64 this_load, load;
@@ -3165,6 +3205,13 @@ static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync)
        unsigned long weight;
        int balanced;
+        /*
+         * If we wake multiple tasks be careful to not bounce
+         * ourselves around too much.
+         */
+        if (wake_wide(p))
+                return 0;
        idx       = sd->wake_idx;
        this_cpu  = smp_processor_id();
        prev_cpu  = task_cpu(p);
@@ -4172,47 +4219,48 @@ static void update_blocked_averages(int cpu)
 }
 /*
- * Compute the cpu's hierarchical load factor for each task group.
+ * Compute the hierarchical load factor for cfs_rq and all its ascendants.
 * This needs to be done in a top-down fashion because the load of a child
 * group is a fraction of its parents load.
 */
-static int tg_load_down(struct task_group *tg, void *data)
+static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq)
-{
-        unsigned long load;
-        long cpu = (long)data;
-        if (!tg->parent) {
-                load = cpu_rq(cpu)->avg.load_avg_contrib;
-        } else {
-                load = tg->parent->cfs_rq[cpu]->h_load;
-                load = div64_ul(load * tg->se[cpu]->avg.load_avg_contrib,
-                                tg->parent->cfs_rq[cpu]->runnable_load_avg + 1);
-        }
-        tg->cfs_rq[cpu]->h_load = load;
-        return 0;
-}
-static void update_h_load(long cpu)
 {
-        struct rq *rq = cpu_rq(cpu);
+        struct rq *rq = rq_of(cfs_rq);
+        struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)];
        unsigned long now = jiffies;
+        unsigned long load;
-        if (rq->h_load_throttle == now)
+        if (cfs_rq->last_h_load_update == now)
                return;
-        rq->h_load_throttle = now;
+        cfs_rq->h_load_next = NULL;
+        for_each_sched_entity(se) {
+                cfs_rq = cfs_rq_of(se);
+                cfs_rq->h_load_next = se;
+                if (cfs_rq->last_h_load_update == now)
+                        break;
+        }
-        rcu_read_lock();
+        if (!se) {
-        walk_tg_tree(tg_load_down, tg_nop, (void *)cpu);
+                cfs_rq->h_load = rq->avg.load_avg_contrib;
-        rcu_read_unlock();
+                cfs_rq->last_h_load_update = now;
+        }
+        while ((se = cfs_rq->h_load_next) != NULL) {
+                load = cfs_rq->h_load;
+                load = div64_ul(load * se->avg.load_avg_contrib,
+                                cfs_rq->runnable_load_avg + 1);
+                cfs_rq = group_cfs_rq(se);
+                cfs_rq->h_load = load;
+                cfs_rq->last_h_load_update = now;
+        }
 }
 static unsigned long task_h_load(struct task_struct *p)
 {
        struct cfs_rq *cfs_rq = task_cfs_rq(p);
+        update_cfs_rq_h_load(cfs_rq);
        return div64_ul(p->se.avg.load_avg_contrib * cfs_rq->h_load,
                        cfs_rq->runnable_load_avg + 1);
 }
@@ -4221,10 +4269,6 @@ static inline void update_blocked_averages(int cpu)
 {
 }
-static inline void update_h_load(long cpu)
-{
-}
 static unsigned long task_h_load(struct task_struct *p)
 {
        return p->se.avg.load_avg_contrib;
@@ -4233,50 +4277,56 @@ static unsigned long task_h_load(struct task_struct *p)
 /********** Helpers for find_busiest_group ************************/
 /*
- * sd_lb_stats - Structure to store the statistics of a sched_domain
- *              during load balancing.
- */
-struct sd_lb_stats {
-        struct sched_group *busiest; /* Busiest group in this sd */
-        struct sched_group *this;  /* Local group in this sd */
-        unsigned long total_load;  /* Total load of all groups in sd */
-        unsigned long total_pwr;   /*   Total power of all groups in sd */
-        unsigned long avg_load;    /* Average load across all groups in sd */
-        /** Statistics of this group */
-        unsigned long this_load;
-        unsigned long this_load_per_task;
-        unsigned long this_nr_running;
-        unsigned long this_has_capacity;
-        unsigned int  this_idle_cpus;
-        /* Statistics of the busiest group */
-        unsigned int  busiest_idle_cpus;
-        unsigned long max_load;
-        unsigned long busiest_load_per_task;
-        unsigned long busiest_nr_running;
-        unsigned long busiest_group_capacity;
-        unsigned long busiest_has_capacity;
-        unsigned int  busiest_group_weight;
-        int group_imb; /* Is there imbalance in this sd */
-};
-/*
 * sg_lb_stats - stats of a sched_group required for load_balancing
 */
 struct sg_lb_stats {
        unsigned long avg_load; /*Avg load across the CPUs of the group */
        unsigned long group_load; /* Total load over the CPUs of the group */
-        unsigned long sum_nr_running; /* Nr tasks running in the group */
        unsigned long sum_weighted_load; /* Weighted load of group's tasks */
-        unsigned long group_capacity;
+        unsigned long load_per_task;
-        unsigned long idle_cpus;
+        unsigned long group_power;
-        unsigned long group_weight;
+        unsigned int sum_nr_running; /* Nr tasks running in the group */
+        unsigned int group_capacity;
+        unsigned int idle_cpus;
+        unsigned int group_weight;
        int group_imb; /* Is there an imbalance in the group ? */
        int group_has_capacity; /* Is there extra capacity in the group? */
 };
+/*
+ * sd_lb_stats - Structure to store the statistics of a sched_domain
+ *               during load balancing.
+ */
+struct sd_lb_stats {
+        struct sched_group *busiest;    /* Busiest group in this sd */
+        struct sched_group *local;      /* Local group in this sd */
+        unsigned long total_load;       /* Total load of all groups in sd */
+        unsigned long total_pwr;        /* Total power of all groups in sd */
+        unsigned long avg_load; /* Average load across all groups in sd */
+        struct sg_lb_stats busiest_stat;/* Statistics of the busiest group */
+        struct sg_lb_stats local_stat;  /* Statistics of the local group */
+};
+static inline void init_sd_lb_stats(struct sd_lb_stats *sds)
+{
+        /*
+         * Skimp on the clearing to avoid duplicate work. We can avoid clearing
+         * local_stat because update_sg_lb_stats() does a full clear/assignment.
+         * We must however clear busiest_stat::avg_load because
+         * update_sd_pick_busiest() reads this before assignment.
+         */
+        *sds = (struct sd_lb_stats){
+                .busiest = NULL,
+                .local = NULL,
+                .total_load = 0UL,
+                .total_pwr = 0UL,
+                .busiest_stat = {
+                        .avg_load = 0UL,
+                },
+        };
+}
 /**
 * get_sd_load_idx - Obtain the load index for a given sched domain.
 * @sd: The sched_domain whose load_idx is to be obtained.
@@ -4460,33 +4510,99 @@ fix_small_capacity(struct sched_domain *sd, struct sched_group *group)
        return 0;
 }
+/*
+ * Group imbalance indicates (and tries to solve) the problem where balancing
+ * groups is inadequate due to tsk_cpus_allowed() constraints.
+ *
+ * Imagine a situation of two groups of 4 cpus each and 4 tasks each with a
+ * cpumask covering 1 cpu of the first group and 3 cpus of the second group.
+ * Something like:
+ *
+ *      { 0 1 2 3 } { 4 5 6 7 }
+ *              *     * * *
+ *
+ * If we were to balance group-wise we'd place two tasks in the first group and
+ * two tasks in the second group. Clearly this is undesired as it will overload
+ * cpu 3 and leave one of the cpus in the second group unused.
+ *
+ * The current solution to this issue is detecting the skew in the first group
+ * by noticing it has a cpu that is overloaded while the remaining cpus are
+ * idle -- or rather, there's a distinct imbalance in the cpus; see
+ * sg_imbalanced().
+ *
+ * When this is so detected; this group becomes a candidate for busiest; see
+ * update_sd_pick_busiest(). And calculcate_imbalance() and
+ * find_busiest_group() avoid some of the usual balance conditional to allow it
+ * to create an effective group imbalance.
+ *
+ * This is a somewhat tricky proposition since the next run might not find the
+ * group imbalance and decide the groups need to be balanced again. A most
+ * subtle and fragile situation.
+ */
+struct sg_imb_stats {
+        unsigned long max_nr_running, min_nr_running;
+        unsigned long max_cpu_load, min_cpu_load;
+};
+static inline void init_sg_imb_stats(struct sg_imb_stats *sgi)
+{
+        sgi->max_cpu_load = sgi->max_nr_running = 0UL;
+        sgi->min_cpu_load = sgi->min_nr_running = ~0UL;
+}
+static inline void
+update_sg_imb_stats(struct sg_imb_stats *sgi,
+                    unsigned long load, unsigned long nr_running)
+{
+        if (load > sgi->max_cpu_load)
+                sgi->max_cpu_load = load;
+        if (sgi->min_cpu_load > load)
+                sgi->min_cpu_load = load;
+        if (nr_running > sgi->max_nr_running)
+                sgi->max_nr_running = nr_running;
+        if (sgi->min_nr_running > nr_running)
+                sgi->min_nr_running = nr_running;
+}
+static inline int
+sg_imbalanced(struct sg_lb_stats *sgs, struct sg_imb_stats *sgi)
+{
+        /*
+         * Consider the group unbalanced when the imbalance is larger
+         * than the average weight of a task.
+         *
+         * APZ: with cgroup the avg task weight can vary wildly and
+         *      might not be a suitable number - should we keep a
+         *      normalized nr_running number somewhere that negates
+         *      the hierarchy?
+         */
+        if ((sgi->max_cpu_load - sgi->min_cpu_load) >= sgs->load_per_task &&
+            (sgi->max_nr_running - sgi->min_nr_running) > 1)
+                return 1;
+        return 0;
+}
 /**
 * update_sg_lb_stats - Update sched_group's statistics for load balancing.
 * @env: The load balancing environment.
 * @group: sched_group whose statistics are to be updated.
 * @load_idx: Load index of sched_domain of this_cpu for load calc.
 * @local_group: Does group contain this_cpu.
- * @balance: Should we balance.
 * @sgs: variable to hold the statistics for this group.
 */
 static inline void update_sg_lb_stats(struct lb_env *env,
                        struct sched_group *group, int load_idx,
-                        int local_group, int *balance, struct sg_lb_stats *sgs)
+                        int local_group, struct sg_lb_stats *sgs)
 {
-        unsigned long nr_running, max_nr_running, min_nr_running;
+        struct sg_imb_stats sgi;
-        unsigned long load, max_cpu_load, min_cpu_load;
+        unsigned long nr_running;
-        unsigned int balance_cpu = -1, first_idle_cpu = 0;
+        unsigned long load;
-        unsigned long avg_load_per_task = 0;
        int i;
-        if (local_group)
+        init_sg_imb_stats(&sgi);
-                balance_cpu = group_balance_cpu(group);
-        /* Tally up the load of all CPUs in the group */
-        max_cpu_load = 0;
-        min_cpu_load = ~0UL;
-        max_nr_running = 0;
-        min_nr_running = ~0UL;
        for_each_cpu_and(i, sched_group_cpus(group), env->cpus) {
                struct rq *rq = cpu_rq(i);
@@ -4495,24 +4611,10 @@ static inline void update_sg_lb_stats(struct lb_env *env,
                /* Bias balancing toward cpus of our domain */
                if (local_group) {
-                        if (idle_cpu(i) && !first_idle_cpu &&
-                                        cpumask_test_cpu(i, sched_group_mask(group))) {
-                                first_idle_cpu = 1;
-                                balance_cpu = i;
-                        }
                        load = target_load(i, load_idx);
                } else {
                        load = source_load(i, load_idx);
-                        if (load > max_cpu_load)
+                        update_sg_imb_stats(&sgi, load, nr_running);
-                                max_cpu_load = load;
-                        if (min_cpu_load > load)
-                                min_cpu_load = load;
-                        if (nr_running > max_nr_running)
-                                max_nr_running = nr_running;
-                        if (min_nr_running > nr_running)
-                                min_nr_running = nr_running;
                }
                sgs->group_load += load;
@@ -4522,46 +4624,25 @@ static inline void update_sg_lb_stats(struct lb_env *env,
                        sgs->idle_cpus++;
        }
-        /*
+        if (local_group && (env->idle != CPU_NEWLY_IDLE ||
-         * First idle cpu or the first cpu(busiest) in this sched group
+                        time_after_eq(jiffies, group->sgp->next_update)))
-         * is eligible for doing load balancing at this and above
+                update_group_power(env->sd, env->dst_cpu);
-         * domains. In the newly idle case, we will allow all the cpu's
-         * to do the newly idle load balance.
-         */
-        if (local_group) {
-                if (env->idle != CPU_NEWLY_IDLE) {
-                        if (balance_cpu != env->dst_cpu) {
-                                *balance = 0;
-                                return;
-                        }
-                        update_group_power(env->sd, env->dst_cpu);
-                } else if (time_after_eq(jiffies, group->sgp->next_update))
-                        update_group_power(env->sd, env->dst_cpu);
-        }
        /* Adjust by relative CPU power of the group */
-        sgs->avg_load = (sgs->group_load*SCHED_POWER_SCALE) / group->sgp->power;
+        sgs->group_power = group->sgp->power;
+        sgs->avg_load = (sgs->group_load*SCHED_POWER_SCALE) / sgs->group_power;
-        /*
-         * Consider the group unbalanced when the imbalance is larger
-         * than the average weight of a task.
-         *
-         * APZ: with cgroup the avg task weight can vary wildly and
-         *      might not be a suitable number - should we keep a
-         *      normalized nr_running number somewhere that negates
-         *      the hierarchy?
-         */
        if (sgs->sum_nr_running)
-                avg_load_per_task = sgs->sum_weighted_load / sgs->sum_nr_running;
+                sgs->load_per_task = sgs->sum_weighted_load / sgs->sum_nr_running;
+        sgs->group_imb = sg_imbalanced(sgs, &sgi);
-        if ((max_cpu_load - min_cpu_load) >= avg_load_per_task &&
+        sgs->group_capacity =
-            (max_nr_running - min_nr_running) > 1)
+                DIV_ROUND_CLOSEST(sgs->group_power, SCHED_POWER_SCALE);
-                sgs->group_imb = 1;
-        sgs->group_capacity = DIV_ROUND_CLOSEST(group->sgp->power,
-                                                SCHED_POWER_SCALE);
        if (!sgs->group_capacity)
                sgs->group_capacity = fix_small_capacity(env->sd, group);
        sgs->group_weight = group->group_weight;
        if (sgs->group_capacity > sgs->sum_nr_running)
@@ -4586,7 +4667,7 @@ static bool update_sd_pick_busiest(struct lb_env *env,
                                   struct sched_group *sg,
                                   struct sg_lb_stats *sgs)
 {
-        if (sgs->avg_load <= sds->max_load)
+        if (sgs->avg_load <= sds->busiest_stat.avg_load)
                return false;
        if (sgs->sum_nr_running > sgs->group_capacity)
@@ -4619,11 +4700,11 @@ static bool update_sd_pick_busiest(struct lb_env *env,
 * @sds: variable to hold the statistics for this sched_domain.
 */
 static inline void update_sd_lb_stats(struct lb_env *env,
-                                        int *balance, struct sd_lb_stats *sds)
+                                        struct sd_lb_stats *sds)
 {
        struct sched_domain *child = env->sd->child;
        struct sched_group *sg = env->sd->groups;
-        struct sg_lb_stats sgs;
+        struct sg_lb_stats tmp_sgs;
        int load_idx, prefer_sibling = 0;
        if (child && child->flags & SD_PREFER_SIBLING)
@@ -4632,17 +4713,17 @@ static inline void update_sd_lb_stats(struct lb_env *env,
        load_idx = get_sd_load_idx(env->sd, env->idle);
        do {
+                struct sg_lb_stats *sgs = &tmp_sgs;
                int local_group;
                local_group = cpumask_test_cpu(env->dst_cpu, sched_group_cpus(sg));
-                memset(&sgs, 0, sizeof(sgs));
+                if (local_group) {
-                update_sg_lb_stats(env, sg, load_idx, local_group, balance, &sgs);
+                        sds->local = sg;
+                        sgs = &sds->local_stat;
-                if (local_group && !(*balance))
+                }
-                        return;
-                sds->total_load += sgs.group_load;
+                memset(sgs, 0, sizeof(*sgs));
-                sds->total_pwr += sg->sgp->power;
+                update_sg_lb_stats(env, sg, load_idx, local_group, sgs);
                /*
                 * In case the child domain prefers tasks go to siblings
@@ -4654,26 +4735,17 @@ static inline void update_sd_lb_stats(struct lb_env *env,
                 * heaviest group when it is already under-utilized (possible
                 * with a large weight task outweighs the tasks on the system).
                 */
-                if (prefer_sibling && !local_group && sds->this_has_capacity)
+                if (prefer_sibling && !local_group &&
-                        sgs.group_capacity = min(sgs.group_capacity, 1UL);
+                                sds->local && sds->local_stat.group_has_capacity)
+                        sgs->group_capacity = min(sgs->group_capacity, 1U);
-                if (local_group) {
+                /* Now, start updating sd_lb_stats */
-                        sds->this_load = sgs.avg_load;
+                sds->total_load += sgs->group_load;
-                        sds->this = sg;
+                sds->total_pwr += sgs->group_power;
-                        sds->this_nr_running = sgs.sum_nr_running;
-                        sds->this_load_per_task = sgs.sum_weighted_load;
+                if (!local_group && update_sd_pick_busiest(env, sds, sg, sgs)) {
-                        sds->this_has_capacity = sgs.group_has_capacity;
-                        sds->this_idle_cpus = sgs.idle_cpus;
-                } else if (update_sd_pick_busiest(env, sds, sg, &sgs)) {
-                        sds->max_load = sgs.avg_load;
                        sds->busiest = sg;
-                        sds->busiest_nr_running = sgs.sum_nr_running;
+                        sds->busiest_stat = *sgs;
-                        sds->busiest_idle_cpus = sgs.idle_cpus;
-                        sds->busiest_group_capacity = sgs.group_capacity;
-                        sds->busiest_load_per_task = sgs.sum_weighted_load;
-                        sds->busiest_has_capacity = sgs.group_has_capacity;
-                        sds->busiest_group_weight = sgs.group_weight;
-                        sds->group_imb = sgs.group_imb;
                }
                sg = sg->next;
@@ -4718,7 +4790,8 @@ static int check_asym_packing(struct lb_env *env, struct sd_lb_stats *sds)
                return 0;
        env->imbalance = DIV_ROUND_CLOSEST(
-                sds->max_load * sds->busiest->sgp->power, SCHED_POWER_SCALE);
+                sds->busiest_stat.avg_load * sds->busiest_stat.group_power,
+                SCHED_POWER_SCALE);
        return 1;
 }
@@ -4736,24 +4809,23 @@ void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds)
        unsigned long tmp, pwr_now = 0, pwr_move = 0;
        unsigned int imbn = 2;
        unsigned long scaled_busy_load_per_task;
+        struct sg_lb_stats *local, *busiest;
-        if (sds->this_nr_running) {
+        local = &sds->local_stat;
-                sds->this_load_per_task /= sds->this_nr_running;
+        busiest = &sds->busiest_stat;
-                if (sds->busiest_load_per_task >
-                                sds->this_load_per_task)
+        if (!local->sum_nr_running)
-                        imbn = 1;
+                local->load_per_task = cpu_avg_load_per_task(env->dst_cpu);
-        } else {
+        else if (busiest->load_per_task > local->load_per_task)
-                sds->this_load_per_task =
+                imbn = 1;
-                        cpu_avg_load_per_task(env->dst_cpu);
-        }
-        scaled_busy_load_per_task = sds->busiest_load_per_task
+        scaled_busy_load_per_task =
-                                         * SCHED_POWER_SCALE;
+                (busiest->load_per_task * SCHED_POWER_SCALE) /
-        scaled_busy_load_per_task /= sds->busiest->sgp->power;
+                busiest->group_power;
-        if (sds->max_load - sds->this_load + scaled_busy_load_per_task >=
+        if (busiest->avg_load - local->avg_load + scaled_busy_load_per_task >=
-                        (scaled_busy_load_per_task * imbn)) {
+            (scaled_busy_load_per_task * imbn)) {
-                env->imbalance = sds->busiest_load_per_task;
+                env->imbalance = busiest->load_per_task;
                return;
        }
@@ -4763,34 +4835,37 @@ void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds)
         * moving them.
         */
-        pwr_now += sds->busiest->sgp->power *
+        pwr_now += busiest->group_power *
-                        min(sds->busiest_load_per_task, sds->max_load);
+                        min(busiest->load_per_task, busiest->avg_load);
-        pwr_now += sds->this->sgp->power *
+        pwr_now += local->group_power *
-                        min(sds->this_load_per_task, sds->this_load);
+                        min(local->load_per_task, local->avg_load);
        pwr_now /= SCHED_POWER_SCALE;
        /* Amount of load we'd subtract */
-        tmp = (sds->busiest_load_per_task * SCHED_POWER_SCALE) /
+        tmp = (busiest->load_per_task * SCHED_POWER_SCALE) /
-                sds->busiest->sgp->power;
+                busiest->group_power;
-        if (sds->max_load > tmp)
+        if (busiest->avg_load > tmp) {
-                pwr_move += sds->busiest->sgp->power *
+                pwr_move += busiest->group_power *
-                        min(sds->busiest_load_per_task, sds->max_load - tmp);
+                            min(busiest->load_per_task,
+                                busiest->avg_load - tmp);
+        }
        /* Amount of load we'd add */
-        if (sds->max_load * sds->busiest->sgp->power <
+        if (busiest->avg_load * busiest->group_power <
-                sds->busiest_load_per_task * SCHED_POWER_SCALE)
+            busiest->load_per_task * SCHED_POWER_SCALE) {
-                tmp = (sds->max_load * sds->busiest->sgp->power) /
+                tmp = (busiest->avg_load * busiest->group_power) /
-                        sds->this->sgp->power;
+                      local->group_power;
-        else
+        } else {
-                tmp = (sds->busiest_load_per_task * SCHED_POWER_SCALE) /
+                tmp = (busiest->load_per_task * SCHED_POWER_SCALE) /
-                        sds->this->sgp->power;
+                      local->group_power;
-        pwr_move += sds->this->sgp->power *
+        }
-                        min(sds->this_load_per_task, sds->this_load + tmp);
+        pwr_move += local->group_power *
+                    min(local->load_per_task, local->avg_load + tmp);
        pwr_move /= SCHED_POWER_SCALE;
        /* Move if we gain throughput */
        if (pwr_move > pwr_now)
-                env->imbalance = sds->busiest_load_per_task;
+                env->imbalance = busiest->load_per_task;
 }
 /**
@@ -4802,11 +4877,18 @@ void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds)
 static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds)
 {
        unsigned long max_pull, load_above_capacity = ~0UL;
+        struct sg_lb_stats *local, *busiest;
-        sds->busiest_load_per_task /= sds->busiest_nr_running;
+        local = &sds->local_stat;
-        if (sds->group_imb) {
+        busiest = &sds->busiest_stat;
-                sds->busiest_load_per_task =
-                        min(sds->busiest_load_per_task, sds->avg_load);
+        if (busiest->group_imb) {
+                /*
+                 * In the group_imb case we cannot rely on group-wide averages
+                 * to ensure cpu-load equilibrium, look at wider averages. XXX
+                 */
+                busiest->load_per_task =
+                        min(busiest->load_per_task, sds->avg_load);
        }
        /*
@@ -4814,21 +4896,22 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s
         * max load less than avg load(as we skip the groups at or below
         * its cpu_power, while calculating max_load..)
         */
-        if (sds->max_load < sds->avg_load) {
+        if (busiest->avg_load < sds->avg_load) {
                env->imbalance = 0;
                return fix_small_imbalance(env, sds);
        }
-        if (!sds->group_imb) {
+        if (!busiest->group_imb) {
                /*
                 * Don't want to pull so many tasks that a group would go idle.
+                 * Except of course for the group_imb case, since then we might
+                 * have to drop below capacity to reach cpu-load equilibrium.
                 */
-                load_above_capacity = (sds->busiest_nr_running -
+                load_above_capacity =
-                                                sds->busiest_group_capacity);
+                        (busiest->sum_nr_running - busiest->group_capacity);
                load_above_capacity *= (SCHED_LOAD_SCALE * SCHED_POWER_SCALE);
+                load_above_capacity /= busiest->group_power;
-                load_above_capacity /= sds->busiest->sgp->power;
        }
        /*
@@ -4838,15 +4921,14 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s
         * we also don't want to reduce the group load below the group capacity
         * (so that we can implement power-savings policies etc). Thus we look
         * for the minimum possible imbalance.
-         * Be careful of negative numbers as they'll appear as very large values
-         * with unsigned longs.
         */
-        max_pull = min(sds->max_load - sds->avg_load, load_above_capacity);
+        max_pull = min(busiest->avg_load - sds->avg_load, load_above_capacity);
        /* How much load to actually move to equalise the imbalance */
-        env->imbalance = min(max_pull * sds->busiest->sgp->power,
+        env->imbalance = min(
-                (sds->avg_load - sds->this_load) * sds->this->sgp->power)
+                max_pull * busiest->group_power,
-                        / SCHED_POWER_SCALE;
+                (sds->avg_load - local->avg_load) * local->group_power
+        ) / SCHED_POWER_SCALE;
        /*
         * if *imbalance is less than the average load per runnable task
@@ -4854,9 +4936,8 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s
         * a think about bumping its value to force at least one task to be
         * moved
         */
-        if (env->imbalance < sds->busiest_load_per_task)
+        if (env->imbalance < busiest->load_per_task)
                return fix_small_imbalance(env, sds);
 }
 /******* find_busiest_group() helpers end here *********************/
@@ -4872,69 +4953,62 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s
 * to restore balance.
 *
 * @env: The load balancing environment.
- * @balance: Pointer to a variable indicating if this_cpu
- *      is the appropriate cpu to perform load balancing at this_level.
 *
 * Return:      - The busiest group if imbalance exists.
 *              - If no imbalance and user has opted for power-savings balance,
 *                 return the least loaded group whose CPUs can be
 *                 put to idle by rebalancing its tasks onto our group.
 */
-static struct sched_group *
+static struct sched_group *find_busiest_group(struct lb_env *env)
-find_busiest_group(struct lb_env *env, int *balance)
 {
+        struct sg_lb_stats *local, *busiest;
        struct sd_lb_stats sds;
-        memset(&sds, 0, sizeof(sds));
+        init_sd_lb_stats(&sds);
        /*
         * Compute the various statistics relavent for load balancing at
         * this level.
         */
-        update_sd_lb_stats(env, balance, &sds);
+        update_sd_lb_stats(env, &sds);
+        local = &sds.local_stat;
-        /*
+        busiest = &sds.busiest_stat;
-         * this_cpu is not the appropriate cpu to perform load balancing at
-         * this level.
-         */
-        if (!(*balance))
-                goto ret;
        if ((env->idle == CPU_IDLE || env->idle == CPU_NEWLY_IDLE) &&
            check_asym_packing(env, &sds))
                return sds.busiest;
        /* There is no busy sibling group to pull tasks from */
-        if (!sds.busiest || sds.busiest_nr_running == 0)
+        if (!sds.busiest || busiest->sum_nr_running == 0)
                goto out_balanced;
        sds.avg_load = (SCHED_POWER_SCALE * sds.total_load) / sds.total_pwr;
        /*
         * If the busiest group is imbalanced the below checks don't
-         * work because they assumes all things are equal, which typically
+         * work because they assume all things are equal, which typically
         * isn't true due to cpus_allowed constraints and the like.
         */
-        if (sds.group_imb)
+        if (busiest->group_imb)
                goto force_balance;
        /* SD_BALANCE_NEWIDLE trumps SMP nice when underutilized */
-        if (env->idle == CPU_NEWLY_IDLE && sds.this_has_capacity &&
+        if (env->idle == CPU_NEWLY_IDLE && local->group_has_capacity &&
-                        !sds.busiest_has_capacity)
+            !busiest->group_has_capacity)
                goto force_balance;
        /*
         * If the local group is more busy than the selected busiest group
         * don't try and pull any tasks.
         */
-        if (sds.this_load >= sds.max_load)
+        if (local->avg_load >= busiest->avg_load)
                goto out_balanced;
        /*
         * Don't pull any tasks if this group is already above the domain
         * average load.
         */
-        if (sds.this_load >= sds.avg_load)
+        if (local->avg_load >= sds.avg_load)
                goto out_balanced;
        if (env->idle == CPU_IDLE) {
@@ -4944,15 +5018,16 @@ find_busiest_group(struct lb_env *env, int *balance)
                 * there is no imbalance between this and busiest group
                 * wrt to idle cpu's, it is balanced.
                 */
-                if ((sds.this_idle_cpus <= sds.busiest_idle_cpus + 1) &&
+                if ((local->idle_cpus < busiest->idle_cpus) &&
-                    sds.busiest_nr_running <= sds.busiest_group_weight)
+                    busiest->sum_nr_running <= busiest->group_weight)
                        goto out_balanced;
        } else {
                /*
                 * In the CPU_NEWLY_IDLE, CPU_NOT_IDLE cases, use
                 * imbalance_pct to be conservative.
                 */
-                if (100 * sds.max_load <= env->sd->imbalance_pct * sds.this_load)
+                if (100 * busiest->avg_load <=
+                                env->sd->imbalance_pct * local->avg_load)
                        goto out_balanced;
        }
@@ -4962,7 +5037,6 @@ force_balance:
        return sds.busiest;
 out_balanced:
-ret:
        env->imbalance = 0;
        return NULL;
 }
@@ -4974,10 +5048,10 @@ static struct rq *find_busiest_queue(struct lb_env *env,
                                     struct sched_group *group)
 {
        struct rq *busiest = NULL, *rq;
-        unsigned long max_load = 0;
+        unsigned long busiest_load = 0, busiest_power = 1;
        int i;
-        for_each_cpu(i, sched_group_cpus(group)) {
+        for_each_cpu_and(i, sched_group_cpus(group), env->cpus) {
                unsigned long power = power_of(i);
                unsigned long capacity = DIV_ROUND_CLOSEST(power,
                                                           SCHED_POWER_SCALE);
@@ -4986,9 +5060,6 @@ static struct rq *find_busiest_queue(struct lb_env *env,
                if (!capacity)
                        capacity = fix_small_capacity(env->sd, group);
-                if (!cpumask_test_cpu(i, env->cpus))
-                        continue;
                rq = cpu_rq(i);
                wl = weighted_cpuload(i);
@@ -5004,11 +5075,15 @@ static struct rq *find_busiest_queue(struct lb_env *env,
                 * the weighted_cpuload() scaled with the cpu power, so that
                 * the load can be moved away from the cpu that is potentially
                 * running at a lower capacity.
+                 *
+                 * Thus we're looking for max(wl_i / power_i), crosswise
+                 * multiplication to rid ourselves of the division works out
+                 * to: wl_i * power_j > wl_j * power_i;  where j is our
+                 * previous maximum.
                 */
-                wl = (wl * SCHED_POWER_SCALE) / power;
+                if (wl * busiest_power > busiest_load * power) {
+                        busiest_load = wl;
-                if (wl > max_load) {
+                        busiest_power = power;
-                        max_load = wl;
                        busiest = rq;
                }
        }
@@ -5045,13 +5120,47 @@ static int need_active_balance(struct lb_env *env)
 static int active_load_balance_cpu_stop(void *data);
+static int should_we_balance(struct lb_env *env)
+{
+        struct sched_group *sg = env->sd->groups;
+        struct cpumask *sg_cpus, *sg_mask;
+        int cpu, balance_cpu = -1;
+        /*
+         * In the newly idle case, we will allow all the cpu's
+         * to do the newly idle load balance.
+         */
+        if (env->idle == CPU_NEWLY_IDLE)
+                return 1;
+        sg_cpus = sched_group_cpus(sg);
+        sg_mask = sched_group_mask(sg);
+        /* Try to find first idle cpu */
+        for_each_cpu_and(cpu, sg_cpus, env->cpus) {
+                if (!cpumask_test_cpu(cpu, sg_mask) || !idle_cpu(cpu))
+                        continue;
+                balance_cpu = cpu;
+                break;
+        }
+        if (balance_cpu == -1)
+                balance_cpu = group_balance_cpu(sg);
+        /*
+         * First idle cpu or the first cpu(busiest) in this sched group
+         * is eligible for doing load balancing at this and above domains.
+         */
+        return balance_cpu != env->dst_cpu;
+}
 /*
 * Check this_cpu to ensure it is balanced within domain. Attempt to move
 * tasks if there is an imbalance.
 */
 static int load_balance(int this_cpu, struct rq *this_rq,
                        struct sched_domain *sd, enum cpu_idle_type idle,
-                        int *balance)
+                        int *continue_balancing)
 {
        int ld_moved, cur_ld_moved, active_balance = 0;
        struct sched_group *group;
@@ -5081,11 +5190,12 @@ static int load_balance(int this_cpu, struct rq *this_rq,
        schedstat_inc(sd, lb_count[idle]);
 redo:
-        group = find_busiest_group(&env, balance);
+        if (!should_we_balance(&env)) {
+                *continue_balancing = 0;
-        if (*balance == 0)
                goto out_balanced;
+        }
+        group = find_busiest_group(&env);
        if (!group) {
                schedstat_inc(sd, lb_nobusyg[idle]);
                goto out_balanced;
@@ -5114,7 +5224,6 @@ redo:
                env.src_rq    = busiest;
                env.loop_max  = min(sysctl_sched_nr_migrate, busiest->nr_running);
-                update_h_load(env.src_cpu);
 more_balance:
                local_irq_save(flags);
                double_rq_lock(env.dst_rq, busiest);
@@ -5298,7 +5407,7 @@ void idle_balance(int this_cpu, struct rq *this_rq)
        rcu_read_lock();
        for_each_domain(this_cpu, sd) {
                unsigned long interval;
-                int balance = 1;
+                int continue_balancing = 1;
                if (!(sd->flags & SD_LOAD_BALANCE))
                        continue;
@@ -5306,7 +5415,8 @@ void idle_balance(int this_cpu, struct rq *this_rq)
                if (sd->flags & SD_BALANCE_NEWIDLE) {
                        /* If we've pulled tasks over stop searching: */
                        pulled_task = load_balance(this_cpu, this_rq,
-                                                   sd, CPU_NEWLY_IDLE, &balance);
+                                                   sd, CPU_NEWLY_IDLE,
+                                                   &continue_balancing);
                }
                interval = msecs_to_jiffies(sd->balance_interval);
@@ -5544,7 +5654,7 @@ void update_max_interval(void)
 */
 static void rebalance_domains(int cpu, enum cpu_idle_type idle)
 {
-        int balance = 1;
+        int continue_balancing = 1;
        struct rq *rq = cpu_rq(cpu);
        unsigned long interval;
        struct sched_domain *sd;
@@ -5576,7 +5686,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
                }
                if (time_after_eq(jiffies, sd->last_balance + interval)) {
-                        if (load_balance(cpu, rq, sd, idle, &balance)) {
+                        if (load_balance(cpu, rq, sd, idle, &continue_balancing)) {
                                /*
                                 * The LBF_SOME_PINNED logic could have changed
                                 * env->dst_cpu, so we can't know our idle
@@ -5599,7 +5709,7 @@ out:
                 * CPU in our sched group which is doing load balancing more
                 * actively.
                 */
-                if (!balance)
+                if (!continue_balancing)
                        break;
        }
        rcu_read_unlock();
@@ -5895,11 +6005,9 @@ static void switched_from_fair(struct rq *rq, struct task_struct *p)
        * and ensure we don't carry in an old decay_count if we
        * switch back.
        */
-        if (p->se.avg.decay_count) {
+        if (se->avg.decay_count) {
-                struct cfs_rq *cfs_rq = cfs_rq_of(&p->se);
+                __synchronize_entity_decay(se);
-                __synchronize_entity_decay(&p->se);
+                subtract_blocked_load_contrib(cfs_rq, se->avg.load_avg_contrib);
-                subtract_blocked_load_contrib(cfs_rq,
-                                p->se.avg.load_avg_contrib);
        }
 #endif
 }
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index ef0a7b2439dd..b3c5653e1dca 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -285,7 +285,6 @@ struct cfs_rq {
        /* Required to track per-cpu representation of a task_group */
        u32 tg_runnable_contrib;
        unsigned long tg_load_contrib;
-#endif /* CONFIG_FAIR_GROUP_SCHED */
        /*
         *   h_load = weight * f(tg)
@@ -294,6 +293,9 @@ struct cfs_rq {
         * this group.
         */
        unsigned long h_load;
+        u64 last_h_load_update;
+        struct sched_entity *h_load_next;
+#endif /* CONFIG_FAIR_GROUP_SCHED */
 #endif /* CONFIG_SMP */
 #ifdef CONFIG_FAIR_GROUP_SCHED
@@ -429,9 +431,6 @@ struct rq {
 #ifdef CONFIG_FAIR_GROUP_SCHED
        /* list of leaf cfs_rq on this cpu: */
        struct list_head leaf_cfs_rq_list;
-#ifdef CONFIG_SMP
-        unsigned long h_load_throttle;
-#endif /* CONFIG_SMP */
 #endif /* CONFIG_FAIR_GROUP_SCHED */
 #ifdef CONFIG_RT_GROUP_SCHED
@@ -595,6 +594,7 @@ static inline struct sched_domain *highest_flag_domain(int cpu, int flag)
 }
 DECLARE_PER_CPU(struct sched_domain *, sd_llc);
+DECLARE_PER_CPU(int, sd_llc_size);
 DECLARE_PER_CPU(int, sd_llc_id);
 struct sched_group_power {
@@ -665,9 +665,9 @@ extern int group_balance_cpu(struct sched_group *sg);
 /*
 * Return the group to which this tasks belongs.
 *
- * We cannot use task_subsys_state() and friends because the cgroup
+ * We cannot use task_css() and friends because the cgroup subsystem
- * subsystem changes that value before the cgroup_subsys::attach() method
+ * changes that value before the cgroup_subsys::attach() method is called,
- * is called, therefore we cannot pin it and might observe the wrong value.
+ * therefore we cannot pin it and might observe the wrong value.
 *
 * The same is true for autogroup's p->signal->autogroup->tg, the autogroup
 * core changes this before calling sched_move_task().
diff --git a/kernel/smp.c b/kernel/smp.c
index fe9f773d7114..449b707fc20d 100644
--- a/kernel/smp.c
+++ b/kernel/smp.c
@@ -186,25 +186,13 @@ void generic_smp_call_function_single_interrupt(void)
        while (!list_empty(&list)) {
                struct call_single_data *csd;
-                unsigned int csd_flags;
                csd = list_entry(list.next, struct call_single_data, list);
                list_del(&csd->list);
-                /*
-                 * 'csd' can be invalid after this call if flags == 0
-                 * (when called through generic_exec_single()),
-                 * so save them away before making the call:
-                 */
-                csd_flags = csd->flags;
                csd->func(csd->info);
-                /*
+                csd_unlock(csd);
-                 * Unlocked CSDs are valid through generic_exec_single():
-                 */
-                if (csd_flags & CSD_FLAG_LOCK)
-                        csd_unlock(csd);
        }
 }
@@ -278,8 +266,6 @@ EXPORT_SYMBOL(smp_call_function_single);
 * @wait: If true, wait until function has completed.
 *
 * Returns 0 on success, else a negative status code (if no cpus were online).
- * Note that @wait will be implicitly turned on in case of allocation failures,
- * since we fall back to on-stack allocation.
 *
 * Selection preference:
 *      1) current cpu if in @mask
diff --git a/kernel/time/Kconfig b/kernel/time/Kconfig
index 70f27e89012b..2b62fe86f9ec 100644
--- a/kernel/time/Kconfig
+++ b/kernel/time/Kconfig
@@ -105,7 +105,6 @@ config NO_HZ_FULL
        select RCU_USER_QS
        select RCU_NOCB_CPU
        select VIRT_CPU_ACCOUNTING_GEN
-        select CONTEXT_TRACKING_FORCE
        select IRQ_WORK
        help
         Adaptively try to shutdown the tick whenever possible, even when
@@ -134,6 +133,56 @@ config NO_HZ_FULL_ALL
         Note the boot CPU will still be kept outside the range to
         handle the timekeeping duty.
+config NO_HZ_FULL_SYSIDLE
+        bool "Detect full-system idle state for full dynticks system"
+        depends on NO_HZ_FULL
+        default n
+        help
+         At least one CPU must keep the scheduling-clock tick running for
+         timekeeping purposes whenever there is a non-idle CPU, where
+         "non-idle" also includes dynticks CPUs as long as they are
+         running non-idle tasks.  Because the underlying adaptive-tick
+         support cannot distinguish between all CPUs being idle and
+         all CPUs each running a single task in dynticks mode, the
+         underlying support simply ensures that there is always a CPU
+         handling the scheduling-clock tick, whether or not all CPUs
+         are idle.  This Kconfig option enables scalable detection of
+         the all-CPUs-idle state, thus allowing the scheduling-clock
+         tick to be disabled when all CPUs are idle.  Note that scalable
+         detection of the all-CPUs-idle state means that larger systems
+         will be slower to declare the all-CPUs-idle state.
+         Say Y if you would like to help debug all-CPUs-idle detection.
+         Say N if you are unsure.
+config NO_HZ_FULL_SYSIDLE_SMALL
+        int "Number of CPUs above which large-system approach is used"
+        depends on NO_HZ_FULL_SYSIDLE
+        range 1 NR_CPUS
+        default 8
+        help
+         The full-system idle detection mechanism takes a lazy approach
+         on large systems, as is required to attain decent scalability.
+         However, on smaller systems, scalability is not anywhere near as
+         large a concern as is energy efficiency.  The sysidle subsystem
+         therefore uses a fast but non-scalable algorithm for small
+         systems and a lazier but scalable algorithm for large systems.
+         This Kconfig parameter defines the number of CPUs in the largest
+         system that will be considered to be "small".
+         The default value will be fine in most cases.  Battery-powered
+         systems that (1) enable NO_HZ_FULL_SYSIDLE, (2) have larger
+         numbers of CPUs, and (3) are suffering from battery-lifetime
+         problems due to long sysidle latencies might wish to experiment
+         with larger values for this Kconfig parameter.  On the other
+         hand, they might be even better served by disabling NO_HZ_FULL
+         entirely, given that NO_HZ_FULL is intended for HPC and
+         real-time workloads that at present do not tend to be run on
+         battery-powered systems.
+         Take the default if you are unsure.
 config NO_HZ
        bool "Old Idle dynticks config"
        depends on !ARCH_USES_GETTIMEOFFSET && GENERIC_CLOCKEVENTS
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index e8a1516cc0a3..3612fc77f834 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -23,6 +23,7 @@
 #include <linux/irq_work.h>
 #include <linux/posix-timers.h>
 #include <linux/perf_event.h>
+#include <linux/context_tracking.h>
 #include <asm/irq_regs.h>
@@ -148,8 +149,8 @@ static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs)
 }
 #ifdef CONFIG_NO_HZ_FULL
-static cpumask_var_t nohz_full_mask;
+cpumask_var_t tick_nohz_full_mask;
-bool have_nohz_full_mask;
+bool tick_nohz_full_running;
 static bool can_stop_full_tick(void)
 {
@@ -182,7 +183,7 @@ static bool can_stop_full_tick(void)
                 * Don't allow the user to think they can get
                 * full NO_HZ with this machine.
                 */
-                WARN_ONCE(have_nohz_full_mask,
+                WARN_ONCE(tick_nohz_full_running,
                          "NO_HZ FULL will not work with unstable sched clock");
                return false;
        }
@@ -197,7 +198,7 @@ static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now);
 * Re-evaluate the need for the tick on the current CPU
 * and restart it if necessary.
 */
-void tick_nohz_full_check(void)
+void __tick_nohz_full_check(void)
 {
        struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
@@ -211,7 +212,7 @@ void tick_nohz_full_check(void)
 static void nohz_full_kick_work_func(struct irq_work *work)
 {
-        tick_nohz_full_check();
+        __tick_nohz_full_check();
 }
 static DEFINE_PER_CPU(struct irq_work, nohz_full_kick_work) = {
@@ -230,7 +231,7 @@ void tick_nohz_full_kick(void)
 static void nohz_full_kick_ipi(void *info)
 {
-        tick_nohz_full_check();
+        __tick_nohz_full_check();
 }
 /*
@@ -239,12 +240,13 @@ static void nohz_full_kick_ipi(void *info)
 */
 void tick_nohz_full_kick_all(void)
 {
-        if (!have_nohz_full_mask)
+        if (!tick_nohz_full_running)
                return;
        preempt_disable();
-        smp_call_function_many(nohz_full_mask,
+        smp_call_function_many(tick_nohz_full_mask,
                               nohz_full_kick_ipi, NULL, false);
+        tick_nohz_full_kick();
        preempt_enable();
 }
@@ -253,7 +255,7 @@ void tick_nohz_full_kick_all(void)
 * It might need the tick due to per task/process properties:
 * perf events, posix cpu timers, ...
 */
-void tick_nohz_task_switch(struct task_struct *tsk)
+void __tick_nohz_task_switch(struct task_struct *tsk)
 {
        unsigned long flags;
@@ -269,31 +271,23 @@ out:
        local_irq_restore(flags);
 }
-int tick_nohz_full_cpu(int cpu)
-{
-        if (!have_nohz_full_mask)
-                return 0;
-        return cpumask_test_cpu(cpu, nohz_full_mask);
-}
 /* Parse the boot-time nohz CPU list from the kernel parameters. */
 static int __init tick_nohz_full_setup(char *str)
 {
        int cpu;
-        alloc_bootmem_cpumask_var(&nohz_full_mask);
+        alloc_bootmem_cpumask_var(&tick_nohz_full_mask);
-        if (cpulist_parse(str, nohz_full_mask) < 0) {
+        if (cpulist_parse(str, tick_nohz_full_mask) < 0) {
                pr_warning("NOHZ: Incorrect nohz_full cpumask\n");
                return 1;
        }
        cpu = smp_processor_id();
-        if (cpumask_test_cpu(cpu, nohz_full_mask)) {
+        if (cpumask_test_cpu(cpu, tick_nohz_full_mask)) {
                pr_warning("NO_HZ: Clearing %d from nohz_full range for timekeeping\n", cpu);
-                cpumask_clear_cpu(cpu, nohz_full_mask);
+                cpumask_clear_cpu(cpu, tick_nohz_full_mask);
        }
-        have_nohz_full_mask = true;
+        tick_nohz_full_running = true;
        return 1;
 }
@@ -311,7 +305,7 @@ static int tick_nohz_cpu_down_callback(struct notifier_block *nfb,
                 * If we handle the timekeeping duty for full dynticks CPUs,
                 * we can't safely shutdown that CPU.
                 */
-                if (have_nohz_full_mask && tick_do_timer_cpu == cpu)
+                if (tick_nohz_full_running && tick_do_timer_cpu == cpu)
                        return NOTIFY_BAD;
                break;
        }
@@ -330,31 +324,34 @@ static int tick_nohz_init_all(void)
        int err = -1;
 #ifdef CONFIG_NO_HZ_FULL_ALL
-        if (!alloc_cpumask_var(&nohz_full_mask, GFP_KERNEL)) {
+        if (!alloc_cpumask_var(&tick_nohz_full_mask, GFP_KERNEL)) {
                pr_err("NO_HZ: Can't allocate full dynticks cpumask\n");
                return err;
        }
        err = 0;
-        cpumask_setall(nohz_full_mask);
+        cpumask_setall(tick_nohz_full_mask);
-        cpumask_clear_cpu(smp_processor_id(), nohz_full_mask);
+        cpumask_clear_cpu(smp_processor_id(), tick_nohz_full_mask);
-        have_nohz_full_mask = true;
+        tick_nohz_full_running = true;
 #endif
        return err;
 }
 void __init tick_nohz_init(void)
 {
-        if (!have_nohz_full_mask) {
+        int cpu;
+        if (!tick_nohz_full_running) {
                if (tick_nohz_init_all() < 0)
                        return;
        }
+        for_each_cpu(cpu, tick_nohz_full_mask)
+                context_tracking_cpu_set(cpu);
        cpu_notifier(tick_nohz_cpu_down_callback, 0);
-        cpulist_scnprintf(nohz_full_buf, sizeof(nohz_full_buf), nohz_full_mask);
+        cpulist_scnprintf(nohz_full_buf, sizeof(nohz_full_buf), tick_nohz_full_mask);
        pr_info("NO_HZ: Full dynticks CPUs: %s.\n", nohz_full_buf);
 }
-#else
-#define have_nohz_full_mask (0)
 #endif
 /*
@@ -732,7 +729,7 @@ static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)
                return false;
        }
-        if (have_nohz_full_mask) {
+        if (tick_nohz_full_enabled()) {
                /*
                 * Keep the tick alive to guarantee timekeeping progression
                 * if there are full dynticks CPUs around
diff --git a/kernel/time/timer_list.c b/kernel/time/timer_list.c
index 3bdf28323012..61ed862cdd37 100644
--- a/kernel/time/timer_list.c
+++ b/kernel/time/timer_list.c
@@ -265,10 +265,9 @@ static inline void timer_list_header(struct seq_file *m, u64 now)
 static int timer_list_show(struct seq_file *m, void *v)
 {
        struct timer_list_iter *iter = v;
-        u64 now = ktime_to_ns(ktime_get());
        if (iter->cpu == -1 && !iter->second_pass)
-                timer_list_header(m, now);
+                timer_list_header(m, iter->now);
        else if (!iter->second_pass)
                print_cpu(m, iter->cpu, iter->now);
 #ifdef CONFIG_GENERIC_CLOCKEVENTS
@@ -298,33 +297,41 @@ void sysrq_timer_list_show(void)
        return;
 }
-static void *timer_list_start(struct seq_file *file, loff_t *offset)
+static void *move_iter(struct timer_list_iter *iter, loff_t offset)
 {
-        struct timer_list_iter *iter = file->private;
+        for (; offset; offset--) {
+                iter->cpu = cpumask_next(iter->cpu, cpu_online_mask);
-        if (!*offset) {
+                if (iter->cpu >= nr_cpu_ids) {
-                iter->cpu = -1;
-                iter->now = ktime_to_ns(ktime_get());
-        } else if (iter->cpu >= nr_cpu_ids) {
 #ifdef CONFIG_GENERIC_CLOCKEVENTS
-                if (!iter->second_pass) {
+                        if (!iter->second_pass) {
-                        iter->cpu = -1;
+                                iter->cpu = -1;
-                        iter->second_pass = true;
+                                iter->second_pass = true;
-                } else
+                        } else
-                        return NULL;
+                                return NULL;
 #else
-                return NULL;
+                        return NULL;
 #endif
+                }
        }
        return iter;
 }
+static void *timer_list_start(struct seq_file *file, loff_t *offset)
+{
+        struct timer_list_iter *iter = file->private;
+        if (!*offset)
+                iter->now = ktime_to_ns(ktime_get());
+        iter->cpu = -1;
+        iter->second_pass = false;
+        return move_iter(iter, *offset);
+}
 static void *timer_list_next(struct seq_file *file, void *v, loff_t *offset)
 {
        struct timer_list_iter *iter = file->private;
-        iter->cpu = cpumask_next(iter->cpu, cpu_online_mask);
        ++*offset;
-        return timer_list_start(file, offset);
+        return move_iter(iter, 1);
 }
 static void timer_list_stop(struct seq_file *seq, void *v)
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index afaae41b0a02..fe39acd4c1aa 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -1022,6 +1022,9 @@ extern struct list_head ftrace_events;
 extern const char *__start___trace_bprintk_fmt[];
 extern const char *__stop___trace_bprintk_fmt[];
+extern const char *__start___tracepoint_str[];
+extern const char *__stop___tracepoint_str[];
 void trace_printk_init_buffers(void);
 void trace_printk_start_comm(void);
 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set);
diff --git a/kernel/trace/trace_printk.c b/kernel/trace/trace_printk.c
index a9077c1b4ad3..2900817ba65c 100644
--- a/kernel/trace/trace_printk.c
+++ b/kernel/trace/trace_printk.c
@@ -244,12 +244,31 @@ static const char **find_next(void *v, loff_t *pos)
 {
        const char **fmt = v;
        int start_index;
+        int last_index;
        start_index = __stop___trace_bprintk_fmt - __start___trace_bprintk_fmt;
        if (*pos < start_index)
                return __start___trace_bprintk_fmt + *pos;
+        /*
+         * The __tracepoint_str section is treated the same as the
+         * __trace_printk_fmt section. The difference is that the
+         * __trace_printk_fmt section should only be used by trace_printk()
+         * in a debugging environment, as if anything exists in that section
+         * the trace_prink() helper buffers are allocated, which would just
+         * waste space in a production environment.
+         *
+         * The __tracepoint_str sections on the other hand are used by
+         * tracepoints which need to map pointers to their strings to
+         * the ASCII text for userspace.
+         */
+        last_index = start_index;
+        start_index = __stop___tracepoint_str - __start___tracepoint_str;
+        if (*pos < last_index + start_index)
+                return __start___tracepoint_str + (*pos - last_index);
        return find_next_mod_format(start_index, v, fmt, pos);
 }
diff --git a/kernel/watchdog.c b/kernel/watchdog.c
index 1241d8c91d5e..51c4f34d258e 100644
--- a/kernel/watchdog.c
+++ b/kernel/watchdog.c
@@ -553,14 +553,6 @@ void __init lockup_detector_init(void)
 {
        set_sample_period();
-#ifdef CONFIG_NO_HZ_FULL
-        if (watchdog_user_enabled) {
-                watchdog_user_enabled = 0;
-                pr_warning("Disabled lockup detectors by default for full dynticks\n");
-                pr_warning("You can reactivate it with 'sysctl -w kernel.watchdog=1'\n");
-        }
-#endif
        if (watchdog_user_enabled)
                watchdog_enable_all_cpus();
 }
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 7f5d4be22034..29b79852a845 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -16,9 +16,10 @@
 *
 * This is the generic async execution mechanism.  Work items as are
 * executed in process context.  The worker pool is shared and
- * automatically managed.  There is one worker pool for each CPU and
+ * automatically managed.  There are two worker pools for each CPU (one for
- * one extra for works which are better served by workers which are
+ * normal work items and the other for high priority ones) and some extra
- * not bound to any specific CPU.
+ * pools for workqueues which are not bound to any specific CPU - the
+ * number of these backing pools is dynamic.
 *
 * Please read Documentation/workqueue.txt for details.
 */
@@ -2033,8 +2034,11 @@ static bool maybe_destroy_workers(struct worker_pool *pool)
 * multiple times.  Does GFP_KERNEL allocations.
 *
 * RETURNS:
- * spin_lock_irq(pool->lock) which may be released and regrabbed
+ * %false if the pool don't need management and the caller can safely start
- * multiple times.  Does GFP_KERNEL allocations.
+ * processing works, %true indicates that the function released pool->lock
+ * and reacquired it to perform some management function and that the
+ * conditions that the caller verified while holding the lock before
+ * calling the function might no longer be true.
 */
 static bool manage_workers(struct worker *worker)
 {
@@ -2201,6 +2205,15 @@ __acquires(&pool->lock)
                dump_stack();
        }
+        /*
+         * The following prevents a kworker from hogging CPU on !PREEMPT
+         * kernels, where a requeueing work item waiting for something to
+         * happen could deadlock with stop_machine as such work item could
+         * indefinitely requeue itself while all other CPUs are trapped in
+         * stop_machine.
+         */
+        cond_resched();
        spin_lock_irq(&pool->lock);
        /* clear cpu intensive status */
@@ -3086,25 +3099,26 @@ static struct workqueue_struct *dev_to_wq(struct device *dev)
        return wq_dev->wq;
 }
-static ssize_t wq_per_cpu_show(struct device *dev,
+static ssize_t per_cpu_show(struct device *dev, struct device_attribute *attr,
-                               struct device_attribute *attr, char *buf)
+                            char *buf)
 {
        struct workqueue_struct *wq = dev_to_wq(dev);
        return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND));
 }
+static DEVICE_ATTR_RO(per_cpu);
-static ssize_t wq_max_active_show(struct device *dev,
+static ssize_t max_active_show(struct device *dev,
-                                  struct device_attribute *attr, char *buf)
+                               struct device_attribute *attr, char *buf)
 {
        struct workqueue_struct *wq = dev_to_wq(dev);
        return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active);
 }
-static ssize_t wq_max_active_store(struct device *dev,
+static ssize_t max_active_store(struct device *dev,
-                                   struct device_attribute *attr,
+                                struct device_attribute *attr, const char *buf,
-                                   const char *buf, size_t count)
+                                size_t count)
 {
        struct workqueue_struct *wq = dev_to_wq(dev);
        int val;
@@ -3115,12 +3129,14 @@ static ssize_t wq_max_active_store(struct device *dev,
        workqueue_set_max_active(wq, val);
        return count;
 }
+static DEVICE_ATTR_RW(max_active);
-static struct device_attribute wq_sysfs_attrs[] = {
+static struct attribute *wq_sysfs_attrs[] = {
-        __ATTR(per_cpu, 0444, wq_per_cpu_show, NULL),
+        &dev_attr_per_cpu.attr,
-        __ATTR(max_active, 0644, wq_max_active_show, wq_max_active_store),
+        &dev_attr_max_active.attr,
-        __ATTR_NULL,
+        NULL,
 };
+ATTRIBUTE_GROUPS(wq_sysfs);
 static ssize_t wq_pool_ids_show(struct device *dev,
                                struct device_attribute *attr, char *buf)
@@ -3270,7 +3286,7 @@ static struct device_attribute wq_sysfs_unbound_attrs[] = {
 static struct bus_type wq_subsys = {
        .name                           = "workqueue",
-        .dev_attrs                      = wq_sysfs_attrs,
+        .dev_groups                     = wq_sysfs_groups,
 };
 static int __init wq_sysfs_init(void)