87 files changed, 4073 insertions, 2445 deletions
diff --git a/kernel/Makefile b/kernel/Makefile
index 86e3285ae7e5..ac0d533eb7de 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -110,6 +110,7 @@ obj-$(CONFIG_USER_RETURN_NOTIFIER) += user-return-notifier.o
 obj-$(CONFIG_PADATA) += padata.o
 obj-$(CONFIG_CRASH_DUMP) += crash_dump.o
 obj-$(CONFIG_JUMP_LABEL) += jump_label.o
+obj-$(CONFIG_CONTEXT_TRACKING) += context_tracking.o
 $(obj)/configs.o: $(obj)/config_data.h
diff --git a/kernel/auditsc.c b/kernel/auditsc.c
index 2f186ed80c40..e37e6a12c5e3 100644
--- a/kernel/auditsc.c
+++ b/kernel/auditsc.c
@@ -200,7 +200,6 @@ struct audit_context {
        struct list_head    names_list; /* anchor for struct audit_names->list */
        char *              filterkey;  /* key for rule that triggered record */
        struct path         pwd;
-        struct audit_context *previous; /* For nested syscalls */
        struct audit_aux_data *aux;
        struct audit_aux_data *aux_pids;
        struct sockaddr_storage *sockaddr;
@@ -1091,29 +1090,13 @@ int audit_alloc(struct task_struct *tsk)
 static inline void audit_free_context(struct audit_context *context)
 {
-        struct audit_context *previous;
+        audit_free_names(context);
-        int                  count = 0;
+        unroll_tree_refs(context, NULL, 0);
+        free_tree_refs(context);
-        do {
+        audit_free_aux(context);
-                previous = context->previous;
+        kfree(context->filterkey);
-                if (previous || (count &&  count < 10)) {
+        kfree(context->sockaddr);
-                        ++count;
+        kfree(context);
-                        printk(KERN_ERR "audit(:%d): major=%d name_count=%d:"
-                               " freeing multiple contexts (%d)\n",
-                               context->serial, context->major,
-                               context->name_count, count);
-                }
-                audit_free_names(context);
-                unroll_tree_refs(context, NULL, 0);
-                free_tree_refs(context);
-                audit_free_aux(context);
-                kfree(context->filterkey);
-                kfree(context->sockaddr);
-                kfree(context);
-                context  = previous;
-        } while (context);
-        if (count >= 10)
-                printk(KERN_ERR "audit: freed %d contexts\n", count);
 }
 void audit_log_task_context(struct audit_buffer *ab)
@@ -1159,7 +1142,7 @@ void audit_log_task_info(struct audit_buffer *ab, struct task_struct *tsk)
        cred = current_cred();
        spin_lock_irq(&tsk->sighand->siglock);
-        if (tsk->signal && tsk->signal->tty && tsk->signal->tty->name)
+        if (tsk->signal && tsk->signal->tty)
                tty = tsk->signal->tty->name;
        else
                tty = "(none)";
@@ -1783,42 +1766,6 @@ void __audit_syscall_entry(int arch, int major,
        if (!context)
                return;
-        /*
-         * This happens only on certain architectures that make system
-         * calls in kernel_thread via the entry.S interface, instead of
-         * with direct calls.  (If you are porting to a new
-         * architecture, hitting this condition can indicate that you
-         * got the _exit/_leave calls backward in entry.S.)
-         *
-         * i386     no
-         * x86_64   no
-         * ppc64    yes (see arch/powerpc/platforms/iseries/misc.S)
-         *
-         * This also happens with vm86 emulation in a non-nested manner
-         * (entries without exits), so this case must be caught.
-         */
-        if (context->in_syscall) {
-                struct audit_context *newctx;
-#if AUDIT_DEBUG
-                printk(KERN_ERR
-                       "audit(:%d) pid=%d in syscall=%d;"
-                       " entering syscall=%d\n",
-                       context->serial, tsk->pid, context->major, major);
-#endif
-                newctx = audit_alloc_context(context->state);
-                if (newctx) {
-                        newctx->previous   = context;
-                        context            = newctx;
-                        tsk->audit_context = newctx;
-                } else  {
-                        /* If we can't alloc a new context, the best we
-                         * can do is to leak memory (any pending putname
-                         * will be lost).  The only other alternative is
-                         * to abandon auditing. */
-                        audit_zero_context(context, context->state);
-                }
-        }
        BUG_ON(context->in_syscall || context->name_count);
        if (!audit_enabled)
@@ -1881,28 +1828,21 @@ void __audit_syscall_exit(int success, long return_code)
        if (!list_empty(&context->killed_trees))
                audit_kill_trees(&context->killed_trees);
-        if (context->previous) {
+        audit_free_names(context);
-                struct audit_context *new_context = context->previous;
+        unroll_tree_refs(context, NULL, 0);
-                context->previous  = NULL;
+        audit_free_aux(context);
-                audit_free_context(context);
+        context->aux = NULL;
-                tsk->audit_context = new_context;
+        context->aux_pids = NULL;
-        } else {
+        context->target_pid = 0;
-                audit_free_names(context);
+        context->target_sid = 0;
-                unroll_tree_refs(context, NULL, 0);
+        context->sockaddr_len = 0;
-                audit_free_aux(context);
+        context->type = 0;
-                context->aux = NULL;
+        context->fds[0] = -1;
-                context->aux_pids = NULL;
+        if (context->state != AUDIT_RECORD_CONTEXT) {
-                context->target_pid = 0;
+                kfree(context->filterkey);
-                context->target_sid = 0;
+                context->filterkey = NULL;
-                context->sockaddr_len = 0;
-                context->type = 0;
-                context->fds[0] = -1;
-                if (context->state != AUDIT_RECORD_CONTEXT) {
-                        kfree(context->filterkey);
-                        context->filterkey = NULL;
-                }
-                tsk->audit_context = context;
        }
+        tsk->audit_context = context;
 }
 static inline void handle_one(const struct inode *inode)
diff --git a/kernel/cgroup.c b/kernel/cgroup.c
index 0dbfba2efa77..4855892798fd 100644
--- a/kernel/cgroup.c
+++ b/kernel/cgroup.c
@@ -138,6 +138,9 @@ struct cgroupfs_root {
        /* Hierarchy-specific flags */
        unsigned long flags;
+        /* IDs for cgroups in this hierarchy */
+        struct ida cgroup_ida;
        /* The path to use for release notifications. */
        char release_agent_path[PATH_MAX];
@@ -171,8 +174,8 @@ struct css_id {
         * The css to which this ID points. This pointer is set to valid value
         * after cgroup is populated. If cgroup is removed, this will be NULL.
         * This pointer is expected to be RCU-safe because destroy()
-         * is called after synchronize_rcu(). But for safe use, css_is_removed()
+         * is called after synchronize_rcu(). But for safe use, css_tryget()
-         * css_tryget() should be used for avoiding race.
+         * should be used for avoiding race.
         */
        struct cgroup_subsys_state __rcu *css;
        /*
@@ -242,6 +245,10 @@ static DEFINE_SPINLOCK(hierarchy_id_lock);
 */
 static int need_forkexit_callback __read_mostly;
+static int cgroup_destroy_locked(struct cgroup *cgrp);
+static int cgroup_addrm_files(struct cgroup *cgrp, struct cgroup_subsys *subsys,
+                              struct cftype cfts[], bool is_add);
 #ifdef CONFIG_PROVE_LOCKING
 int cgroup_lock_is_held(void)
 {
@@ -294,11 +301,6 @@ static int notify_on_release(const struct cgroup *cgrp)
        return test_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
 }
-static int clone_children(const struct cgroup *cgrp)
-{
-        return test_bit(CGRP_CLONE_CHILDREN, &cgrp->flags);
-}
 /*
 * for_each_subsys() allows you to iterate on each subsystem attached to
 * an active hierarchy
@@ -782,12 +784,12 @@ static struct cgroup *task_cgroup_from_root(struct task_struct *task,
 *      The task_lock() exception
 *
 * The need for this exception arises from the action of
- * cgroup_attach_task(), which overwrites one tasks cgroup pointer with
+ * cgroup_attach_task(), which overwrites one task's cgroup pointer with
 * another.  It does so using cgroup_mutex, however there are
 * several performance critical places that need to reference
 * task->cgroup without the expense of grabbing a system global
 * mutex.  Therefore except as noted below, when dereferencing or, as
- * in cgroup_attach_task(), modifying a task'ss cgroup pointer we use
+ * in cgroup_attach_task(), modifying a task's cgroup pointer we use
 * task_lock(), which acts on a spinlock (task->alloc_lock) already in
 * the task_struct routinely used for such matters.
 *
@@ -854,30 +856,6 @@ static struct inode *cgroup_new_inode(umode_t mode, struct super_block *sb)
        return inode;
 }
-/*
- * Call subsys's pre_destroy handler.
- * This is called before css refcnt check.
- */
-static int cgroup_call_pre_destroy(struct cgroup *cgrp)
-{
-        struct cgroup_subsys *ss;
-        int ret = 0;
-        for_each_subsys(cgrp->root, ss) {
-                if (!ss->pre_destroy)
-                        continue;
-                ret = ss->pre_destroy(cgrp);
-                if (ret) {
-                        /* ->pre_destroy() failure is being deprecated */
-                        WARN_ON_ONCE(!ss->__DEPRECATED_clear_css_refs);
-                        break;
-                }
-        }
-        return ret;
-}
 static void cgroup_diput(struct dentry *dentry, struct inode *inode)
 {
        /* is dentry a directory ? if so, kfree() associated cgroup */
@@ -898,7 +876,7 @@ static void cgroup_diput(struct dentry *dentry, struct inode *inode)
                 * Release the subsystem state objects.
                 */
                for_each_subsys(cgrp->root, ss)
-                        ss->destroy(cgrp);
+                        ss->css_free(cgrp);
                cgrp->root->number_of_cgroups--;
                mutex_unlock(&cgroup_mutex);
@@ -917,6 +895,7 @@ static void cgroup_diput(struct dentry *dentry, struct inode *inode)
                simple_xattrs_free(&cgrp->xattrs);
+                ida_simple_remove(&cgrp->root->cgroup_ida, cgrp->id);
                kfree_rcu(cgrp, rcu_head);
        } else {
                struct cfent *cfe = __d_cfe(dentry);
@@ -987,7 +966,7 @@ static void cgroup_clear_directory(struct dentry *dir, bool base_files,
                if (!test_bit(ss->subsys_id, &subsys_mask))
                        continue;
                list_for_each_entry(set, &ss->cftsets, node)
-                        cgroup_rm_file(cgrp, set->cfts);
+                        cgroup_addrm_files(cgrp, NULL, set->cfts, false);
        }
        if (base_files) {
                while (!list_empty(&cgrp->files))
@@ -1015,33 +994,6 @@ static void cgroup_d_remove_dir(struct dentry *dentry)
 }
 /*
- * A queue for waiters to do rmdir() cgroup. A tasks will sleep when
- * cgroup->count == 0 && list_empty(&cgroup->children) && subsys has some
- * reference to css->refcnt. In general, this refcnt is expected to goes down
- * to zero, soon.
- *
- * CGRP_WAIT_ON_RMDIR flag is set under cgroup's inode->i_mutex;
- */
-static DECLARE_WAIT_QUEUE_HEAD(cgroup_rmdir_waitq);
-static void cgroup_wakeup_rmdir_waiter(struct cgroup *cgrp)
-{
-        if (unlikely(test_and_clear_bit(CGRP_WAIT_ON_RMDIR, &cgrp->flags)))
-                wake_up_all(&cgroup_rmdir_waitq);
-}
-void cgroup_exclude_rmdir(struct cgroup_subsys_state *css)
-{
-        css_get(css);
-}
-void cgroup_release_and_wakeup_rmdir(struct cgroup_subsys_state *css)
-{
-        cgroup_wakeup_rmdir_waiter(css->cgroup);
-        css_put(css);
-}
-/*
 * Call with cgroup_mutex held. Drops reference counts on modules, including
 * any duplicate ones that parse_cgroupfs_options took. If this function
 * returns an error, no reference counts are touched.
@@ -1150,7 +1102,7 @@ static int cgroup_show_options(struct seq_file *seq, struct dentry *dentry)
                seq_puts(seq, ",xattr");
        if (strlen(root->release_agent_path))
                seq_printf(seq, ",release_agent=%s", root->release_agent_path);
-        if (clone_children(&root->top_cgroup))
+        if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->top_cgroup.flags))
                seq_puts(seq, ",clone_children");
        if (strlen(root->name))
                seq_printf(seq, ",name=%s", root->name);
@@ -1162,7 +1114,7 @@ struct cgroup_sb_opts {
        unsigned long subsys_mask;
        unsigned long flags;
        char *release_agent;
-        bool clone_children;
+        bool cpuset_clone_children;
        char *name;
        /* User explicitly requested empty subsystem */
        bool none;
@@ -1213,7 +1165,7 @@ static int parse_cgroupfs_options(char *data, struct cgroup_sb_opts *opts)
                        continue;
                }
                if (!strcmp(token, "clone_children")) {
-                        opts->clone_children = true;
+                        opts->cpuset_clone_children = true;
                        continue;
                }
                if (!strcmp(token, "xattr")) {
@@ -1381,7 +1333,6 @@ static int cgroup_remount(struct super_block *sb, int *flags, char *data)
        if (ret)
                goto out_unlock;
-        /* See feature-removal-schedule.txt */
        if (opts.subsys_mask != root->actual_subsys_mask || opts.release_agent)
                pr_warning("cgroup: option changes via remount are deprecated (pid=%d comm=%s)\n",
                           task_tgid_nr(current), current->comm);
@@ -1397,14 +1348,21 @@ static int cgroup_remount(struct super_block *sb, int *flags, char *data)
                goto out_unlock;
        }
+        /*
+         * Clear out the files of subsystems that should be removed, do
+         * this before rebind_subsystems, since rebind_subsystems may
+         * change this hierarchy's subsys_list.
+         */
+        cgroup_clear_directory(cgrp->dentry, false, removed_mask);
        ret = rebind_subsystems(root, opts.subsys_mask);
        if (ret) {
+                /* rebind_subsystems failed, re-populate the removed files */
+                cgroup_populate_dir(cgrp, false, removed_mask);
                drop_parsed_module_refcounts(opts.subsys_mask);
                goto out_unlock;
        }
-        /* clear out any existing files and repopulate subsystem files */
-        cgroup_clear_directory(cgrp->dentry, false, removed_mask);
        /* re-populate subsystem files */
        cgroup_populate_dir(cgrp, false, added_mask);
@@ -1432,6 +1390,7 @@ static void init_cgroup_housekeeping(struct cgroup *cgrp)
        INIT_LIST_HEAD(&cgrp->children);
        INIT_LIST_HEAD(&cgrp->files);
        INIT_LIST_HEAD(&cgrp->css_sets);
+        INIT_LIST_HEAD(&cgrp->allcg_node);
        INIT_LIST_HEAD(&cgrp->release_list);
        INIT_LIST_HEAD(&cgrp->pidlists);
        mutex_init(&cgrp->pidlist_mutex);
@@ -1450,8 +1409,8 @@ static void init_cgroup_root(struct cgroupfs_root *root)
        root->number_of_cgroups = 1;
        cgrp->root = root;
        cgrp->top_cgroup = cgrp;
-        list_add_tail(&cgrp->allcg_node, &root->allcg_list);
        init_cgroup_housekeeping(cgrp);
+        list_add_tail(&cgrp->allcg_node, &root->allcg_list);
 }
 static bool init_root_id(struct cgroupfs_root *root)
@@ -1518,12 +1477,13 @@ static struct cgroupfs_root *cgroup_root_from_opts(struct cgroup_sb_opts *opts)
        root->subsys_mask = opts->subsys_mask;
        root->flags = opts->flags;
+        ida_init(&root->cgroup_ida);
        if (opts->release_agent)
                strcpy(root->release_agent_path, opts->release_agent);
        if (opts->name)
                strcpy(root->name, opts->name);
-        if (opts->clone_children)
+        if (opts->cpuset_clone_children)
-                set_bit(CGRP_CLONE_CHILDREN, &root->top_cgroup.flags);
+                set_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->top_cgroup.flags);
        return root;
 }
@@ -1536,6 +1496,7 @@ static void cgroup_drop_root(struct cgroupfs_root *root)
        spin_lock(&hierarchy_id_lock);
        ida_remove(&hierarchy_ida, root->hierarchy_id);
        spin_unlock(&hierarchy_id_lock);
+        ida_destroy(&root->cgroup_ida);
        kfree(root);
 }
@@ -1701,7 +1662,6 @@ static struct dentry *cgroup_mount(struct file_system_type *fs_type,
                free_cg_links(&tmp_cg_links);
-                BUG_ON(!list_empty(&root_cgrp->sibling));
                BUG_ON(!list_empty(&root_cgrp->children));
                BUG_ON(root->number_of_cgroups != 1);
@@ -1750,7 +1710,6 @@ static void cgroup_kill_sb(struct super_block *sb) {
        BUG_ON(root->number_of_cgroups != 1);
        BUG_ON(!list_empty(&cgrp->children));
-        BUG_ON(!list_empty(&cgrp->sibling));
        mutex_lock(&cgroup_mutex);
        mutex_lock(&cgroup_root_mutex);
@@ -1808,9 +1767,11 @@ static struct kobject *cgroup_kobj;
 */
 int cgroup_path(const struct cgroup *cgrp, char *buf, int buflen)
 {
+        struct dentry *dentry = cgrp->dentry;
        char *start;
-        struct dentry *dentry = rcu_dereference_check(cgrp->dentry,
-                                                      cgroup_lock_is_held());
+        rcu_lockdep_assert(rcu_read_lock_held() || cgroup_lock_is_held(),
+                           "cgroup_path() called without proper locking");
        if (!dentry || cgrp == dummytop) {
                /*
@@ -1821,9 +1782,9 @@ int cgroup_path(const struct cgroup *cgrp, char *buf, int buflen)
                return 0;
        }
-        start = buf + buflen;
+        start = buf + buflen - 1;
-        *--start = '\0';
+        *start = '\0';
        for (;;) {
                int len = dentry->d_name.len;
@@ -1834,8 +1795,7 @@ int cgroup_path(const struct cgroup *cgrp, char *buf, int buflen)
                if (!cgrp)
                        break;
-                dentry = rcu_dereference_check(cgrp->dentry,
+                dentry = cgrp->dentry;
-                                               cgroup_lock_is_held());
                if (!cgrp->parent)
                        continue;
                if (--start < buf)
@@ -1930,9 +1890,7 @@ EXPORT_SYMBOL_GPL(cgroup_taskset_size);
 /*
 * cgroup_task_migrate - move a task from one cgroup to another.
 *
- * 'guarantee' is set if the caller promises that a new css_set for the task
+ * Must be called with cgroup_mutex and threadgroup locked.
- * will already exist. If not set, this function might sleep, and can fail with
- * -ENOMEM. Must be called with cgroup_mutex and threadgroup locked.
 */
 static void cgroup_task_migrate(struct cgroup *cgrp, struct cgroup *oldcgrp,
                                struct task_struct *tsk, struct css_set *newcg)
@@ -2025,12 +1983,6 @@ int cgroup_attach_task(struct cgroup *cgrp, struct task_struct *tsk)
        }
        synchronize_rcu();
-        /*
-         * wake up rmdir() waiter. the rmdir should fail since the cgroup
-         * is no longer empty.
-         */
-        cgroup_wakeup_rmdir_waiter(cgrp);
 out:
        if (retval) {
                for_each_subsys(root, ss) {
@@ -2200,7 +2152,6 @@ static int cgroup_attach_proc(struct cgroup *cgrp, struct task_struct *leader)
         * step 5: success! and cleanup
         */
        synchronize_rcu();
-        cgroup_wakeup_rmdir_waiter(cgrp);
        retval = 0;
 out_put_css_set_refs:
        if (retval) {
@@ -2711,10 +2662,17 @@ static int cgroup_create_file(struct dentry *dentry, umode_t mode,
                /* start off with i_nlink == 2 (for "." entry) */
                inc_nlink(inode);
+                inc_nlink(dentry->d_parent->d_inode);
-                /* start with the directory inode held, so that we can
+                /*
-                 * populate it without racing with another mkdir */
+                 * Control reaches here with cgroup_mutex held.
-                mutex_lock_nested(&inode->i_mutex, I_MUTEX_CHILD);
+                 * @inode->i_mutex should nest outside cgroup_mutex but we
+                 * want to populate it immediately without releasing
+                 * cgroup_mutex.  As @inode isn't visible to anyone else
+                 * yet, trylock will always succeed without affecting
+                 * lockdep checks.
+                 */
+                WARN_ON_ONCE(!mutex_trylock(&inode->i_mutex));
        } else if (S_ISREG(mode)) {
                inode->i_size = 0;
                inode->i_fop = &cgroup_file_operations;
@@ -2725,32 +2683,6 @@ static int cgroup_create_file(struct dentry *dentry, umode_t mode,
        return 0;
 }
-/*
- * cgroup_create_dir - create a directory for an object.
- * @cgrp: the cgroup we create the directory for. It must have a valid
- *        ->parent field. And we are going to fill its ->dentry field.
- * @dentry: dentry of the new cgroup
- * @mode: mode to set on new directory.
- */
-static int cgroup_create_dir(struct cgroup *cgrp, struct dentry *dentry,
-                                umode_t mode)
-{
-        struct dentry *parent;
-        int error = 0;
-        parent = cgrp->parent->dentry;
-        error = cgroup_create_file(dentry, S_IFDIR | mode, cgrp->root->sb);
-        if (!error) {
-                dentry->d_fsdata = cgrp;
-                inc_nlink(parent->d_inode);
-                rcu_assign_pointer(cgrp->dentry, dentry);
-                dget(dentry);
-        }
-        dput(dentry);
-        return error;
-}
 /**
 * cgroup_file_mode - deduce file mode of a control file
 * @cft: the control file in question
@@ -2791,12 +2723,6 @@ static int cgroup_add_file(struct cgroup *cgrp, struct cgroup_subsys *subsys,
        simple_xattrs_init(&cft->xattrs);
-        /* does @cft->flags tell us to skip creation on @cgrp? */
-        if ((cft->flags & CFTYPE_NOT_ON_ROOT) && !cgrp->parent)
-                return 0;
-        if ((cft->flags & CFTYPE_ONLY_ON_ROOT) && cgrp->parent)
-                return 0;
        if (subsys && !test_bit(ROOT_NOPREFIX, &cgrp->root->flags)) {
                strcpy(name, subsys->name);
                strcat(name, ".");
@@ -2837,6 +2763,12 @@ static int cgroup_addrm_files(struct cgroup *cgrp, struct cgroup_subsys *subsys,
        int err, ret = 0;
        for (cft = cfts; cft->name[0] != '\0'; cft++) {
+                /* does cft->flags tell us to skip this file on @cgrp? */
+                if ((cft->flags & CFTYPE_NOT_ON_ROOT) && !cgrp->parent)
+                        continue;
+                if ((cft->flags & CFTYPE_ONLY_ON_ROOT) && cgrp->parent)
+                        continue;
                if (is_add)
                        err = cgroup_add_file(cgrp, subsys, cft);
                else
@@ -3044,6 +2976,92 @@ static void cgroup_enable_task_cg_lists(void)
        write_unlock(&css_set_lock);
 }
+/**
+ * cgroup_next_descendant_pre - find the next descendant for pre-order walk
+ * @pos: the current position (%NULL to initiate traversal)
+ * @cgroup: cgroup whose descendants to walk
+ *
+ * To be used by cgroup_for_each_descendant_pre().  Find the next
+ * descendant to visit for pre-order traversal of @cgroup's descendants.
+ */
+struct cgroup *cgroup_next_descendant_pre(struct cgroup *pos,
+                                          struct cgroup *cgroup)
+{
+        struct cgroup *next;
+        WARN_ON_ONCE(!rcu_read_lock_held());
+        /* if first iteration, pretend we just visited @cgroup */
+        if (!pos) {
+                if (list_empty(&cgroup->children))
+                        return NULL;
+                pos = cgroup;
+        }
+        /* visit the first child if exists */
+        next = list_first_or_null_rcu(&pos->children, struct cgroup, sibling);
+        if (next)
+                return next;
+        /* no child, visit my or the closest ancestor's next sibling */
+        do {
+                next = list_entry_rcu(pos->sibling.next, struct cgroup,
+                                      sibling);
+                if (&next->sibling != &pos->parent->children)
+                        return next;
+                pos = pos->parent;
+        } while (pos != cgroup);
+        return NULL;
+}
+EXPORT_SYMBOL_GPL(cgroup_next_descendant_pre);
+static struct cgroup *cgroup_leftmost_descendant(struct cgroup *pos)
+{
+        struct cgroup *last;
+        do {
+                last = pos;
+                pos = list_first_or_null_rcu(&pos->children, struct cgroup,
+                                             sibling);
+        } while (pos);
+        return last;
+}
+/**
+ * cgroup_next_descendant_post - find the next descendant for post-order walk
+ * @pos: the current position (%NULL to initiate traversal)
+ * @cgroup: cgroup whose descendants to walk
+ *
+ * To be used by cgroup_for_each_descendant_post().  Find the next
+ * descendant to visit for post-order traversal of @cgroup's descendants.
+ */
+struct cgroup *cgroup_next_descendant_post(struct cgroup *pos,
+                                           struct cgroup *cgroup)
+{
+        struct cgroup *next;
+        WARN_ON_ONCE(!rcu_read_lock_held());
+        /* if first iteration, visit the leftmost descendant */
+        if (!pos) {
+                next = cgroup_leftmost_descendant(cgroup);
+                return next != cgroup ? next : NULL;
+        }
+        /* if there's an unvisited sibling, visit its leftmost descendant */
+        next = list_entry_rcu(pos->sibling.next, struct cgroup, sibling);
+        if (&next->sibling != &pos->parent->children)
+                return cgroup_leftmost_descendant(next);
+        /* no sibling left, visit parent */
+        next = pos->parent;
+        return next != cgroup ? next : NULL;
+}
+EXPORT_SYMBOL_GPL(cgroup_next_descendant_post);
 void cgroup_iter_start(struct cgroup *cgrp, struct cgroup_iter *it)
        __acquires(css_set_lock)
 {
@@ -3757,7 +3775,7 @@ static int cgroup_event_wake(wait_queue_t *wait, unsigned mode,
        if (flags & POLLHUP) {
                __remove_wait_queue(event->wqh, &event->wait);
                spin_lock(&cgrp->event_list_lock);
-                list_del(&event->list);
+                list_del_init(&event->list);
                spin_unlock(&cgrp->event_list_lock);
                /*
                 * We are in atomic context, but cgroup_event_remove() may
@@ -3894,7 +3912,7 @@ fail:
 static u64 cgroup_clone_children_read(struct cgroup *cgrp,
                                    struct cftype *cft)
 {
-        return clone_children(cgrp);
+        return test_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags);
 }
 static int cgroup_clone_children_write(struct cgroup *cgrp,
@@ -3902,9 +3920,9 @@ static int cgroup_clone_children_write(struct cgroup *cgrp,
                                     u64 val)
 {
        if (val)
-                set_bit(CGRP_CLONE_CHILDREN, &cgrp->flags);
+                set_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags);
        else
-                clear_bit(CGRP_CLONE_CHILDREN, &cgrp->flags);
+                clear_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags);
        return 0;
 }
@@ -4017,19 +4035,57 @@ static void init_cgroup_css(struct cgroup_subsys_state *css,
        css->flags = 0;
        css->id = NULL;
        if (cgrp == dummytop)
-                set_bit(CSS_ROOT, &css->flags);
+                css->flags |= CSS_ROOT;
        BUG_ON(cgrp->subsys[ss->subsys_id]);
        cgrp->subsys[ss->subsys_id] = css;
        /*
-         * If !clear_css_refs, css holds an extra ref to @cgrp->dentry
+         * css holds an extra ref to @cgrp->dentry which is put on the last
-         * which is put on the last css_put().  dput() requires process
+         * css_put().  dput() requires process context, which css_put() may
-         * context, which css_put() may be called without.  @css->dput_work
+         * be called without.  @css->dput_work will be used to invoke
-         * will be used to invoke dput() asynchronously from css_put().
+         * dput() asynchronously from css_put().
         */
        INIT_WORK(&css->dput_work, css_dput_fn);
-        if (ss->__DEPRECATED_clear_css_refs)
+}
-                set_bit(CSS_CLEAR_CSS_REFS, &css->flags);
+/* invoke ->post_create() on a new CSS and mark it online if successful */
+static int online_css(struct cgroup_subsys *ss, struct cgroup *cgrp)
+{
+        int ret = 0;
+        lockdep_assert_held(&cgroup_mutex);
+        if (ss->css_online)
+                ret = ss->css_online(cgrp);
+        if (!ret)
+                cgrp->subsys[ss->subsys_id]->flags |= CSS_ONLINE;
+        return ret;
+}
+/* if the CSS is online, invoke ->pre_destory() on it and mark it offline */
+static void offline_css(struct cgroup_subsys *ss, struct cgroup *cgrp)
+        __releases(&cgroup_mutex) __acquires(&cgroup_mutex)
+{
+        struct cgroup_subsys_state *css = cgrp->subsys[ss->subsys_id];
+        lockdep_assert_held(&cgroup_mutex);
+        if (!(css->flags & CSS_ONLINE))
+                return;
+        /*
+         * css_offline() should be called with cgroup_mutex unlocked.  See
+         * 3fa59dfbc3 ("cgroup: fix potential deadlock in pre_destroy") for
+         * details.  This temporary unlocking should go away once
+         * cgroup_mutex is unexported from controllers.
+         */
+        if (ss->css_offline) {
+                mutex_unlock(&cgroup_mutex);
+                ss->css_offline(cgrp);
+                mutex_lock(&cgroup_mutex);
+        }
+        cgrp->subsys[ss->subsys_id]->flags &= ~CSS_ONLINE;
 }
 /*
@@ -4049,10 +4105,27 @@ static long cgroup_create(struct cgroup *parent, struct dentry *dentry,
        struct cgroup_subsys *ss;
        struct super_block *sb = root->sb;
+        /* allocate the cgroup and its ID, 0 is reserved for the root */
        cgrp = kzalloc(sizeof(*cgrp), GFP_KERNEL);
        if (!cgrp)
                return -ENOMEM;
+        cgrp->id = ida_simple_get(&root->cgroup_ida, 1, 0, GFP_KERNEL);
+        if (cgrp->id < 0)
+                goto err_free_cgrp;
+        /*
+         * Only live parents can have children.  Note that the liveliness
+         * check isn't strictly necessary because cgroup_mkdir() and
+         * cgroup_rmdir() are fully synchronized by i_mutex; however, do it
+         * anyway so that locking is contained inside cgroup proper and we
+         * don't get nasty surprises if we ever grow another caller.
+         */
+        if (!cgroup_lock_live_group(parent)) {
+                err = -ENODEV;
+                goto err_free_id;
+        }
        /* Grab a reference on the superblock so the hierarchy doesn't
         * get deleted on unmount if there are child cgroups.  This
         * can be done outside cgroup_mutex, since the sb can't
@@ -4060,8 +4133,6 @@ static long cgroup_create(struct cgroup *parent, struct dentry *dentry,
         * fs */
        atomic_inc(&sb->s_active);
-        mutex_lock(&cgroup_mutex);
        init_cgroup_housekeeping(cgrp);
        cgrp->parent = parent;
@@ -4071,26 +4142,51 @@ static long cgroup_create(struct cgroup *parent, struct dentry *dentry,
        if (notify_on_release(parent))
                set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
-        if (clone_children(parent))
+        if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &parent->flags))
-                set_bit(CGRP_CLONE_CHILDREN, &cgrp->flags);
+                set_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags);
        for_each_subsys(root, ss) {
                struct cgroup_subsys_state *css;
-                css = ss->create(cgrp);
+                css = ss->css_alloc(cgrp);
                if (IS_ERR(css)) {
                        err = PTR_ERR(css);
-                        goto err_destroy;
+                        goto err_free_all;
                }
                init_cgroup_css(css, ss, cgrp);
                if (ss->use_id) {
                        err = alloc_css_id(ss, parent, cgrp);
                        if (err)
-                                goto err_destroy;
+                                goto err_free_all;
                }
-                /* At error, ->destroy() callback has to free assigned ID. */
+        }
-                if (clone_children(parent) && ss->post_clone)
-                        ss->post_clone(cgrp);
+        /*
+         * Create directory.  cgroup_create_file() returns with the new
+         * directory locked on success so that it can be populated without
+         * dropping cgroup_mutex.
+         */
+        err = cgroup_create_file(dentry, S_IFDIR | mode, sb);
+        if (err < 0)
+                goto err_free_all;
+        lockdep_assert_held(&dentry->d_inode->i_mutex);
+        /* allocation complete, commit to creation */
+        dentry->d_fsdata = cgrp;
+        cgrp->dentry = dentry;
+        list_add_tail(&cgrp->allcg_node, &root->allcg_list);
+        list_add_tail_rcu(&cgrp->sibling, &cgrp->parent->children);
+        root->number_of_cgroups++;
+        /* each css holds a ref to the cgroup's dentry */
+        for_each_subsys(root, ss)
+                dget(dentry);
+        /* creation succeeded, notify subsystems */
+        for_each_subsys(root, ss) {
+                err = online_css(ss, cgrp);
+                if (err)
+                        goto err_destroy;
                if (ss->broken_hierarchy && !ss->warned_broken_hierarchy &&
                    parent->parent) {
@@ -4102,50 +4198,34 @@ static long cgroup_create(struct cgroup *parent, struct dentry *dentry,
                }
        }
-        list_add(&cgrp->sibling, &cgrp->parent->children);
-        root->number_of_cgroups++;
-        err = cgroup_create_dir(cgrp, dentry, mode);
-        if (err < 0)
-                goto err_remove;
-        /* If !clear_css_refs, each css holds a ref to the cgroup's dentry */
-        for_each_subsys(root, ss)
-                if (!ss->__DEPRECATED_clear_css_refs)
-                        dget(dentry);
-        /* The cgroup directory was pre-locked for us */
-        BUG_ON(!mutex_is_locked(&cgrp->dentry->d_inode->i_mutex));
-        list_add_tail(&cgrp->allcg_node, &root->allcg_list);
        err = cgroup_populate_dir(cgrp, true, root->subsys_mask);
-        /* If err < 0, we have a half-filled directory - oh well ;) */
+        if (err)
+                goto err_destroy;
        mutex_unlock(&cgroup_mutex);
        mutex_unlock(&cgrp->dentry->d_inode->i_mutex);
        return 0;
- err_remove:
+err_free_all:
-        list_del(&cgrp->sibling);
-        root->number_of_cgroups--;
- err_destroy:
        for_each_subsys(root, ss) {
                if (cgrp->subsys[ss->subsys_id])
-                        ss->destroy(cgrp);
+                        ss->css_free(cgrp);
        }
        mutex_unlock(&cgroup_mutex);
        /* Release the reference count that we took on the superblock */
        deactivate_super(sb);
+err_free_id:
+        ida_simple_remove(&root->cgroup_ida, cgrp->id);
+err_free_cgrp:
        kfree(cgrp);
        return err;
+err_destroy:
+        cgroup_destroy_locked(cgrp);
+        mutex_unlock(&cgroup_mutex);
+        mutex_unlock(&dentry->d_inode->i_mutex);
+        return err;
 }
 static int cgroup_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
@@ -4197,153 +4277,60 @@ static int cgroup_has_css_refs(struct cgroup *cgrp)
        return 0;
 }
-/*
+static int cgroup_destroy_locked(struct cgroup *cgrp)
- * Atomically mark all (or else none) of the cgroup's CSS objects as
+        __releases(&cgroup_mutex) __acquires(&cgroup_mutex)
- * CSS_REMOVED. Return true on success, or false if the cgroup has
- * busy subsystems. Call with cgroup_mutex held
- *
- * Depending on whether a subsys has __DEPRECATED_clear_css_refs set or
- * not, cgroup removal behaves differently.
- *
- * If clear is set, css refcnt for the subsystem should be zero before
- * cgroup removal can be committed.  This is implemented by
- * CGRP_WAIT_ON_RMDIR and retry logic around ->pre_destroy(), which may be
- * called multiple times until all css refcnts reach zero and is allowed to
- * veto removal on any invocation.  This behavior is deprecated and will be
- * removed as soon as the existing user (memcg) is updated.
- *
- * If clear is not set, each css holds an extra reference to the cgroup's
- * dentry and cgroup removal proceeds regardless of css refs.
- * ->pre_destroy() will be called at least once and is not allowed to fail.
- * On the last put of each css, whenever that may be, the extra dentry ref
- * is put so that dentry destruction happens only after all css's are
- * released.
- */
-static int cgroup_clear_css_refs(struct cgroup *cgrp)
 {
+        struct dentry *d = cgrp->dentry;
+        struct cgroup *parent = cgrp->parent;
+        DEFINE_WAIT(wait);
+        struct cgroup_event *event, *tmp;
        struct cgroup_subsys *ss;
-        unsigned long flags;
+        LIST_HEAD(tmp_list);
-        bool failed = false;
+        lockdep_assert_held(&d->d_inode->i_mutex);
+        lockdep_assert_held(&cgroup_mutex);
-        local_irq_save(flags);
+        if (atomic_read(&cgrp->count) || !list_empty(&cgrp->children))
+                return -EBUSY;
        /*
-         * Block new css_tryget() by deactivating refcnt.  If all refcnts
+         * Block new css_tryget() by deactivating refcnt and mark @cgrp
-         * for subsystems w/ clear_css_refs set were 1 at the moment of
+         * removed.  This makes future css_tryget() and child creation
-         * deactivation, we succeeded.
+         * attempts fail thus maintaining the removal conditions verified
+         * above.
         */
        for_each_subsys(cgrp->root, ss) {
                struct cgroup_subsys_state *css = cgrp->subsys[ss->subsys_id];
                WARN_ON(atomic_read(&css->refcnt) < 0);
                atomic_add(CSS_DEACT_BIAS, &css->refcnt);
-                if (ss->__DEPRECATED_clear_css_refs)
-                        failed |= css_refcnt(css) != 1;
-        }
-        /*
-         * If succeeded, set REMOVED and put all the base refs; otherwise,
-         * restore refcnts to positive values.  Either way, all in-progress
-         * css_tryget() will be released.
-         */
-        for_each_subsys(cgrp->root, ss) {
-                struct cgroup_subsys_state *css = cgrp->subsys[ss->subsys_id];
-                if (!failed) {
-                        set_bit(CSS_REMOVED, &css->flags);
-                        css_put(css);
-                } else {
-                        atomic_sub(CSS_DEACT_BIAS, &css->refcnt);
-                }
        }
+        set_bit(CGRP_REMOVED, &cgrp->flags);
-        local_irq_restore(flags);
+        /* tell subsystems to initate destruction */
-        return !failed;
+        for_each_subsys(cgrp->root, ss)
-}
+                offline_css(ss, cgrp);
-static int cgroup_rmdir(struct inode *unused_dir, struct dentry *dentry)
-{
-        struct cgroup *cgrp = dentry->d_fsdata;
-        struct dentry *d;
-        struct cgroup *parent;
-        DEFINE_WAIT(wait);
-        struct cgroup_event *event, *tmp;
-        int ret;
-        /* the vfs holds both inode->i_mutex already */
-again:
-        mutex_lock(&cgroup_mutex);
-        if (atomic_read(&cgrp->count) != 0) {
-                mutex_unlock(&cgroup_mutex);
-                return -EBUSY;
-        }
-        if (!list_empty(&cgrp->children)) {
-                mutex_unlock(&cgroup_mutex);
-                return -EBUSY;
-        }
-        mutex_unlock(&cgroup_mutex);
-        /*
-         * In general, subsystem has no css->refcnt after pre_destroy(). But
-         * in racy cases, subsystem may have to get css->refcnt after
-         * pre_destroy() and it makes rmdir return with -EBUSY. This sometimes
-         * make rmdir return -EBUSY too often. To avoid that, we use waitqueue
-         * for cgroup's rmdir. CGRP_WAIT_ON_RMDIR is for synchronizing rmdir
-         * and subsystem's reference count handling. Please see css_get/put
-         * and css_tryget() and cgroup_wakeup_rmdir_waiter() implementation.
-         */
-        set_bit(CGRP_WAIT_ON_RMDIR, &cgrp->flags);
        /*
-         * Call pre_destroy handlers of subsys. Notify subsystems
+         * Put all the base refs.  Each css holds an extra reference to the
-         * that rmdir() request comes.
+         * cgroup's dentry and cgroup removal proceeds regardless of css
+         * refs.  On the last put of each css, whenever that may be, the
+         * extra dentry ref is put so that dentry destruction happens only
+         * after all css's are released.
         */
-        ret = cgroup_call_pre_destroy(cgrp);
+        for_each_subsys(cgrp->root, ss)
-        if (ret) {
+                css_put(cgrp->subsys[ss->subsys_id]);
-                clear_bit(CGRP_WAIT_ON_RMDIR, &cgrp->flags);
-                return ret;
-        }
-        mutex_lock(&cgroup_mutex);
-        parent = cgrp->parent;
-        if (atomic_read(&cgrp->count) || !list_empty(&cgrp->children)) {
-                clear_bit(CGRP_WAIT_ON_RMDIR, &cgrp->flags);
-                mutex_unlock(&cgroup_mutex);
-                return -EBUSY;
-        }
-        prepare_to_wait(&cgroup_rmdir_waitq, &wait, TASK_INTERRUPTIBLE);
-        if (!cgroup_clear_css_refs(cgrp)) {
-                mutex_unlock(&cgroup_mutex);
-                /*
-                 * Because someone may call cgroup_wakeup_rmdir_waiter() before
-                 * prepare_to_wait(), we need to check this flag.
-                 */
-                if (test_bit(CGRP_WAIT_ON_RMDIR, &cgrp->flags))
-                        schedule();
-                finish_wait(&cgroup_rmdir_waitq, &wait);
-                clear_bit(CGRP_WAIT_ON_RMDIR, &cgrp->flags);
-                if (signal_pending(current))
-                        return -EINTR;
-                goto again;
-        }
-        /* NO css_tryget() can success after here. */
-        finish_wait(&cgroup_rmdir_waitq, &wait);
-        clear_bit(CGRP_WAIT_ON_RMDIR, &cgrp->flags);
        raw_spin_lock(&release_list_lock);
-        set_bit(CGRP_REMOVED, &cgrp->flags);
        if (!list_empty(&cgrp->release_list))
                list_del_init(&cgrp->release_list);
        raw_spin_unlock(&release_list_lock);
        /* delete this cgroup from parent->children */
-        list_del_init(&cgrp->sibling);
+        list_del_rcu(&cgrp->sibling);
        list_del_init(&cgrp->allcg_node);
-        d = dget(cgrp->dentry);
+        dget(d);
        cgroup_d_remove_dir(d);
        dput(d);
@@ -4353,21 +4340,35 @@ again:
        /*
         * Unregister events and notify userspace.
         * Notify userspace about cgroup removing only after rmdir of cgroup
-         * directory to avoid race between userspace and kernelspace
+         * directory to avoid race between userspace and kernelspace. Use
+         * a temporary list to avoid a deadlock with cgroup_event_wake(). Since
+         * cgroup_event_wake() is called with the wait queue head locked,
+         * remove_wait_queue() cannot be called while holding event_list_lock.
         */
        spin_lock(&cgrp->event_list_lock);
-        list_for_each_entry_safe(event, tmp, &cgrp->event_list, list) {
+        list_splice_init(&cgrp->event_list, &tmp_list);
-                list_del(&event->list);
+        spin_unlock(&cgrp->event_list_lock);
+        list_for_each_entry_safe(event, tmp, &tmp_list, list) {
+                list_del_init(&event->list);
                remove_wait_queue(event->wqh, &event->wait);
                eventfd_signal(event->eventfd, 1);
                schedule_work(&event->remove);
        }
-        spin_unlock(&cgrp->event_list_lock);
-        mutex_unlock(&cgroup_mutex);
        return 0;
 }
+static int cgroup_rmdir(struct inode *unused_dir, struct dentry *dentry)
+{
+        int ret;
+        mutex_lock(&cgroup_mutex);
+        ret = cgroup_destroy_locked(dentry->d_fsdata);
+        mutex_unlock(&cgroup_mutex);
+        return ret;
+}
 static void __init_or_module cgroup_init_cftsets(struct cgroup_subsys *ss)
 {
        INIT_LIST_HEAD(&ss->cftsets);
@@ -4388,13 +4389,15 @@ static void __init cgroup_init_subsys(struct cgroup_subsys *ss)
        printk(KERN_INFO "Initializing cgroup subsys %s\n", ss->name);
+        mutex_lock(&cgroup_mutex);
        /* init base cftset */
        cgroup_init_cftsets(ss);
        /* Create the top cgroup state for this subsystem */
        list_add(&ss->sibling, &rootnode.subsys_list);
        ss->root = &rootnode;
-        css = ss->create(dummytop);
+        css = ss->css_alloc(dummytop);
        /* We don't handle early failures gracefully */
        BUG_ON(IS_ERR(css));
        init_cgroup_css(css, ss, dummytop);
@@ -4403,7 +4406,7 @@ static void __init cgroup_init_subsys(struct cgroup_subsys *ss)
         * pointer to this state - since the subsystem is
         * newly registered, all tasks and hence the
         * init_css_set is in the subsystem's top cgroup. */
-        init_css_set.subsys[ss->subsys_id] = dummytop->subsys[ss->subsys_id];
+        init_css_set.subsys[ss->subsys_id] = css;
        need_forkexit_callback |= ss->fork || ss->exit;
@@ -4413,6 +4416,9 @@ static void __init cgroup_init_subsys(struct cgroup_subsys *ss)
        BUG_ON(!list_empty(&init_task.tasks));
        ss->active = 1;
+        BUG_ON(online_css(ss, dummytop));
+        mutex_unlock(&cgroup_mutex);
        /* this function shouldn't be used with modular subsystems, since they
         * need to register a subsys_id, among other things */
@@ -4430,12 +4436,12 @@ static void __init cgroup_init_subsys(struct cgroup_subsys *ss)
 */
 int __init_or_module cgroup_load_subsys(struct cgroup_subsys *ss)
 {
-        int i;
        struct cgroup_subsys_state *css;
+        int i, ret;
        /* check name and function validity */
        if (ss->name == NULL || strlen(ss->name) > MAX_CGROUP_TYPE_NAMELEN ||
-            ss->create == NULL || ss->destroy == NULL)
+            ss->css_alloc == NULL || ss->css_free == NULL)
                return -EINVAL;
        /*
@@ -4464,10 +4470,11 @@ int __init_or_module cgroup_load_subsys(struct cgroup_subsys *ss)
        subsys[ss->subsys_id] = ss;
        /*
-         * no ss->create seems to need anything important in the ss struct, so
+         * no ss->css_alloc seems to need anything important in the ss
-         * this can happen first (i.e. before the rootnode attachment).
+         * struct, so this can happen first (i.e. before the rootnode
+         * attachment).
         */
-        css = ss->create(dummytop);
+        css = ss->css_alloc(dummytop);
        if (IS_ERR(css)) {
                /* failure case - need to deassign the subsys[] slot. */
                subsys[ss->subsys_id] = NULL;
@@ -4482,14 +4489,9 @@ int __init_or_module cgroup_load_subsys(struct cgroup_subsys *ss)
        init_cgroup_css(css, ss, dummytop);
        /* init_idr must be after init_cgroup_css because it sets css->id. */
        if (ss->use_id) {
-                int ret = cgroup_init_idr(ss, css);
+                ret = cgroup_init_idr(ss, css);
-                if (ret) {
+                if (ret)
-                        dummytop->subsys[ss->subsys_id] = NULL;
+                        goto err_unload;
-                        ss->destroy(dummytop);
-                        subsys[ss->subsys_id] = NULL;
-                        mutex_unlock(&cgroup_mutex);
-                        return ret;
-                }
        }
        /*
@@ -4522,10 +4524,19 @@ int __init_or_module cgroup_load_subsys(struct cgroup_subsys *ss)
        write_unlock(&css_set_lock);
        ss->active = 1;
+        ret = online_css(ss, dummytop);
+        if (ret)
+                goto err_unload;
        /* success! */
        mutex_unlock(&cgroup_mutex);
        return 0;
+err_unload:
+        mutex_unlock(&cgroup_mutex);
+        /* @ss can't be mounted here as try_module_get() would fail */
+        cgroup_unload_subsys(ss);
+        return ret;
 }
 EXPORT_SYMBOL_GPL(cgroup_load_subsys);
@@ -4552,6 +4563,15 @@ void cgroup_unload_subsys(struct cgroup_subsys *ss)
        BUG_ON(ss->root != &rootnode);
        mutex_lock(&cgroup_mutex);
+        offline_css(ss, dummytop);
+        ss->active = 0;
+        if (ss->use_id) {
+                idr_remove_all(&ss->idr);
+                idr_destroy(&ss->idr);
+        }
        /* deassign the subsys_id */
        subsys[ss->subsys_id] = NULL;
@@ -4567,7 +4587,6 @@ void cgroup_unload_subsys(struct cgroup_subsys *ss)
                struct css_set *cg = link->cg;
                hlist_del(&cg->hlist);
-                BUG_ON(!cg->subsys[ss->subsys_id]);
                cg->subsys[ss->subsys_id] = NULL;
                hhead = css_set_hash(cg->subsys);
                hlist_add_head(&cg->hlist, hhead);
@@ -4575,12 +4594,12 @@ void cgroup_unload_subsys(struct cgroup_subsys *ss)
        write_unlock(&css_set_lock);
        /*
-         * remove subsystem's css from the dummytop and free it - need to free
+         * remove subsystem's css from the dummytop and free it - need to
-         * before marking as null because ss->destroy needs the cgrp->subsys
+         * free before marking as null because ss->css_free needs the
-         * pointer to find their state. note that this also takes care of
+         * cgrp->subsys pointer to find their state. note that this also
-         * freeing the css_id.
+         * takes care of freeing the css_id.
         */
-        ss->destroy(dummytop);
+        ss->css_free(dummytop);
        dummytop->subsys[ss->subsys_id] = NULL;
        mutex_unlock(&cgroup_mutex);
@@ -4624,8 +4643,8 @@ int __init cgroup_init_early(void)
                BUG_ON(!ss->name);
                BUG_ON(strlen(ss->name) > MAX_CGROUP_TYPE_NAMELEN);
-                BUG_ON(!ss->create);
+                BUG_ON(!ss->css_alloc);
-                BUG_ON(!ss->destroy);
+                BUG_ON(!ss->css_free);
                if (ss->subsys_id != i) {
                        printk(KERN_ERR "cgroup: Subsys %s id == %d\n",
                               ss->name, ss->subsys_id);
@@ -4832,44 +4851,19 @@ void cgroup_fork(struct task_struct *child)
 }
 /**
- * cgroup_fork_callbacks - run fork callbacks
- * @child: the new task
- *
- * Called on a new task very soon before adding it to the
- * tasklist. No need to take any locks since no-one can
- * be operating on this task.
- */
-void cgroup_fork_callbacks(struct task_struct *child)
-{
-        if (need_forkexit_callback) {
-                int i;
-                for (i = 0; i < CGROUP_SUBSYS_COUNT; i++) {
-                        struct cgroup_subsys *ss = subsys[i];
-                        /*
-                         * forkexit callbacks are only supported for
-                         * builtin subsystems.
-                         */
-                        if (!ss || ss->module)
-                                continue;
-                        if (ss->fork)
-                                ss->fork(child);
-                }
-        }
-}
-/**
 * cgroup_post_fork - called on a new task after adding it to the task list
 * @child: the task in question
 *
- * Adds the task to the list running through its css_set if necessary.
+ * Adds the task to the list running through its css_set if necessary and
- * Has to be after the task is visible on the task list in case we race
+ * call the subsystem fork() callbacks.  Has to be after the task is
- * with the first call to cgroup_iter_start() - to guarantee that the
+ * visible on the task list in case we race with the first call to
- * new task ends up on its list.
+ * cgroup_iter_start() - to guarantee that the new task ends up on its
+ * list.
 */
 void cgroup_post_fork(struct task_struct *child)
 {
+        int i;
        /*
         * use_task_css_set_links is set to 1 before we walk the tasklist
         * under the tasklist_lock and we read it here after we added the child
@@ -4889,7 +4883,30 @@ void cgroup_post_fork(struct task_struct *child)
                task_unlock(child);
                write_unlock(&css_set_lock);
        }
+        /*
+         * Call ss->fork().  This must happen after @child is linked on
+         * css_set; otherwise, @child might change state between ->fork()
+         * and addition to css_set.
+         */
+        if (need_forkexit_callback) {
+                for (i = 0; i < CGROUP_SUBSYS_COUNT; i++) {
+                        struct cgroup_subsys *ss = subsys[i];
+                        /*
+                         * fork/exit callbacks are supported only for
+                         * builtin subsystems and we don't need further
+                         * synchronization as they never go away.
+                         */
+                        if (!ss || ss->module)
+                                continue;
+                        if (ss->fork)
+                                ss->fork(child);
+                }
+        }
 }
 /**
 * cgroup_exit - detach cgroup from exiting task
 * @tsk: pointer to task_struct of exiting process
@@ -5022,15 +5039,17 @@ static void check_for_release(struct cgroup *cgrp)
 /* Caller must verify that the css is not for root cgroup */
 bool __css_tryget(struct cgroup_subsys_state *css)
 {
-        do {
+        while (true) {
-                int v = css_refcnt(css);
+                int t, v;
-                if (atomic_cmpxchg(&css->refcnt, v, v + 1) == v)
+                v = css_refcnt(css);
+                t = atomic_cmpxchg(&css->refcnt, v, v + 1);
+                if (likely(t == v))
                        return true;
+                else if (t < 0)
+                        return false;
                cpu_relax();
-        } while (!test_bit(CSS_REMOVED, &css->flags));
+        }
-        return false;
 }
 EXPORT_SYMBOL_GPL(__css_tryget);
@@ -5049,11 +5068,9 @@ void __css_put(struct cgroup_subsys_state *css)
                        set_bit(CGRP_RELEASABLE, &cgrp->flags);
                        check_for_release(cgrp);
                }
-                cgroup_wakeup_rmdir_waiter(cgrp);
                break;
        case 0:
-                if (!test_bit(CSS_CLEAR_CSS_REFS, &css->flags))
+                schedule_work(&css->dput_work);
-                        schedule_work(&css->dput_work);
                break;
        }
        rcu_read_unlock();
@@ -5439,7 +5456,7 @@ struct cgroup_subsys_state *cgroup_css_from_dir(struct file *f, int id)
 }
 #ifdef CONFIG_CGROUP_DEBUG
-static struct cgroup_subsys_state *debug_create(struct cgroup *cont)
+static struct cgroup_subsys_state *debug_css_alloc(struct cgroup *cont)
 {
        struct cgroup_subsys_state *css = kzalloc(sizeof(*css), GFP_KERNEL);
@@ -5449,7 +5466,7 @@ static struct cgroup_subsys_state *debug_create(struct cgroup *cont)
        return css;
 }
-static void debug_destroy(struct cgroup *cont)
+static void debug_css_free(struct cgroup *cont)
 {
        kfree(cont->subsys[debug_subsys_id]);
 }
@@ -5578,8 +5595,8 @@ static struct cftype debug_files[] =  {
 struct cgroup_subsys debug_subsys = {
        .name = "debug",
-        .create = debug_create,
+        .css_alloc = debug_css_alloc,
-        .destroy = debug_destroy,
+        .css_free = debug_css_free,
        .subsys_id = debug_subsys_id,
        .base_cftypes = debug_files,
 };
diff --git a/kernel/cgroup_freezer.c b/kernel/cgroup_freezer.c
index b1724ce98981..75dda1ea5026 100644
--- a/kernel/cgroup_freezer.c
+++ b/kernel/cgroup_freezer.c
@@ -22,24 +22,33 @@
 #include <linux/freezer.h>
 #include <linux/seq_file.h>
-enum freezer_state {
+/*
-        CGROUP_THAWED = 0,
+ * A cgroup is freezing if any FREEZING flags are set.  FREEZING_SELF is
-        CGROUP_FREEZING,
+ * set if "FROZEN" is written to freezer.state cgroupfs file, and cleared
-        CGROUP_FROZEN,
+ * for "THAWED".  FREEZING_PARENT is set if the parent freezer is FREEZING
+ * for whatever reason.  IOW, a cgroup has FREEZING_PARENT set if one of
+ * its ancestors has FREEZING_SELF set.
+ */
+enum freezer_state_flags {
+        CGROUP_FREEZER_ONLINE   = (1 << 0), /* freezer is fully online */
+        CGROUP_FREEZING_SELF    = (1 << 1), /* this freezer is freezing */
+        CGROUP_FREEZING_PARENT  = (1 << 2), /* the parent freezer is freezing */
+        CGROUP_FROZEN           = (1 << 3), /* this and its descendants frozen */
+        /* mask for all FREEZING flags */
+        CGROUP_FREEZING         = CGROUP_FREEZING_SELF | CGROUP_FREEZING_PARENT,
 };
 struct freezer {
-        struct cgroup_subsys_state css;
+        struct cgroup_subsys_state      css;
-        enum freezer_state state;
+        unsigned int                    state;
-        spinlock_t lock; /* protects _writes_ to state */
+        spinlock_t                      lock;
 };
-static inline struct freezer *cgroup_freezer(
+static inline struct freezer *cgroup_freezer(struct cgroup *cgroup)
-                struct cgroup *cgroup)
 {
-        return container_of(
+        return container_of(cgroup_subsys_state(cgroup, freezer_subsys_id),
-                cgroup_subsys_state(cgroup, freezer_subsys_id),
+                            struct freezer, css);
-                struct freezer, css);
 }
 static inline struct freezer *task_freezer(struct task_struct *task)
@@ -48,14 +57,21 @@ static inline struct freezer *task_freezer(struct task_struct *task)
                            struct freezer, css);
 }
+static struct freezer *parent_freezer(struct freezer *freezer)
+{
+        struct cgroup *pcg = freezer->css.cgroup->parent;
+        if (pcg)
+                return cgroup_freezer(pcg);
+        return NULL;
+}
 bool cgroup_freezing(struct task_struct *task)
 {
-        enum freezer_state state;
        bool ret;
        rcu_read_lock();
-        state = task_freezer(task)->state;
+        ret = task_freezer(task)->state & CGROUP_FREEZING;
-        ret = state == CGROUP_FREEZING || state == CGROUP_FROZEN;
        rcu_read_unlock();
        return ret;
@@ -65,70 +81,18 @@ bool cgroup_freezing(struct task_struct *task)
 * cgroups_write_string() limits the size of freezer state strings to
 * CGROUP_LOCAL_BUFFER_SIZE
 */
-static const char *freezer_state_strs[] = {
+static const char *freezer_state_strs(unsigned int state)
-        "THAWED",
+{
-        "FREEZING",
+        if (state & CGROUP_FROZEN)
-        "FROZEN",
+                return "FROZEN";
+        if (state & CGROUP_FREEZING)
+                return "FREEZING";
+        return "THAWED";
 };
-/*
- * State diagram
- * Transitions are caused by userspace writes to the freezer.state file.
- * The values in parenthesis are state labels. The rest are edge labels.
- *
- * (THAWED) --FROZEN--> (FREEZING) --FROZEN--> (FROZEN)
- *    ^ ^                    |                     |
- *    | \_______THAWED_______/                     |
- *    \__________________________THAWED____________/
- */
 struct cgroup_subsys freezer_subsys;
-/* Locks taken and their ordering
+static struct cgroup_subsys_state *freezer_css_alloc(struct cgroup *cgroup)
- * ------------------------------
- * cgroup_mutex (AKA cgroup_lock)
- * freezer->lock
- * css_set_lock
- * task->alloc_lock (AKA task_lock)
- * task->sighand->siglock
- *
- * cgroup code forces css_set_lock to be taken before task->alloc_lock
- *
- * freezer_create(), freezer_destroy():
- * cgroup_mutex [ by cgroup core ]
- *
- * freezer_can_attach():
- * cgroup_mutex (held by caller of can_attach)
- *
- * freezer_fork() (preserving fork() performance means can't take cgroup_mutex):
- * freezer->lock
- *  sighand->siglock (if the cgroup is freezing)
- *
- * freezer_read():
- * cgroup_mutex
- *  freezer->lock
- *   write_lock css_set_lock (cgroup iterator start)
- *    task->alloc_lock
- *   read_lock css_set_lock (cgroup iterator start)
- *
- * freezer_write() (freeze):
- * cgroup_mutex
- *  freezer->lock
- *   write_lock css_set_lock (cgroup iterator start)
- *    task->alloc_lock
- *   read_lock css_set_lock (cgroup iterator start)
- *    sighand->siglock (fake signal delivery inside freeze_task())
- *
- * freezer_write() (unfreeze):
- * cgroup_mutex
- *  freezer->lock
- *   write_lock css_set_lock (cgroup iterator start)
- *    task->alloc_lock
- *   read_lock css_set_lock (cgroup iterator start)
- *    task->alloc_lock (inside __thaw_task(), prevents race with refrigerator())
- *     sighand->siglock
- */
-static struct cgroup_subsys_state *freezer_create(struct cgroup *cgroup)
 {
        struct freezer *freezer;
@@ -137,160 +101,244 @@ static struct cgroup_subsys_state *freezer_create(struct cgroup *cgroup)
                return ERR_PTR(-ENOMEM);
        spin_lock_init(&freezer->lock);
-        freezer->state = CGROUP_THAWED;
        return &freezer->css;
 }
-static void freezer_destroy(struct cgroup *cgroup)
+/**
+ * freezer_css_online - commit creation of a freezer cgroup
+ * @cgroup: cgroup being created
+ *
+ * We're committing to creation of @cgroup.  Mark it online and inherit
+ * parent's freezing state while holding both parent's and our
+ * freezer->lock.
+ */
+static int freezer_css_online(struct cgroup *cgroup)
+{
+        struct freezer *freezer = cgroup_freezer(cgroup);
+        struct freezer *parent = parent_freezer(freezer);
+        /*
+         * The following double locking and freezing state inheritance
+         * guarantee that @cgroup can never escape ancestors' freezing
+         * states.  See cgroup_for_each_descendant_pre() for details.
+         */
+        if (parent)
+                spin_lock_irq(&parent->lock);
+        spin_lock_nested(&freezer->lock, SINGLE_DEPTH_NESTING);
+        freezer->state |= CGROUP_FREEZER_ONLINE;
+        if (parent && (parent->state & CGROUP_FREEZING)) {
+                freezer->state |= CGROUP_FREEZING_PARENT | CGROUP_FROZEN;
+                atomic_inc(&system_freezing_cnt);
+        }
+        spin_unlock(&freezer->lock);
+        if (parent)
+                spin_unlock_irq(&parent->lock);
+        return 0;
+}
+/**
+ * freezer_css_offline - initiate destruction of @cgroup
+ * @cgroup: cgroup being destroyed
+ *
+ * @cgroup is going away.  Mark it dead and decrement system_freezing_count
+ * if it was holding one.
+ */
+static void freezer_css_offline(struct cgroup *cgroup)
 {
        struct freezer *freezer = cgroup_freezer(cgroup);
-        if (freezer->state != CGROUP_THAWED)
+        spin_lock_irq(&freezer->lock);
+        if (freezer->state & CGROUP_FREEZING)
                atomic_dec(&system_freezing_cnt);
-        kfree(freezer);
+        freezer->state = 0;
+        spin_unlock_irq(&freezer->lock);
 }
-/* task is frozen or will freeze immediately when next it gets woken */
+static void freezer_css_free(struct cgroup *cgroup)
-static bool is_task_frozen_enough(struct task_struct *task)
 {
-        return frozen(task) ||
+        kfree(cgroup_freezer(cgroup));
-                (task_is_stopped_or_traced(task) && freezing(task));
 }
 /*
- * The call to cgroup_lock() in the freezer.state write method prevents
+ * Tasks can be migrated into a different freezer anytime regardless of its
- * a write to that file racing against an attach, and hence the
+ * current state.  freezer_attach() is responsible for making new tasks
- * can_attach() result will remain valid until the attach completes.
+ * conform to the current state.
+ *
+ * Freezer state changes and task migration are synchronized via
+ * @freezer->lock.  freezer_attach() makes the new tasks conform to the
+ * current state and all following state changes can see the new tasks.
 */
-static int freezer_can_attach(struct cgroup *new_cgroup,
+static void freezer_attach(struct cgroup *new_cgrp, struct cgroup_taskset *tset)
-                              struct cgroup_taskset *tset)
 {
-        struct freezer *freezer;
+        struct freezer *freezer = cgroup_freezer(new_cgrp);
        struct task_struct *task;
+        bool clear_frozen = false;
+        spin_lock_irq(&freezer->lock);
        /*
-         * Anything frozen can't move or be moved to/from.
+         * Make the new tasks conform to the current state of @new_cgrp.
+         * For simplicity, when migrating any task to a FROZEN cgroup, we
+         * revert it to FREEZING and let update_if_frozen() determine the
+         * correct state later.
+         *
+         * Tasks in @tset are on @new_cgrp but may not conform to its
+         * current state before executing the following - !frozen tasks may
+         * be visible in a FROZEN cgroup and frozen tasks in a THAWED one.
         */
-        cgroup_taskset_for_each(task, new_cgroup, tset)
+        cgroup_taskset_for_each(task, new_cgrp, tset) {
-                if (cgroup_freezing(task))
+                if (!(freezer->state & CGROUP_FREEZING)) {
-                        return -EBUSY;
+                        __thaw_task(task);
+                } else {
+                        freeze_task(task);
+                        freezer->state &= ~CGROUP_FROZEN;
+                        clear_frozen = true;
+                }
+        }
-        freezer = cgroup_freezer(new_cgroup);
+        spin_unlock_irq(&freezer->lock);
-        if (freezer->state != CGROUP_THAWED)
-                return -EBUSY;
-        return 0;
+        /*
+         * Propagate FROZEN clearing upwards.  We may race with
+         * update_if_frozen(), but as long as both work bottom-up, either
+         * update_if_frozen() sees child's FROZEN cleared or we clear the
+         * parent's FROZEN later.  No parent w/ !FROZEN children can be
+         * left FROZEN.
+         */
+        while (clear_frozen && (freezer = parent_freezer(freezer))) {
+                spin_lock_irq(&freezer->lock);
+                freezer->state &= ~CGROUP_FROZEN;
+                clear_frozen = freezer->state & CGROUP_FREEZING;
+                spin_unlock_irq(&freezer->lock);
+        }
 }
 static void freezer_fork(struct task_struct *task)
 {
        struct freezer *freezer;
-        /*
-         * No lock is needed, since the task isn't on tasklist yet,
-         * so it can't be moved to another cgroup, which means the
-         * freezer won't be removed and will be valid during this
-         * function call.  Nevertheless, apply RCU read-side critical
-         * section to suppress RCU lockdep false positives.
-         */
        rcu_read_lock();
        freezer = task_freezer(task);
-        rcu_read_unlock();
        /*
         * The root cgroup is non-freezable, so we can skip the
         * following check.
         */
        if (!freezer->css.cgroup->parent)
-                return;
+                goto out;
        spin_lock_irq(&freezer->lock);
-        BUG_ON(freezer->state == CGROUP_FROZEN);
+        if (freezer->state & CGROUP_FREEZING)
-        /* Locking avoids race with FREEZING -> THAWED transitions. */
-        if (freezer->state == CGROUP_FREEZING)
                freeze_task(task);
        spin_unlock_irq(&freezer->lock);
+out:
+        rcu_read_unlock();
 }
-/*
+/**
- * caller must hold freezer->lock
+ * update_if_frozen - update whether a cgroup finished freezing
+ * @cgroup: cgroup of interest
+ *
+ * Once FREEZING is initiated, transition to FROZEN is lazily updated by
+ * calling this function.  If the current state is FREEZING but not FROZEN,
+ * this function checks whether all tasks of this cgroup and the descendant
+ * cgroups finished freezing and, if so, sets FROZEN.
+ *
+ * The caller is responsible for grabbing RCU read lock and calling
+ * update_if_frozen() on all descendants prior to invoking this function.
+ *
+ * Task states and freezer state might disagree while tasks are being
+ * migrated into or out of @cgroup, so we can't verify task states against
+ * @freezer state here.  See freezer_attach() for details.
 */
-static void update_if_frozen(struct cgroup *cgroup,
+static void update_if_frozen(struct cgroup *cgroup)
-                                 struct freezer *freezer)
 {
+        struct freezer *freezer = cgroup_freezer(cgroup);
+        struct cgroup *pos;
        struct cgroup_iter it;
        struct task_struct *task;
-        unsigned int nfrozen = 0, ntotal = 0;
-        enum freezer_state old_state = freezer->state;
-        cgroup_iter_start(cgroup, &it);
+        WARN_ON_ONCE(!rcu_read_lock_held());
-        while ((task = cgroup_iter_next(cgroup, &it))) {
-                ntotal++;
+        spin_lock_irq(&freezer->lock);
-                if (freezing(task) && is_task_frozen_enough(task))
-                        nfrozen++;
+        if (!(freezer->state & CGROUP_FREEZING) ||
+            (freezer->state & CGROUP_FROZEN))
+                goto out_unlock;
+        /* are all (live) children frozen? */
+        cgroup_for_each_child(pos, cgroup) {
+                struct freezer *child = cgroup_freezer(pos);
+                if ((child->state & CGROUP_FREEZER_ONLINE) &&
+                    !(child->state & CGROUP_FROZEN))
+                        goto out_unlock;
        }
-        if (old_state == CGROUP_THAWED) {
+        /* are all tasks frozen? */
-                BUG_ON(nfrozen > 0);
+        cgroup_iter_start(cgroup, &it);
-        } else if (old_state == CGROUP_FREEZING) {
-                if (nfrozen == ntotal)
+        while ((task = cgroup_iter_next(cgroup, &it))) {
-                        freezer->state = CGROUP_FROZEN;
+                if (freezing(task)) {
-        } else { /* old_state == CGROUP_FROZEN */
+                        /*
-                BUG_ON(nfrozen != ntotal);
+                         * freezer_should_skip() indicates that the task
+                         * should be skipped when determining freezing
+                         * completion.  Consider it frozen in addition to
+                         * the usual frozen condition.
+                         */
+                        if (!frozen(task) && !freezer_should_skip(task))
+                                goto out_iter_end;
+                }
        }
+        freezer->state |= CGROUP_FROZEN;
+out_iter_end:
        cgroup_iter_end(cgroup, &it);
+out_unlock:
+        spin_unlock_irq(&freezer->lock);
 }
 static int freezer_read(struct cgroup *cgroup, struct cftype *cft,
                        struct seq_file *m)
 {
-        struct freezer *freezer;
+        struct cgroup *pos;
-        enum freezer_state state;
-        if (!cgroup_lock_live_group(cgroup))
+        rcu_read_lock();
-                return -ENODEV;
-        freezer = cgroup_freezer(cgroup);
+        /* update states bottom-up */
-        spin_lock_irq(&freezer->lock);
+        cgroup_for_each_descendant_post(pos, cgroup)
-        state = freezer->state;
+                update_if_frozen(pos);
-        if (state == CGROUP_FREEZING) {
+        update_if_frozen(cgroup);
-                /* We change from FREEZING to FROZEN lazily if the cgroup was
-                 * only partially frozen when we exitted write. */
+        rcu_read_unlock();
-                update_if_frozen(cgroup, freezer);
-                state = freezer->state;
-        }
-        spin_unlock_irq(&freezer->lock);
-        cgroup_unlock();
-        seq_puts(m, freezer_state_strs[state]);
+        seq_puts(m, freezer_state_strs(cgroup_freezer(cgroup)->state));
        seq_putc(m, '\n');
        return 0;
 }
-static int try_to_freeze_cgroup(struct cgroup *cgroup, struct freezer *freezer)
+static void freeze_cgroup(struct freezer *freezer)
 {
+        struct cgroup *cgroup = freezer->css.cgroup;
        struct cgroup_iter it;
        struct task_struct *task;
-        unsigned int num_cant_freeze_now = 0;
        cgroup_iter_start(cgroup, &it);
-        while ((task = cgroup_iter_next(cgroup, &it))) {
+        while ((task = cgroup_iter_next(cgroup, &it)))
-                if (!freeze_task(task))
+                freeze_task(task);
-                        continue;
-                if (is_task_frozen_enough(task))
-                        continue;
-                if (!freezing(task) && !freezer_should_skip(task))
-                        num_cant_freeze_now++;
-        }
        cgroup_iter_end(cgroup, &it);
-        return num_cant_freeze_now ? -EBUSY : 0;
 }
-static void unfreeze_cgroup(struct cgroup *cgroup, struct freezer *freezer)
+static void unfreeze_cgroup(struct freezer *freezer)
 {
+        struct cgroup *cgroup = freezer->css.cgroup;
        struct cgroup_iter it;
        struct task_struct *task;
@@ -300,59 +348,111 @@ static void unfreeze_cgroup(struct cgroup *cgroup, struct freezer *freezer)
        cgroup_iter_end(cgroup, &it);
 }
-static int freezer_change_state(struct cgroup *cgroup,
+/**
-                                enum freezer_state goal_state)
+ * freezer_apply_state - apply state change to a single cgroup_freezer
+ * @freezer: freezer to apply state change to
+ * @freeze: whether to freeze or unfreeze
+ * @state: CGROUP_FREEZING_* flag to set or clear
+ *
+ * Set or clear @state on @cgroup according to @freeze, and perform
+ * freezing or thawing as necessary.
+ */
+static void freezer_apply_state(struct freezer *freezer, bool freeze,
+                                unsigned int state)
 {
-        struct freezer *freezer;
+        /* also synchronizes against task migration, see freezer_attach() */
-        int retval = 0;
+        lockdep_assert_held(&freezer->lock);
-        freezer = cgroup_freezer(cgroup);
-        spin_lock_irq(&freezer->lock);
+        if (!(freezer->state & CGROUP_FREEZER_ONLINE))
+                return;
-        update_if_frozen(cgroup, freezer);
+        if (freeze) {
+                if (!(freezer->state & CGROUP_FREEZING))
-        switch (goal_state) {
-        case CGROUP_THAWED:
-                if (freezer->state != CGROUP_THAWED)
-                        atomic_dec(&system_freezing_cnt);
-                freezer->state = CGROUP_THAWED;
-                unfreeze_cgroup(cgroup, freezer);
-                break;
-        case CGROUP_FROZEN:
-                if (freezer->state == CGROUP_THAWED)
                        atomic_inc(&system_freezing_cnt);
-                freezer->state = CGROUP_FREEZING;
+                freezer->state |= state;
-                retval = try_to_freeze_cgroup(cgroup, freezer);
+                freeze_cgroup(freezer);
-                break;
+        } else {
-        default:
+                bool was_freezing = freezer->state & CGROUP_FREEZING;
-                BUG();
+                freezer->state &= ~state;
+                if (!(freezer->state & CGROUP_FREEZING)) {
+                        if (was_freezing)
+                                atomic_dec(&system_freezing_cnt);
+                        freezer->state &= ~CGROUP_FROZEN;
+                        unfreeze_cgroup(freezer);
+                }
        }
+}
+/**
+ * freezer_change_state - change the freezing state of a cgroup_freezer
+ * @freezer: freezer of interest
+ * @freeze: whether to freeze or thaw
+ *
+ * Freeze or thaw @freezer according to @freeze.  The operations are
+ * recursive - all descendants of @freezer will be affected.
+ */
+static void freezer_change_state(struct freezer *freezer, bool freeze)
+{
+        struct cgroup *pos;
+        /* update @freezer */
+        spin_lock_irq(&freezer->lock);
+        freezer_apply_state(freezer, freeze, CGROUP_FREEZING_SELF);
        spin_unlock_irq(&freezer->lock);
-        return retval;
+        /*
+         * Update all its descendants in pre-order traversal.  Each
+         * descendant will try to inherit its parent's FREEZING state as
+         * CGROUP_FREEZING_PARENT.
+         */
+        rcu_read_lock();
+        cgroup_for_each_descendant_pre(pos, freezer->css.cgroup) {
+                struct freezer *pos_f = cgroup_freezer(pos);
+                struct freezer *parent = parent_freezer(pos_f);
+                /*
+                 * Our update to @parent->state is already visible which is
+                 * all we need.  No need to lock @parent.  For more info on
+                 * synchronization, see freezer_post_create().
+                 */
+                spin_lock_irq(&pos_f->lock);
+                freezer_apply_state(pos_f, parent->state & CGROUP_FREEZING,
+                                    CGROUP_FREEZING_PARENT);
+                spin_unlock_irq(&pos_f->lock);
+        }
+        rcu_read_unlock();
 }
-static int freezer_write(struct cgroup *cgroup,
+static int freezer_write(struct cgroup *cgroup, struct cftype *cft,
-                         struct cftype *cft,
                         const char *buffer)
 {
-        int retval;
+        bool freeze;
-        enum freezer_state goal_state;
-        if (strcmp(buffer, freezer_state_strs[CGROUP_THAWED]) == 0)
+        if (strcmp(buffer, freezer_state_strs(0)) == 0)
-                goal_state = CGROUP_THAWED;
+                freeze = false;
-        else if (strcmp(buffer, freezer_state_strs[CGROUP_FROZEN]) == 0)
+        else if (strcmp(buffer, freezer_state_strs(CGROUP_FROZEN)) == 0)
-                goal_state = CGROUP_FROZEN;
+                freeze = true;
        else
                return -EINVAL;
-        if (!cgroup_lock_live_group(cgroup))
+        freezer_change_state(cgroup_freezer(cgroup), freeze);
-                return -ENODEV;
+        return 0;
-        retval = freezer_change_state(cgroup, goal_state);
+}
-        cgroup_unlock();
-        return retval;
+static u64 freezer_self_freezing_read(struct cgroup *cgroup, struct cftype *cft)
+{
+        struct freezer *freezer = cgroup_freezer(cgroup);
+        return (bool)(freezer->state & CGROUP_FREEZING_SELF);
+}
+static u64 freezer_parent_freezing_read(struct cgroup *cgroup, struct cftype *cft)
+{
+        struct freezer *freezer = cgroup_freezer(cgroup);
+        return (bool)(freezer->state & CGROUP_FREEZING_PARENT);
 }
 static struct cftype files[] = {
@@ -362,23 +462,27 @@ static struct cftype files[] = {
                .read_seq_string = freezer_read,
                .write_string = freezer_write,
        },
+        {
+                .name = "self_freezing",
+                .flags = CFTYPE_NOT_ON_ROOT,
+                .read_u64 = freezer_self_freezing_read,
+        },
+        {
+                .name = "parent_freezing",
+                .flags = CFTYPE_NOT_ON_ROOT,
+                .read_u64 = freezer_parent_freezing_read,
+        },
        { }     /* terminate */
 };
 struct cgroup_subsys freezer_subsys = {
        .name           = "freezer",
-        .create         = freezer_create,
+        .css_alloc      = freezer_css_alloc,
-        .destroy        = freezer_destroy,
+        .css_online     = freezer_css_online,
+        .css_offline    = freezer_css_offline,
+        .css_free       = freezer_css_free,
        .subsys_id      = freezer_subsys_id,
-        .can_attach     = freezer_can_attach,
+        .attach         = freezer_attach,
        .fork           = freezer_fork,
        .base_cftypes   = files,
-        /*
-         * freezer subsys doesn't handle hierarchy at all.  Frozen state
-         * should be inherited through the hierarchy - if a parent is
-         * frozen, all its children should be frozen.  Fix it and remove
-         * the following.
-         */
-        .broken_hierarchy = true,
 };
diff --git a/kernel/compat.c b/kernel/compat.c
index c28a306ae05c..f6150e92dfc9 100644
--- a/kernel/compat.c
+++ b/kernel/compat.c
@@ -1215,6 +1215,23 @@ compat_sys_sysinfo(struct compat_sysinfo __user *info)
        return 0;
 }
+#ifdef __ARCH_WANT_COMPAT_SYS_SCHED_RR_GET_INTERVAL
+asmlinkage long compat_sys_sched_rr_get_interval(compat_pid_t pid,
+                                                 struct compat_timespec __user *interval)
+{
+        struct timespec t;
+        int ret;
+        mm_segment_t old_fs = get_fs();
+        set_fs(KERNEL_DS);
+        ret = sys_sched_rr_get_interval(pid, (struct timespec __user *)&t);
+        set_fs(old_fs);
+        if (put_compat_timespec(&t, interval))
+                return -EFAULT;
+        return ret;
+}
+#endif /* __ARCH_WANT_COMPAT_SYS_SCHED_RR_GET_INTERVAL */
 /*
 * Allocate user-space memory for the duration of a single system call,
 * in order to marshall parameters inside a compat thunk.
diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c
new file mode 100644
index 000000000000..e0e07fd55508
--- /dev/null
+++ b/kernel/context_tracking.c
@@ -0,0 +1,83 @@
+#include <linux/context_tracking.h>
+#include <linux/rcupdate.h>
+#include <linux/sched.h>
+#include <linux/percpu.h>
+#include <linux/hardirq.h>
+struct context_tracking {
+        /*
+         * When active is false, hooks are not set to
+         * minimize overhead: TIF flags are cleared
+         * and calls to user_enter/exit are ignored. This
+         * may be further optimized using static keys.
+         */
+        bool active;
+        enum {
+                IN_KERNEL = 0,
+                IN_USER,
+        } state;
+};
+static DEFINE_PER_CPU(struct context_tracking, context_tracking) = {
+#ifdef CONFIG_CONTEXT_TRACKING_FORCE
+        .active = true,
+#endif
+};
+void user_enter(void)
+{
+        unsigned long flags;
+        /*
+         * Some contexts may involve an exception occuring in an irq,
+         * leading to that nesting:
+         * rcu_irq_enter() rcu_user_exit() rcu_user_exit() rcu_irq_exit()
+         * This would mess up the dyntick_nesting count though. And rcu_irq_*()
+         * helpers are enough to protect RCU uses inside the exception. So
+         * just return immediately if we detect we are in an IRQ.
+         */
+        if (in_interrupt())
+                return;
+        WARN_ON_ONCE(!current->mm);
+        local_irq_save(flags);
+        if (__this_cpu_read(context_tracking.active) &&
+            __this_cpu_read(context_tracking.state) != IN_USER) {
+                __this_cpu_write(context_tracking.state, IN_USER);
+                rcu_user_enter();
+        }
+        local_irq_restore(flags);
+}
+void user_exit(void)
+{
+        unsigned long flags;
+        /*
+         * Some contexts may involve an exception occuring in an irq,
+         * leading to that nesting:
+         * rcu_irq_enter() rcu_user_exit() rcu_user_exit() rcu_irq_exit()
+         * This would mess up the dyntick_nesting count though. And rcu_irq_*()
+         * helpers are enough to protect RCU uses inside the exception. So
+         * just return immediately if we detect we are in an IRQ.
+         */
+        if (in_interrupt())
+                return;
+        local_irq_save(flags);
+        if (__this_cpu_read(context_tracking.state) == IN_USER) {
+                __this_cpu_write(context_tracking.state, IN_KERNEL);
+                rcu_user_exit();
+        }
+        local_irq_restore(flags);
+}
+void context_tracking_task_switch(struct task_struct *prev,
+                             struct task_struct *next)
+{
+        if (__this_cpu_read(context_tracking.active)) {
+                clear_tsk_thread_flag(prev, TIF_NOHZ);
+                set_tsk_thread_flag(next, TIF_NOHZ);
+        }
+}
diff --git a/kernel/cpu.c b/kernel/cpu.c
index 42bd331ee0ab..3046a503242c 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -348,11 +348,13 @@ static int __cpuinit _cpu_up(unsigned int cpu, int tasks_frozen)
        unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0;
        struct task_struct *idle;
-        if (cpu_online(cpu) || !cpu_present(cpu))
-                return -EINVAL;
        cpu_hotplug_begin();
+        if (cpu_online(cpu) || !cpu_present(cpu)) {
+                ret = -EINVAL;
+                goto out;
+        }
        idle = idle_thread_get(cpu);
        if (IS_ERR(idle)) {
                ret = PTR_ERR(idle);
@@ -601,6 +603,11 @@ cpu_hotplug_pm_callback(struct notifier_block *nb,
 static int __init cpu_hotplug_pm_sync_init(void)
 {
+        /*
+         * cpu_hotplug_pm_callback has higher priority than x86
+         * bsp_pm_callback which depends on cpu_hotplug_pm_callback
+         * to disable cpu hotplug to avoid cpu hotplug race.
+         */
        pm_notifier(cpu_hotplug_pm_callback, 0);
        return 0;
 }
diff --git a/kernel/cpuset.c b/kernel/cpuset.c
index f33c7153b6d7..7bb63eea6eb8 100644
--- a/kernel/cpuset.c
+++ b/kernel/cpuset.c
@@ -302,10 +302,10 @@ static void guarantee_online_cpus(const struct cpuset *cs,
 * are online, with memory.  If none are online with memory, walk
 * up the cpuset hierarchy until we find one that does have some
 * online mems.  If we get all the way to the top and still haven't
- * found any online mems, return node_states[N_HIGH_MEMORY].
+ * found any online mems, return node_states[N_MEMORY].
 *
 * One way or another, we guarantee to return some non-empty subset
- * of node_states[N_HIGH_MEMORY].
+ * of node_states[N_MEMORY].
 *
 * Call with callback_mutex held.
 */
@@ -313,14 +313,14 @@ static void guarantee_online_cpus(const struct cpuset *cs,
 static void guarantee_online_mems(const struct cpuset *cs, nodemask_t *pmask)
 {
        while (cs && !nodes_intersects(cs->mems_allowed,
-                                        node_states[N_HIGH_MEMORY]))
+                                        node_states[N_MEMORY]))
                cs = cs->parent;
        if (cs)
                nodes_and(*pmask, cs->mems_allowed,
-                                        node_states[N_HIGH_MEMORY]);
+                                        node_states[N_MEMORY]);
        else
-                *pmask = node_states[N_HIGH_MEMORY];
+                *pmask = node_states[N_MEMORY];
-        BUG_ON(!nodes_intersects(*pmask, node_states[N_HIGH_MEMORY]));
+        BUG_ON(!nodes_intersects(*pmask, node_states[N_MEMORY]));
 }
 /*
@@ -1100,7 +1100,7 @@ static int update_nodemask(struct cpuset *cs, struct cpuset *trialcs,
                return -ENOMEM;
        /*
-         * top_cpuset.mems_allowed tracks node_stats[N_HIGH_MEMORY];
+         * top_cpuset.mems_allowed tracks node_stats[N_MEMORY];
         * it's read-only
         */
        if (cs == &top_cpuset) {
@@ -1122,7 +1122,7 @@ static int update_nodemask(struct cpuset *cs, struct cpuset *trialcs,
                        goto done;
                if (!nodes_subset(trialcs->mems_allowed,
-                                node_states[N_HIGH_MEMORY])) {
+                                node_states[N_MEMORY])) {
                        retval =  -EINVAL;
                        goto done;
                }
@@ -1784,56 +1784,20 @@ static struct cftype files[] = {
 };
 /*
- * post_clone() is called during cgroup_create() when the
+ *      cpuset_css_alloc - allocate a cpuset css
- * clone_children mount argument was specified.  The cgroup
- * can not yet have any tasks.
- *
- * Currently we refuse to set up the cgroup - thereby
- * refusing the task to be entered, and as a result refusing
- * the sys_unshare() or clone() which initiated it - if any
- * sibling cpusets have exclusive cpus or mem.
- *
- * If this becomes a problem for some users who wish to
- * allow that scenario, then cpuset_post_clone() could be
- * changed to grant parent->cpus_allowed-sibling_cpus_exclusive
- * (and likewise for mems) to the new cgroup. Called with cgroup_mutex
- * held.
- */
-static void cpuset_post_clone(struct cgroup *cgroup)
-{
-        struct cgroup *parent, *child;
-        struct cpuset *cs, *parent_cs;
-        parent = cgroup->parent;
-        list_for_each_entry(child, &parent->children, sibling) {
-                cs = cgroup_cs(child);
-                if (is_mem_exclusive(cs) || is_cpu_exclusive(cs))
-                        return;
-        }
-        cs = cgroup_cs(cgroup);
-        parent_cs = cgroup_cs(parent);
-        mutex_lock(&callback_mutex);
-        cs->mems_allowed = parent_cs->mems_allowed;
-        cpumask_copy(cs->cpus_allowed, parent_cs->cpus_allowed);
-        mutex_unlock(&callback_mutex);
-        return;
-}
-/*
- *      cpuset_create - create a cpuset
 *      cont:   control group that the new cpuset will be part of
 */
-static struct cgroup_subsys_state *cpuset_create(struct cgroup *cont)
+static struct cgroup_subsys_state *cpuset_css_alloc(struct cgroup *cont)
 {
-        struct cpuset *cs;
+        struct cgroup *parent_cg = cont->parent;
-        struct cpuset *parent;
+        struct cgroup *tmp_cg;
+        struct cpuset *parent, *cs;
-        if (!cont->parent) {
+        if (!parent_cg)
                return &top_cpuset.css;
-        }
+        parent = cgroup_cs(parent_cg);
-        parent = cgroup_cs(cont->parent);
        cs = kmalloc(sizeof(*cs), GFP_KERNEL);
        if (!cs)
                return ERR_PTR(-ENOMEM);
@@ -1855,7 +1819,36 @@ static struct cgroup_subsys_state *cpuset_create(struct cgroup *cont)
        cs->parent = parent;
        number_of_cpusets++;
-        return &cs->css ;
+        if (!test_bit(CGRP_CPUSET_CLONE_CHILDREN, &cont->flags))
+                goto skip_clone;
+        /*
+         * Clone @parent's configuration if CGRP_CPUSET_CLONE_CHILDREN is
+         * set.  This flag handling is implemented in cgroup core for
+         * histrical reasons - the flag may be specified during mount.
+         *
+         * Currently, if any sibling cpusets have exclusive cpus or mem, we
+         * refuse to clone the configuration - thereby refusing the task to
+         * be entered, and as a result refusing the sys_unshare() or
+         * clone() which initiated it.  If this becomes a problem for some
+         * users who wish to allow that scenario, then this could be
+         * changed to grant parent->cpus_allowed-sibling_cpus_exclusive
+         * (and likewise for mems) to the new cgroup.
+         */
+        list_for_each_entry(tmp_cg, &parent_cg->children, sibling) {
+                struct cpuset *tmp_cs = cgroup_cs(tmp_cg);
+                if (is_mem_exclusive(tmp_cs) || is_cpu_exclusive(tmp_cs))
+                        goto skip_clone;
+        }
+        mutex_lock(&callback_mutex);
+        cs->mems_allowed = parent->mems_allowed;
+        cpumask_copy(cs->cpus_allowed, parent->cpus_allowed);
+        mutex_unlock(&callback_mutex);
+skip_clone:
+        return &cs->css;
 }
 /*
@@ -1864,7 +1857,7 @@ static struct cgroup_subsys_state *cpuset_create(struct cgroup *cont)
 * will call async_rebuild_sched_domains().
 */
-static void cpuset_destroy(struct cgroup *cont)
+static void cpuset_css_free(struct cgroup *cont)
 {
        struct cpuset *cs = cgroup_cs(cont);
@@ -1878,11 +1871,10 @@ static void cpuset_destroy(struct cgroup *cont)
 struct cgroup_subsys cpuset_subsys = {
        .name = "cpuset",
-        .create = cpuset_create,
+        .css_alloc = cpuset_css_alloc,
-        .destroy = cpuset_destroy,
+        .css_free = cpuset_css_free,
        .can_attach = cpuset_can_attach,
        .attach = cpuset_attach,
-        .post_clone = cpuset_post_clone,
        .subsys_id = cpuset_subsys_id,
        .base_cftypes = files,
        .early_init = 1,
@@ -2034,7 +2026,7 @@ static struct cpuset *cpuset_next(struct list_head *queue)
 * before dropping down to the next.  It always processes a node before
 * any of its children.
 *
- * In the case of memory hot-unplug, it will remove nodes from N_HIGH_MEMORY
+ * In the case of memory hot-unplug, it will remove nodes from N_MEMORY
 * if all present pages from a node are offlined.
 */
 static void
@@ -2073,7 +2065,7 @@ scan_cpusets_upon_hotplug(struct cpuset *root, enum hotplug_event event)
                        /* Continue past cpusets with all mems online */
                        if (nodes_subset(cp->mems_allowed,
-                                        node_states[N_HIGH_MEMORY]))
+                                        node_states[N_MEMORY]))
                                continue;
                        oldmems = cp->mems_allowed;
@@ -2081,7 +2073,7 @@ scan_cpusets_upon_hotplug(struct cpuset *root, enum hotplug_event event)
                        /* Remove offline mems from this cpuset. */
                        mutex_lock(&callback_mutex);
                        nodes_and(cp->mems_allowed, cp->mems_allowed,
-                                                node_states[N_HIGH_MEMORY]);
+                                                node_states[N_MEMORY]);
                        mutex_unlock(&callback_mutex);
                        /* Move tasks from the empty cpuset to a parent */
@@ -2134,8 +2126,8 @@ void cpuset_update_active_cpus(bool cpu_online)
 #ifdef CONFIG_MEMORY_HOTPLUG
 /*
- * Keep top_cpuset.mems_allowed tracking node_states[N_HIGH_MEMORY].
+ * Keep top_cpuset.mems_allowed tracking node_states[N_MEMORY].
- * Call this routine anytime after node_states[N_HIGH_MEMORY] changes.
+ * Call this routine anytime after node_states[N_MEMORY] changes.
 * See cpuset_update_active_cpus() for CPU hotplug handling.
 */
 static int cpuset_track_online_nodes(struct notifier_block *self,
@@ -2148,7 +2140,7 @@ static int cpuset_track_online_nodes(struct notifier_block *self,
        case MEM_ONLINE:
                oldmems = top_cpuset.mems_allowed;
                mutex_lock(&callback_mutex);
-                top_cpuset.mems_allowed = node_states[N_HIGH_MEMORY];
+                top_cpuset.mems_allowed = node_states[N_MEMORY];
                mutex_unlock(&callback_mutex);
                update_tasks_nodemask(&top_cpuset, &oldmems, NULL);
                break;
@@ -2177,7 +2169,7 @@ static int cpuset_track_online_nodes(struct notifier_block *self,
 void __init cpuset_init_smp(void)
 {
        cpumask_copy(top_cpuset.cpus_allowed, cpu_active_mask);
-        top_cpuset.mems_allowed = node_states[N_HIGH_MEMORY];
+        top_cpuset.mems_allowed = node_states[N_MEMORY];
        hotplug_memory_notifier(cpuset_track_online_nodes, 10);
@@ -2245,7 +2237,7 @@ void cpuset_init_current_mems_allowed(void)
 *
 * Description: Returns the nodemask_t mems_allowed of the cpuset
 * attached to the specified @tsk.  Guaranteed to return some non-empty
- * subset of node_states[N_HIGH_MEMORY], even if this means going outside the
+ * subset of node_states[N_MEMORY], even if this means going outside the
 * tasks cpuset.
 **/
diff --git a/kernel/cred.c b/kernel/cred.c
index 709d521903f6..e0573a43c7df 100644
--- a/kernel/cred.c
+++ b/kernel/cred.c
@@ -30,17 +30,6 @@
 static struct kmem_cache *cred_jar;
 /*
- * The common credentials for the initial task's thread group
- */
-#ifdef CONFIG_KEYS
-static struct thread_group_cred init_tgcred = {
-        .usage  = ATOMIC_INIT(2),
-        .tgid   = 0,
-        .lock   = __SPIN_LOCK_UNLOCKED(init_cred.tgcred.lock),
-};
-#endif
-/*
 * The initial credentials for the initial task
 */
 struct cred init_cred = {
@@ -65,9 +54,6 @@ struct cred init_cred = {
        .user                   = INIT_USER,
        .user_ns                = &init_user_ns,
        .group_info             = &init_groups,
-#ifdef CONFIG_KEYS
-        .tgcred                 = &init_tgcred,
-#endif
 };
 static inline void set_cred_subscribers(struct cred *cred, int n)
@@ -96,36 +82,6 @@ static inline void alter_cred_subscribers(const struct cred *_cred, int n)
 }
 /*
- * Dispose of the shared task group credentials
- */
-#ifdef CONFIG_KEYS
-static void release_tgcred_rcu(struct rcu_head *rcu)
-{
-        struct thread_group_cred *tgcred =
-                container_of(rcu, struct thread_group_cred, rcu);
-        BUG_ON(atomic_read(&tgcred->usage) != 0);
-        key_put(tgcred->session_keyring);
-        key_put(tgcred->process_keyring);
-        kfree(tgcred);
-}
-#endif
-/*
- * Release a set of thread group credentials.
- */
-static void release_tgcred(struct cred *cred)
-{
-#ifdef CONFIG_KEYS
-        struct thread_group_cred *tgcred = cred->tgcred;
-        if (atomic_dec_and_test(&tgcred->usage))
-                call_rcu(&tgcred->rcu, release_tgcred_rcu);
-#endif
-}
-/*
 * The RCU callback to actually dispose of a set of credentials
 */
 static void put_cred_rcu(struct rcu_head *rcu)
@@ -150,9 +106,10 @@ static void put_cred_rcu(struct rcu_head *rcu)
 #endif
        security_cred_free(cred);
+        key_put(cred->session_keyring);
+        key_put(cred->process_keyring);
        key_put(cred->thread_keyring);
        key_put(cred->request_key_auth);
-        release_tgcred(cred);
        if (cred->group_info)
                put_group_info(cred->group_info);
        free_uid(cred->user);
@@ -246,15 +203,6 @@ struct cred *cred_alloc_blank(void)
        if (!new)
                return NULL;
-#ifdef CONFIG_KEYS
-        new->tgcred = kzalloc(sizeof(*new->tgcred), GFP_KERNEL);
-        if (!new->tgcred) {
-                kmem_cache_free(cred_jar, new);
-                return NULL;
-        }
-        atomic_set(&new->tgcred->usage, 1);
-#endif
        atomic_set(&new->usage, 1);
 #ifdef CONFIG_DEBUG_CREDENTIALS
        new->magic = CRED_MAGIC;
@@ -308,9 +256,10 @@ struct cred *prepare_creds(void)
        get_user_ns(new->user_ns);
 #ifdef CONFIG_KEYS
+        key_get(new->session_keyring);
+        key_get(new->process_keyring);
        key_get(new->thread_keyring);
        key_get(new->request_key_auth);
-        atomic_inc(&new->tgcred->usage);
 #endif
 #ifdef CONFIG_SECURITY
@@ -334,39 +283,20 @@ EXPORT_SYMBOL(prepare_creds);
 */
 struct cred *prepare_exec_creds(void)
 {
-        struct thread_group_cred *tgcred = NULL;
        struct cred *new;
-#ifdef CONFIG_KEYS
-        tgcred = kmalloc(sizeof(*tgcred), GFP_KERNEL);
-        if (!tgcred)
-                return NULL;
-#endif
        new = prepare_creds();
-        if (!new) {
+        if (!new)
-                kfree(tgcred);
                return new;
-        }
 #ifdef CONFIG_KEYS
        /* newly exec'd tasks don't get a thread keyring */
        key_put(new->thread_keyring);
        new->thread_keyring = NULL;
-        /* create a new per-thread-group creds for all this set of threads to
-         * share */
-        memcpy(tgcred, new->tgcred, sizeof(struct thread_group_cred));
-        atomic_set(&tgcred->usage, 1);
-        spin_lock_init(&tgcred->lock);
        /* inherit the session keyring; new process keyring */
-        key_get(tgcred->session_keyring);
+        key_put(new->process_keyring);
-        tgcred->process_keyring = NULL;
+        new->process_keyring = NULL;
-        release_tgcred(new);
-        new->tgcred = tgcred;
 #endif
        return new;
@@ -383,9 +313,6 @@ struct cred *prepare_exec_creds(void)
 */
 int copy_creds(struct task_struct *p, unsigned long clone_flags)
 {
-#ifdef CONFIG_KEYS
-        struct thread_group_cred *tgcred;
-#endif
        struct cred *new;
        int ret;
@@ -425,22 +352,12 @@ int copy_creds(struct task_struct *p, unsigned long clone_flags)
                        install_thread_keyring_to_cred(new);
        }
-        /* we share the process and session keyrings between all the threads in
+        /* The process keyring is only shared between the threads in a process;
-         * a process - this is slightly icky as we violate COW credentials a
+         * anything outside of those threads doesn't inherit.
-         * bit */
+         */
        if (!(clone_flags & CLONE_THREAD)) {
-                tgcred = kmalloc(sizeof(*tgcred), GFP_KERNEL);
+                key_put(new->process_keyring);
-                if (!tgcred) {
+                new->process_keyring = NULL;
-                        ret = -ENOMEM;
-                        goto error_put;
-                }
-                atomic_set(&tgcred->usage, 1);
-                spin_lock_init(&tgcred->lock);
-                tgcred->process_keyring = NULL;
-                tgcred->session_keyring = key_get(new->tgcred->session_keyring);
-                release_tgcred(new);
-                new->tgcred = tgcred;
        }
 #endif
@@ -668,9 +585,6 @@ void __init cred_init(void)
 */
 struct cred *prepare_kernel_cred(struct task_struct *daemon)
 {
-#ifdef CONFIG_KEYS
-        struct thread_group_cred *tgcred;
-#endif
        const struct cred *old;
        struct cred *new;
@@ -678,14 +592,6 @@ struct cred *prepare_kernel_cred(struct task_struct *daemon)
        if (!new)
                return NULL;
-#ifdef CONFIG_KEYS
-        tgcred = kmalloc(sizeof(*tgcred), GFP_KERNEL);
-        if (!tgcred) {
-                kmem_cache_free(cred_jar, new);
-                return NULL;
-        }
-#endif
        kdebug("prepare_kernel_cred() alloc %p", new);
        if (daemon)
@@ -703,13 +609,10 @@ struct cred *prepare_kernel_cred(struct task_struct *daemon)
        get_group_info(new->group_info);
 #ifdef CONFIG_KEYS
-        atomic_set(&tgcred->usage, 1);
+        new->session_keyring = NULL;
-        spin_lock_init(&tgcred->lock);
+        new->process_keyring = NULL;
-        tgcred->process_keyring = NULL;
-        tgcred->session_keyring = NULL;
-        new->tgcred = tgcred;
-        new->request_key_auth = NULL;
        new->thread_keyring = NULL;
+        new->request_key_auth = NULL;
        new->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING;
 #endif
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 738f3564e83b..301079d06f24 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -7434,7 +7434,7 @@ unlock:
 device_initcall(perf_event_sysfs_init);
 #ifdef CONFIG_CGROUP_PERF
-static struct cgroup_subsys_state *perf_cgroup_create(struct cgroup *cont)
+static struct cgroup_subsys_state *perf_cgroup_css_alloc(struct cgroup *cont)
 {
        struct perf_cgroup *jc;
@@ -7451,7 +7451,7 @@ static struct cgroup_subsys_state *perf_cgroup_create(struct cgroup *cont)
        return &jc->css;
 }
-static void perf_cgroup_destroy(struct cgroup *cont)
+static void perf_cgroup_css_free(struct cgroup *cont)
 {
        struct perf_cgroup *jc;
        jc = container_of(cgroup_subsys_state(cont, perf_subsys_id),
@@ -7492,8 +7492,8 @@ static void perf_cgroup_exit(struct cgroup *cgrp, struct cgroup *old_cgrp,
 struct cgroup_subsys perf_subsys = {
        .name           = "perf_event",
        .subsys_id      = perf_subsys_id,
-        .create         = perf_cgroup_create,
+        .css_alloc      = perf_cgroup_css_alloc,
-        .destroy        = perf_cgroup_destroy,
+        .css_free       = perf_cgroup_css_free,
        .exit           = perf_cgroup_exit,
        .attach         = perf_cgroup_attach,
diff --git a/kernel/events/hw_breakpoint.c b/kernel/events/hw_breakpoint.c
index 9a7b487c6fe2..fe8a916507ed 100644
--- a/kernel/events/hw_breakpoint.c
+++ b/kernel/events/hw_breakpoint.c
@@ -111,14 +111,16 @@ static unsigned int max_task_bp_pinned(int cpu, enum bp_type_idx type)
 * Count the number of breakpoints of the same type and same task.
 * The given event must be not on the list.
 */
-static int task_bp_pinned(struct perf_event *bp, enum bp_type_idx type)
+static int task_bp_pinned(int cpu, struct perf_event *bp, enum bp_type_idx type)
 {
        struct task_struct *tsk = bp->hw.bp_target;
        struct perf_event *iter;
        int count = 0;
        list_for_each_entry(iter, &bp_task_head, hw.bp_list) {
-                if (iter->hw.bp_target == tsk && find_slot_idx(iter) == type)
+                if (iter->hw.bp_target == tsk &&
+                    find_slot_idx(iter) == type &&
+                    cpu == iter->cpu)
                        count += hw_breakpoint_weight(iter);
        }
@@ -141,7 +143,7 @@ fetch_bp_busy_slots(struct bp_busy_slots *slots, struct perf_event *bp,
                if (!tsk)
                        slots->pinned += max_task_bp_pinned(cpu, type);
                else
-                        slots->pinned += task_bp_pinned(bp, type);
+                        slots->pinned += task_bp_pinned(cpu, bp, type);
                slots->flexible = per_cpu(nr_bp_flexible[type], cpu);
                return;
@@ -154,7 +156,7 @@ fetch_bp_busy_slots(struct bp_busy_slots *slots, struct perf_event *bp,
                if (!tsk)
                        nr += max_task_bp_pinned(cpu, type);
                else
-                        nr += task_bp_pinned(bp, type);
+                        nr += task_bp_pinned(cpu, bp, type);
                if (nr > slots->pinned)
                        slots->pinned = nr;
@@ -188,7 +190,7 @@ static void toggle_bp_task_slot(struct perf_event *bp, int cpu, bool enable,
        int old_idx = 0;
        int idx = 0;
-        old_count = task_bp_pinned(bp, type);
+        old_count = task_bp_pinned(cpu, bp, type);
        old_idx = old_count - 1;
        idx = old_idx + weight;
diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index 5cc4e7e42e68..dea7acfbb071 100644
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -33,6 +33,7 @@
 #include <linux/ptrace.h>       /* user_enable_single_step */
 #include <linux/kdebug.h>       /* notifier mechanism */
 #include "../../mm/internal.h"  /* munlock_vma_page */
+#include <linux/percpu-rwsem.h>
 #include <linux/uprobes.h>
@@ -71,6 +72,8 @@ static struct mutex uprobes_mutex[UPROBES_HASH_SZ];
 static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ];
 #define uprobes_mmap_hash(v)    (&uprobes_mmap_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ])
+static struct percpu_rw_semaphore dup_mmap_sem;
 /*
 * uprobe_events allows us to skip the uprobe_mmap if there are no uprobe
 * events active at this time.  Probably a fine grained per inode count is
@@ -766,10 +769,13 @@ static int register_for_each_vma(struct uprobe *uprobe, bool is_register)
        struct map_info *info;
        int err = 0;
+        percpu_down_write(&dup_mmap_sem);
        info = build_map_info(uprobe->inode->i_mapping,
                                        uprobe->offset, is_register);
-        if (IS_ERR(info))
+        if (IS_ERR(info)) {
-                return PTR_ERR(info);
+                err = PTR_ERR(info);
+                goto out;
+        }
        while (info) {
                struct mm_struct *mm = info->mm;
@@ -799,7 +805,8 @@ static int register_for_each_vma(struct uprobe *uprobe, bool is_register)
                mmput(mm);
                info = free_map_info(info);
        }
+ out:
+        percpu_up_write(&dup_mmap_sem);
        return err;
 }
@@ -1131,6 +1138,16 @@ void uprobe_clear_state(struct mm_struct *mm)
        kfree(area);
 }
+void uprobe_start_dup_mmap(void)
+{
+        percpu_down_read(&dup_mmap_sem);
+}
+void uprobe_end_dup_mmap(void)
+{
+        percpu_up_read(&dup_mmap_sem);
+}
 void uprobe_dup_mmap(struct mm_struct *oldmm, struct mm_struct *newmm)
 {
        newmm->uprobes_state.xol_area = NULL;
@@ -1199,6 +1216,11 @@ static unsigned long xol_get_insn_slot(struct uprobe *uprobe, unsigned long slot
        vaddr = kmap_atomic(area->page);
        memcpy(vaddr + offset, uprobe->arch.insn, MAX_UINSN_BYTES);
        kunmap_atomic(vaddr);
+        /*
+         * We probably need flush_icache_user_range() but it needs vma.
+         * This should work on supported architectures too.
+         */
+        flush_dcache_page(area->page);
        return current->utask->xol_vaddr;
 }
@@ -1430,16 +1452,6 @@ static struct uprobe *find_active_uprobe(unsigned long bp_vaddr, int *is_swbp)
        return uprobe;
 }
-void __weak arch_uprobe_enable_step(struct arch_uprobe *arch)
-{
-        user_enable_single_step(current);
-}
-void __weak arch_uprobe_disable_step(struct arch_uprobe *arch)
-{
-        user_disable_single_step(current);
-}
 /*
 * Run handler and ask thread to singlestep.
 * Ensure all non-fatal signals cannot interrupt thread while it singlesteps.
@@ -1493,7 +1505,6 @@ static void handle_swbp(struct pt_regs *regs)
                goto out;
        if (!pre_ssout(uprobe, regs, bp_vaddr)) {
-                arch_uprobe_enable_step(&uprobe->arch);
                utask->active_uprobe = uprobe;
                utask->state = UTASK_SSTEP;
                return;
@@ -1525,7 +1536,6 @@ static void handle_singlestep(struct uprobe_task *utask, struct pt_regs *regs)
        else
                WARN_ON_ONCE(1);
-        arch_uprobe_disable_step(&uprobe->arch);
        put_uprobe(uprobe);
        utask->active_uprobe = NULL;
        utask->state = UTASK_RUNNING;
@@ -1604,6 +1614,9 @@ static int __init init_uprobes(void)
                mutex_init(&uprobes_mmap_mutex[i]);
        }
+        if (percpu_init_rwsem(&dup_mmap_sem))
+                return -ENOMEM;
        return register_die_notifier(&uprobe_exception_nb);
 }
 module_init(init_uprobes);
diff --git a/kernel/exit.c b/kernel/exit.c
index d7fe58db4527..b4df21937216 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -310,43 +310,6 @@ kill_orphaned_pgrp(struct task_struct *tsk, struct task_struct *parent)
        }
 }
-/**
- * reparent_to_kthreadd - Reparent the calling kernel thread to kthreadd
- *
- * If a kernel thread is launched as a result of a system call, or if
- * it ever exits, it should generally reparent itself to kthreadd so it
- * isn't in the way of other processes and is correctly cleaned up on exit.
- *
- * The various task state such as scheduling policy and priority may have
- * been inherited from a user process, so we reset them to sane values here.
- *
- * NOTE that reparent_to_kthreadd() gives the caller full capabilities.
- */
-static void reparent_to_kthreadd(void)
-{
-        write_lock_irq(&tasklist_lock);
-        ptrace_unlink(current);
-        /* Reparent to init */
-        current->real_parent = current->parent = kthreadd_task;
-        list_move_tail(&current->sibling, &current->real_parent->children);
-        /* Set the exit signal to SIGCHLD so we signal init on exit */
-        current->exit_signal = SIGCHLD;
-        if (task_nice(current) < 0)
-                set_user_nice(current, 0);
-        /* cpus_allowed? */
-        /* rt_priority? */
-        /* signals? */
-        memcpy(current->signal->rlim, init_task.signal->rlim,
-               sizeof(current->signal->rlim));
-        atomic_inc(&init_cred.usage);
-        commit_creds(&init_cred);
-        write_unlock_irq(&tasklist_lock);
-}
 void __set_special_pids(struct pid *pid)
 {
        struct task_struct *curr = current->group_leader;
@@ -358,13 +321,6 @@ void __set_special_pids(struct pid *pid)
                change_pid(curr, PIDTYPE_PGID, pid);
 }
-static void set_special_pids(struct pid *pid)
-{
-        write_lock_irq(&tasklist_lock);
-        __set_special_pids(pid);
-        write_unlock_irq(&tasklist_lock);
-}
 /*
 * Let kernel threads use this to say that they allow a certain signal.
 * Must not be used if kthread was cloned with CLONE_SIGHAND.
@@ -404,54 +360,6 @@ int disallow_signal(int sig)
 EXPORT_SYMBOL(disallow_signal);
-/*
- *      Put all the gunge required to become a kernel thread without
- *      attached user resources in one place where it belongs.
- */
-void daemonize(const char *name, ...)
-{
-        va_list args;
-        sigset_t blocked;
-        va_start(args, name);
-        vsnprintf(current->comm, sizeof(current->comm), name, args);
-        va_end(args);
-        /*
-         * If we were started as result of loading a module, close all of the
-         * user space pages.  We don't need them, and if we didn't close them
-         * they would be locked into memory.
-         */
-        exit_mm(current);
-        /*
-         * We don't want to get frozen, in case system-wide hibernation
-         * or suspend transition begins right now.
-         */
-        current->flags |= (PF_NOFREEZE | PF_KTHREAD);
-        if (current->nsproxy != &init_nsproxy) {
-                get_nsproxy(&init_nsproxy);
-                switch_task_namespaces(current, &init_nsproxy);
-        }
-        set_special_pids(&init_struct_pid);
-        proc_clear_tty(current);
-        /* Block and flush all signals */
-        sigfillset(&blocked);
-        sigprocmask(SIG_BLOCK, &blocked, NULL);
-        flush_signals(current);
-        /* Become as one with the init task */
-        daemonize_fs_struct();
-        daemonize_descriptors();
-        reparent_to_kthreadd();
-}
-EXPORT_SYMBOL(daemonize);
 #ifdef CONFIG_MM_OWNER
 /*
 * A task is exiting.   If it owned this mm, find a new owner for the mm.
@@ -1174,11 +1082,11 @@ static int wait_task_zombie(struct wait_opts *wo, struct task_struct *p)
                 * as other threads in the parent group can be right
                 * here reaping other children at the same time.
                 *
-                 * We use thread_group_times() to get times for the thread
+                 * We use thread_group_cputime_adjusted() to get times for the thread
                 * group, which consolidates times for all threads in the
                 * group including the group leader.
                 */
-                thread_group_times(p, &tgutime, &tgstime);
+                thread_group_cputime_adjusted(p, &tgutime, &tgstime);
                spin_lock_irq(&p->real_parent->sighand->siglock);
                psig = p->real_parent->signal;
                sig = p->signal;
diff --git a/kernel/fork.c b/kernel/fork.c
index 38e53b87402c..c36c4e301efe 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -352,6 +352,7 @@ static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
        unsigned long charge;
        struct mempolicy *pol;
+        uprobe_start_dup_mmap();
        down_write(&oldmm->mmap_sem);
        flush_cache_dup_mm(oldmm);
        uprobe_dup_mmap(oldmm, mm);
@@ -469,6 +470,7 @@ out:
        up_write(&mm->mmap_sem);
        flush_tlb_mm(oldmm);
        up_write(&oldmm->mmap_sem);
+        uprobe_end_dup_mmap();
        return retval;
 fail_nomem_anon_vma_fork:
        mpol_put(pol);
@@ -821,6 +823,9 @@ struct mm_struct *dup_mm(struct task_struct *tsk)
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
        mm->pmd_huge_pte = NULL;
 #endif
+#ifdef CONFIG_NUMA_BALANCING
+        mm->first_nid = NUMA_PTE_SCAN_INIT;
+#endif
        if (!mm_init(mm, tsk))
                goto fail_nomem;
@@ -1125,7 +1130,6 @@ static void posix_cpu_timers_init(struct task_struct *tsk)
 */
 static struct task_struct *copy_process(unsigned long clone_flags,
                                        unsigned long stack_start,
-                                        struct pt_regs *regs,
                                        unsigned long stack_size,
                                        int __user *child_tidptr,
                                        struct pid *pid,
@@ -1133,7 +1137,6 @@ static struct task_struct *copy_process(unsigned long clone_flags,
 {
        int retval;
        struct task_struct *p;
-        int cgroup_callbacks_done = 0;
        if ((clone_flags & (CLONE_NEWNS|CLONE_FS)) == (CLONE_NEWNS|CLONE_FS))
                return ERR_PTR(-EINVAL);
@@ -1220,7 +1223,7 @@ static struct task_struct *copy_process(unsigned long clone_flags,
        p->utime = p->stime = p->gtime = 0;
        p->utimescaled = p->stimescaled = 0;
 #ifndef CONFIG_VIRT_CPU_ACCOUNTING
-        p->prev_utime = p->prev_stime = 0;
+        p->prev_cputime.utime = p->prev_cputime.stime = 0;
 #endif
 #if defined(SPLIT_RSS_COUNTING)
        memset(&p->rss_stat, 0, sizeof(p->rss_stat));
@@ -1318,7 +1321,7 @@ static struct task_struct *copy_process(unsigned long clone_flags,
        retval = copy_io(clone_flags, p);
        if (retval)
                goto bad_fork_cleanup_namespaces;
-        retval = copy_thread(clone_flags, stack_start, stack_size, p, regs);
+        retval = copy_thread(clone_flags, stack_start, stack_size, p);
        if (retval)
                goto bad_fork_cleanup_io;
@@ -1391,12 +1394,6 @@ static struct task_struct *copy_process(unsigned long clone_flags,
        INIT_LIST_HEAD(&p->thread_group);
        p->task_works = NULL;
-        /* Now that the task is set up, run cgroup callbacks if
-         * necessary. We need to run them before the task is visible
-         * on the tasklist. */
-        cgroup_fork_callbacks(p);
-        cgroup_callbacks_done = 1;
        /* Need tasklist lock for parent etc handling! */
        write_lock_irq(&tasklist_lock);
@@ -1501,7 +1498,7 @@ bad_fork_cleanup_cgroup:
 #endif
        if (clone_flags & CLONE_THREAD)
                threadgroup_change_end(current);
-        cgroup_exit(p, cgroup_callbacks_done);
+        cgroup_exit(p, 0);
        delayacct_tsk_free(p);
        module_put(task_thread_info(p)->exec_domain->module);
 bad_fork_cleanup_count:
@@ -1513,12 +1510,6 @@ fork_out:
        return ERR_PTR(retval);
 }
-noinline struct pt_regs * __cpuinit __attribute__((weak)) idle_regs(struct pt_regs *regs)
-{
-        memset(regs, 0, sizeof(struct pt_regs));
-        return regs;
-}
 static inline void init_idle_pids(struct pid_link *links)
 {
        enum pid_type type;
@@ -1532,10 +1523,7 @@ static inline void init_idle_pids(struct pid_link *links)
 struct task_struct * __cpuinit fork_idle(int cpu)
 {
        struct task_struct *task;
-        struct pt_regs regs;
+        task = copy_process(CLONE_VM, 0, 0, NULL, &init_struct_pid, 0);
-        task = copy_process(CLONE_VM, 0, idle_regs(&regs), 0, NULL,
-                            &init_struct_pid, 0);
        if (!IS_ERR(task)) {
                init_idle_pids(task->pids);
                init_idle(task, cpu);
@@ -1552,7 +1540,6 @@ struct task_struct * __cpuinit fork_idle(int cpu)
 */
 long do_fork(unsigned long clone_flags,
              unsigned long stack_start,
-              struct pt_regs *regs,
              unsigned long stack_size,
              int __user *parent_tidptr,
              int __user *child_tidptr)
@@ -1576,7 +1563,7 @@ long do_fork(unsigned long clone_flags,
         * requested, no event is reported; otherwise, report if the event
         * for the type of forking is enabled.
         */
-        if (!(clone_flags & CLONE_UNTRACED) && likely(user_mode(regs))) {
+        if (!(clone_flags & CLONE_UNTRACED)) {
                if (clone_flags & CLONE_VFORK)
                        trace = PTRACE_EVENT_VFORK;
                else if ((clone_flags & CSIGNAL) != SIGCHLD)
@@ -1588,7 +1575,7 @@ long do_fork(unsigned long clone_flags,
                        trace = 0;
        }
-        p = copy_process(clone_flags, stack_start, regs, stack_size,
+        p = copy_process(clone_flags, stack_start, stack_size,
                         child_tidptr, NULL, trace);
        /*
         * Do this prior waking up the new thread - the thread pointer
@@ -1632,11 +1619,54 @@ long do_fork(unsigned long clone_flags,
 */
 pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags)
 {
-        return do_fork(flags|CLONE_VM|CLONE_UNTRACED, (unsigned long)fn, NULL,
+        return do_fork(flags|CLONE_VM|CLONE_UNTRACED, (unsigned long)fn,
                (unsigned long)arg, NULL, NULL);
 }
 #endif
+#ifdef __ARCH_WANT_SYS_FORK
+SYSCALL_DEFINE0(fork)
+{
+#ifdef CONFIG_MMU
+        return do_fork(SIGCHLD, 0, 0, NULL, NULL);
+#else
+        /* can not support in nommu mode */
+        return(-EINVAL);
+#endif
+}
+#endif
+#ifdef __ARCH_WANT_SYS_VFORK
+SYSCALL_DEFINE0(vfork)
+{
+        return do_fork(CLONE_VFORK | CLONE_VM | SIGCHLD, 0, 
+                        0, NULL, NULL);
+}
+#endif
+#ifdef __ARCH_WANT_SYS_CLONE
+#ifdef CONFIG_CLONE_BACKWARDS
+SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
+                 int __user *, parent_tidptr,
+                 int, tls_val,
+                 int __user *, child_tidptr)
+#elif defined(CONFIG_CLONE_BACKWARDS2)
+SYSCALL_DEFINE5(clone, unsigned long, newsp, unsigned long, clone_flags,
+                 int __user *, parent_tidptr,
+                 int __user *, child_tidptr,
+                 int, tls_val)
+#else
+SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
+                 int __user *, parent_tidptr,
+                 int __user *, child_tidptr,
+                 int, tls_val)
+#endif
+{
+        return do_fork(clone_flags, newsp, 0,
+                parent_tidptr, child_tidptr);
+}
+#endif
 #ifndef ARCH_MIN_MMSTRUCT_ALIGN
 #define ARCH_MIN_MMSTRUCT_ALIGN 0
 #endif
diff --git a/kernel/freezer.c b/kernel/freezer.c
index 11f82a4d4eae..c38893b0efba 100644
--- a/kernel/freezer.c
+++ b/kernel/freezer.c
@@ -116,17 +116,10 @@ bool freeze_task(struct task_struct *p)
                return false;
        }
-        if (!(p->flags & PF_KTHREAD)) {
+        if (!(p->flags & PF_KTHREAD))
                fake_signal_wake_up(p);
-                /*
+        else
-                 * fake_signal_wake_up() goes through p's scheduler
-                 * lock and guarantees that TASK_STOPPED/TRACED ->
-                 * TASK_RUNNING transition can't race with task state
-                 * testing in try_to_freeze_tasks().
-                 */
-        } else {
                wake_up_state(p, TASK_INTERRUPTIBLE);
-        }
        spin_unlock_irqrestore(&freezer_lock, flags);
        return true;
diff --git a/kernel/futex.c b/kernel/futex.c
index 3717e7b306e0..19eb089ca003 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -716,7 +716,7 @@ static int futex_lock_pi_atomic(u32 __user *uaddr, struct futex_hash_bucket *hb,
                                struct futex_pi_state **ps,
                                struct task_struct *task, int set_waiters)
 {
-        int lock_taken, ret, ownerdied = 0;
+        int lock_taken, ret, force_take = 0;
        u32 uval, newval, curval, vpid = task_pid_vnr(task);
 retry:
@@ -755,17 +755,15 @@ retry:
        newval = curval | FUTEX_WAITERS;
        /*
-         * There are two cases, where a futex might have no owner (the
+         * Should we force take the futex? See below.
-         * owner TID is 0): OWNER_DIED. We take over the futex in this
-         * case. We also do an unconditional take over, when the owner
-         * of the futex died.
-         *
-         * This is safe as we are protected by the hash bucket lock !
         */
-        if (unlikely(ownerdied || !(curval & FUTEX_TID_MASK))) {
+        if (unlikely(force_take)) {
-                /* Keep the OWNER_DIED bit */
+                /*
+                 * Keep the OWNER_DIED and the WAITERS bit and set the
+                 * new TID value.
+                 */
                newval = (curval & ~FUTEX_TID_MASK) | vpid;
-                ownerdied = 0;
+                force_take = 0;
                lock_taken = 1;
        }
@@ -775,7 +773,7 @@ retry:
                goto retry;
        /*
-         * We took the lock due to owner died take over.
+         * We took the lock due to forced take over.
         */
        if (unlikely(lock_taken))
                return 1;
@@ -790,20 +788,25 @@ retry:
                switch (ret) {
                case -ESRCH:
                        /*
-                         * No owner found for this futex. Check if the
+                         * We failed to find an owner for this
-                         * OWNER_DIED bit is set to figure out whether
+                         * futex. So we have no pi_state to block
-                         * this is a robust futex or not.
+                         * on. This can happen in two cases:
+                         *
+                         * 1) The owner died
+                         * 2) A stale FUTEX_WAITERS bit
+                         *
+                         * Re-read the futex value.
                         */
                        if (get_futex_value_locked(&curval, uaddr))
                                return -EFAULT;
                        /*
-                         * We simply start over in case of a robust
+                         * If the owner died or we have a stale
-                         * futex. The code above will take the futex
+                         * WAITERS bit the owner TID in the user space
-                         * and return happy.
+                         * futex is 0.
                         */
-                        if (curval & FUTEX_OWNER_DIED) {
+                        if (!(curval & FUTEX_TID_MASK)) {
-                                ownerdied = 1;
+                                force_take = 1;
                                goto retry;
                        }
                default:
@@ -840,6 +843,9 @@ static void wake_futex(struct futex_q *q)
 {
        struct task_struct *p = q->task;
+        if (WARN(q->pi_state || q->rt_waiter, "refusing to wake PI futex\n"))
+                return;
        /*
         * We set q->lock_ptr = NULL _before_ we wake up the task. If
         * a non-futex wake up happens on another CPU then the task
@@ -1075,6 +1081,10 @@ retry_private:
        plist_for_each_entry_safe(this, next, head, list) {
                if (match_futex (&this->key, &key1)) {
+                        if (this->pi_state || this->rt_waiter) {
+                                ret = -EINVAL;
+                                goto out_unlock;
+                        }
                        wake_futex(this);
                        if (++ret >= nr_wake)
                                break;
@@ -1087,6 +1097,10 @@ retry_private:
                op_ret = 0;
                plist_for_each_entry_safe(this, next, head, list) {
                        if (match_futex (&this->key, &key2)) {
+                                if (this->pi_state || this->rt_waiter) {
+                                        ret = -EINVAL;
+                                        goto out_unlock;
+                                }
                                wake_futex(this);
                                if (++op_ret >= nr_wake2)
                                        break;
@@ -1095,6 +1109,7 @@ retry_private:
                ret += op_ret;
        }
+out_unlock:
        double_unlock_hb(hb1, hb2);
 out_put_keys:
        put_futex_key(&key2);
@@ -1384,9 +1399,13 @@ retry_private:
                /*
                 * FUTEX_WAIT_REQEUE_PI and FUTEX_CMP_REQUEUE_PI should always
                 * be paired with each other and no other futex ops.
+                 *
+                 * We should never be requeueing a futex_q with a pi_state,
+                 * which is awaiting a futex_unlock_pi().
                 */
                if ((requeue_pi && !this->rt_waiter) ||
-                    (!requeue_pi && this->rt_waiter)) {
+                    (!requeue_pi && this->rt_waiter) ||
+                    this->pi_state) {
                        ret = -EINVAL;
                        break;
                }
diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c
index 57d86d07221e..3aca9f29d30e 100644
--- a/kernel/irq/chip.c
+++ b/kernel/irq/chip.c
@@ -272,6 +272,7 @@ void handle_nested_irq(unsigned int irq)
        raw_spin_lock_irq(&desc->lock);
+        desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
        kstat_incr_irqs_this_cpu(irq, desc);
        action = desc->action;
diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c
index 4e69e24d3d7d..96f3a1d9c379 100644
--- a/kernel/irq/irqdomain.c
+++ b/kernel/irq/irqdomain.c
@@ -177,8 +177,8 @@ struct irq_domain *irq_domain_add_simple(struct device_node *of_node,
                        irq_base = irq_alloc_descs(first_irq, first_irq, size,
                                                   of_node_to_nid(of_node));
                        if (irq_base < 0) {
-                                WARN(1, "Cannot allocate irq_descs @ IRQ%d, assuming pre-allocated\n",
+                                pr_info("Cannot allocate irq_descs @ IRQ%d, assuming pre-allocated\n",
-                                     first_irq);
+                                        first_irq);
                                irq_base = first_irq;
                        }
                } else
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c
index 4c69326aa773..35c70c9e24d8 100644
--- a/kernel/irq/manage.c
+++ b/kernel/irq/manage.c
@@ -616,6 +616,22 @@ int __irq_set_trigger(struct irq_desc *desc, unsigned int irq,
        return ret;
 }
+#ifdef CONFIG_HARDIRQS_SW_RESEND
+int irq_set_parent(int irq, int parent_irq)
+{
+        unsigned long flags;
+        struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
+        if (!desc)
+                return -EINVAL;
+        desc->parent_irq = parent_irq;
+        irq_put_desc_unlock(desc, flags);
+        return 0;
+}
+#endif
 /*
 * Default primary interrupt handler for threaded interrupts. Is
 * assigned as primary handler when request_threaded_irq is called
@@ -716,6 +732,7 @@ static void
 irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action)
 {
        cpumask_var_t mask;
+        bool valid = true;
        if (!test_and_clear_bit(IRQTF_AFFINITY, &action->thread_flags))
                return;
@@ -730,10 +747,18 @@ irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action)
        }
        raw_spin_lock_irq(&desc->lock);
-        cpumask_copy(mask, desc->irq_data.affinity);
+        /*
+         * This code is triggered unconditionally. Check the affinity
+         * mask pointer. For CPU_MASK_OFFSTACK=n this is optimized out.
+         */
+        if (desc->irq_data.affinity)
+                cpumask_copy(mask, desc->irq_data.affinity);
+        else
+                valid = false;
        raw_spin_unlock_irq(&desc->lock);
-        set_cpus_allowed_ptr(current, mask);
+        if (valid)
+                set_cpus_allowed_ptr(current, mask);
        free_cpumask_var(mask);
 }
 #else
@@ -833,6 +858,8 @@ static int irq_thread(void *data)
        init_task_work(&on_exit_work, irq_thread_dtor);
        task_work_add(current, &on_exit_work, false);
+        irq_thread_check_affinity(desc, action);
        while (!irq_wait_for_interrupt(action)) {
                irqreturn_t action_ret;
@@ -936,6 +963,16 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
                 */
                get_task_struct(t);
                new->thread = t;
+                /*
+                 * Tell the thread to set its affinity. This is
+                 * important for shared interrupt handlers as we do
+                 * not invoke setup_affinity() for the secondary
+                 * handlers as everything is already set up. Even for
+                 * interrupts marked with IRQF_NO_BALANCE this is
+                 * correct as we want the thread to move to the cpu(s)
+                 * on which the requesting code placed the interrupt.
+                 */
+                set_bit(IRQTF_AFFINITY, &new->thread_flags);
        }
        if (!alloc_cpumask_var(&mask, GFP_KERNEL)) {
diff --git a/kernel/irq/resend.c b/kernel/irq/resend.c
index 6454db7b6a4d..9065107f083e 100644
--- a/kernel/irq/resend.c
+++ b/kernel/irq/resend.c
@@ -74,6 +74,14 @@ void check_irq_resend(struct irq_desc *desc, unsigned int irq)
                if (!desc->irq_data.chip->irq_retrigger ||
                    !desc->irq_data.chip->irq_retrigger(&desc->irq_data)) {
 #ifdef CONFIG_HARDIRQS_SW_RESEND
+                        /*
+                         * If the interrupt has a parent irq and runs
+                         * in the thread context of the parent irq,
+                         * retrigger the parent.
+                         */
+                        if (desc->parent_irq &&
+                            irq_settings_is_nested_thread(desc))
+                                irq = desc->parent_irq;
                        /* Set it pending and activate the softirq: */
                        set_bit(irq, irqs_resend);
                        tasklet_schedule(&resend_tasklet);
diff --git a/kernel/ksysfs.c b/kernel/ksysfs.c
index 4e316e1acf58..6ada93c23a9a 100644
--- a/kernel/ksysfs.c
+++ b/kernel/ksysfs.c
@@ -26,7 +26,6 @@ static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
 static struct kobj_attribute _name##_attr = \
        __ATTR(_name, 0644, _name##_show, _name##_store)
-#if defined(CONFIG_HOTPLUG)
 /* current uevent sequence number */
 static ssize_t uevent_seqnum_show(struct kobject *kobj,
                                  struct kobj_attribute *attr, char *buf)
@@ -54,7 +53,7 @@ static ssize_t uevent_helper_store(struct kobject *kobj,
        return count;
 }
 KERNEL_ATTR_RW(uevent_helper);
-#endif
 #ifdef CONFIG_PROFILING
 static ssize_t profiling_show(struct kobject *kobj,
@@ -141,6 +140,23 @@ static ssize_t fscaps_show(struct kobject *kobj,
 }
 KERNEL_ATTR_RO(fscaps);
+int rcu_expedited;
+static ssize_t rcu_expedited_show(struct kobject *kobj,
+                                  struct kobj_attribute *attr, char *buf)
+{
+        return sprintf(buf, "%d\n", rcu_expedited);
+}
+static ssize_t rcu_expedited_store(struct kobject *kobj,
+                                   struct kobj_attribute *attr,
+                                   const char *buf, size_t count)
+{
+        if (kstrtoint(buf, 0, &rcu_expedited))
+                return -EINVAL;
+        return count;
+}
+KERNEL_ATTR_RW(rcu_expedited);
 /*
 * Make /sys/kernel/notes give the raw contents of our kernel .notes section.
 */
@@ -169,10 +185,8 @@ EXPORT_SYMBOL_GPL(kernel_kobj);
 static struct attribute * kernel_attrs[] = {
        &fscaps_attr.attr,
-#if defined(CONFIG_HOTPLUG)
        &uevent_seqnum_attr.attr,
        &uevent_helper_attr.attr,
-#endif
 #ifdef CONFIG_PROFILING
        &profiling_attr.attr,
 #endif
@@ -182,6 +196,7 @@ static struct attribute * kernel_attrs[] = {
        &kexec_crash_size_attr.attr,
        &vmcoreinfo_attr.attr,
 #endif
+        &rcu_expedited_attr.attr,
        NULL
 };
diff --git a/kernel/kthread.c b/kernel/kthread.c
index 29fb60caecb5..691dc2ef9baf 100644
--- a/kernel/kthread.c
+++ b/kernel/kthread.c
@@ -428,7 +428,7 @@ int kthreadd(void *unused)
        set_task_comm(tsk, "kthreadd");
        ignore_signals(tsk);
        set_cpus_allowed_ptr(tsk, cpu_all_mask);
-        set_mems_allowed(node_states[N_HIGH_MEMORY]);
+        set_mems_allowed(node_states[N_MEMORY]);
        current->flags |= PF_NOFREEZE;
diff --git a/kernel/lockdep_proc.c b/kernel/lockdep_proc.c
index 91c32a0b612c..b2c71c5873e4 100644
--- a/kernel/lockdep_proc.c
+++ b/kernel/lockdep_proc.c
@@ -39,7 +39,7 @@ static void l_stop(struct seq_file *m, void *v)
 static void print_name(struct seq_file *m, struct lock_class *class)
 {
-        char str[128];
+        char str[KSYM_NAME_LEN];
        const char *name = class->name;
        if (!name) {
diff --git a/kernel/modsign_pubkey.c b/kernel/modsign_pubkey.c
index 4646eb2c3820..767e559dfb10 100644
--- a/kernel/modsign_pubkey.c
+++ b/kernel/modsign_pubkey.c
@@ -21,10 +21,10 @@ struct key *modsign_keyring;
 extern __initdata const u8 modsign_certificate_list[];
 extern __initdata const u8 modsign_certificate_list_end[];
 asm(".section .init.data,\"aw\"\n"
-    "modsign_certificate_list:\n"
+    SYMBOL_PREFIX "modsign_certificate_list:\n"
    ".incbin \"signing_key.x509\"\n"
    ".incbin \"extra_certificates\"\n"
-    "modsign_certificate_list_end:"
+    SYMBOL_PREFIX "modsign_certificate_list_end:"
    );
 /*
diff --git a/kernel/module.c b/kernel/module.c
index 6085f5ef88ea..808bd62e1723 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -372,9 +372,6 @@ static bool check_symbol(const struct symsearch *syms,
                        printk(KERN_WARNING "Symbol %s is being used "
                               "by a non-GPL module, which will not "
                               "be allowed in the future\n", fsa->name);
-                        printk(KERN_WARNING "Please see the file "
-                               "Documentation/feature-removal-schedule.txt "
-                               "in the kernel source tree for more details.\n");
                }
        }
@@ -2293,12 +2290,17 @@ static void layout_symtab(struct module *mod, struct load_info *info)
        src = (void *)info->hdr + symsect->sh_offset;
        nsrc = symsect->sh_size / sizeof(*src);
+        /* strtab always starts with a nul, so offset 0 is the empty string. */
+        strtab_size = 1;
        /* Compute total space required for the core symbols' strtab. */
-        for (ndst = i = strtab_size = 1; i < nsrc; ++i, ++src)
+        for (ndst = i = 0; i < nsrc; i++) {
-                if (is_core_symbol(src, info->sechdrs, info->hdr->e_shnum)) {
+                if (i == 0 ||
-                        strtab_size += strlen(&info->strtab[src->st_name]) + 1;
+                    is_core_symbol(src+i, info->sechdrs, info->hdr->e_shnum)) {
+                        strtab_size += strlen(&info->strtab[src[i].st_name])+1;
                        ndst++;
                }
+        }
        /* Append room for core symbols at end of core part. */
        info->symoffs = ALIGN(mod->core_size, symsect->sh_addralign ?: 1);
@@ -2332,15 +2334,15 @@ static void add_kallsyms(struct module *mod, const struct load_info *info)
        mod->core_symtab = dst = mod->module_core + info->symoffs;
        mod->core_strtab = s = mod->module_core + info->stroffs;
        src = mod->symtab;
-        *dst = *src;
        *s++ = 0;
-        for (ndst = i = 1; i < mod->num_symtab; ++i, ++src) {
+        for (ndst = i = 0; i < mod->num_symtab; i++) {
-                if (!is_core_symbol(src, info->sechdrs, info->hdr->e_shnum))
+                if (i == 0 ||
-                        continue;
+                    is_core_symbol(src+i, info->sechdrs, info->hdr->e_shnum)) {
+                        dst[ndst] = src[i];
-                dst[ndst] = *src;
+                        dst[ndst++].st_name = s - mod->core_strtab;
-                dst[ndst++].st_name = s - mod->core_strtab;
+                        s += strlcpy(s, &mod->strtab[src[i].st_name],
-                s += strlcpy(s, &mod->strtab[src->st_name], KSYM_NAME_LEN) + 1;
+                                     KSYM_NAME_LEN) + 1;
+                }
        }
        mod->core_num_syms = ndst;
 }
diff --git a/kernel/module_signing.c b/kernel/module_signing.c
index ea1b1df5dbb0..f2970bddc5ea 100644
--- a/kernel/module_signing.c
+++ b/kernel/module_signing.c
@@ -27,13 +27,13 @@
 *      - Information block
 */
 struct module_signature {
-        enum pkey_algo          algo : 8;       /* Public-key crypto algorithm */
+        u8      algo;           /* Public-key crypto algorithm [enum pkey_algo] */
-        enum pkey_hash_algo     hash : 8;       /* Digest algorithm */
+        u8      hash;           /* Digest algorithm [enum pkey_hash_algo] */
-        enum pkey_id_type       id_type : 8;    /* Key identifier type */
+        u8      id_type;        /* Key identifier type [enum pkey_id_type] */
-        u8                      signer_len;     /* Length of signer's name */
+        u8      signer_len;     /* Length of signer's name */
-        u8                      key_id_len;     /* Length of key identifier */
+        u8      key_id_len;     /* Length of key identifier */
-        u8                      __pad[3];
+        u8      __pad[3];
-        __be32                  sig_len;        /* Length of signature data */
+        __be32  sig_len;        /* Length of signature data */
 };
 /*
diff --git a/kernel/padata.c b/kernel/padata.c
index 89fe3d1b9efb..072f4ee4eb89 100644
--- a/kernel/padata.c
+++ b/kernel/padata.c
@@ -171,7 +171,7 @@ static struct padata_priv *padata_get_next(struct parallel_data *pd)
 {
        int cpu, num_cpus;
        unsigned int next_nr, next_index;
-        struct padata_parallel_queue *queue, *next_queue;
+        struct padata_parallel_queue *next_queue;
        struct padata_priv *padata;
        struct padata_list *reorder;
@@ -204,8 +204,7 @@ static struct padata_priv *padata_get_next(struct parallel_data *pd)
                goto out;
        }
-        queue = per_cpu_ptr(pd->pqueue, smp_processor_id());
+        if (__this_cpu_read(pd->pqueue->cpu_index) == next_queue->cpu_index) {
-        if (queue->cpu_index == next_queue->cpu_index) {
                padata = ERR_PTR(-ENODATA);
                goto out;
        }
diff --git a/kernel/pid.c b/kernel/pid.c
index 3026ddae0a34..36aa02ff17d6 100644
--- a/kernel/pid.c
+++ b/kernel/pid.c
@@ -1,8 +1,8 @@
 /*
 * Generic pidhash and scalable, time-bounded PID allocator
 *
- * (C) 2002-2003 William Irwin, IBM
+ * (C) 2002-2003 Nadia Yvette Chambers, IBM
- * (C) 2004 William Irwin, Oracle
+ * (C) 2004 Nadia Yvette Chambers, Oracle
 * (C) 2002-2004 Ingo Molnar, Red Hat
 *
 * pid-structures are backing objects for tasks sharing a given ID to chain
@@ -84,21 +84,6 @@ struct pid_namespace init_pid_ns = {
 };
 EXPORT_SYMBOL_GPL(init_pid_ns);
-int is_container_init(struct task_struct *tsk)
-{
-        int ret = 0;
-        struct pid *pid;
-        rcu_read_lock();
-        pid = task_pid(tsk);
-        if (pid != NULL && pid->numbers[pid->level].nr == 1)
-                ret = 1;
-        rcu_read_unlock();
-        return ret;
-}
-EXPORT_SYMBOL(is_container_init);
 /*
 * Note: disable interrupts while the pidmap_lock is held as an
 * interrupt might come in and do read_lock(&tasklist_lock).
diff --git a/kernel/posix-cpu-timers.c b/kernel/posix-cpu-timers.c
index 125cb67daa21..d73840271dce 100644
--- a/kernel/posix-cpu-timers.c
+++ b/kernel/posix-cpu-timers.c
@@ -217,30 +217,6 @@ static int cpu_clock_sample(const clockid_t which_clock, struct task_struct *p,
        return 0;
 }
-void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times)
-{
-        struct signal_struct *sig = tsk->signal;
-        struct task_struct *t;
-        times->utime = sig->utime;
-        times->stime = sig->stime;
-        times->sum_exec_runtime = sig->sum_sched_runtime;
-        rcu_read_lock();
-        /* make sure we can trust tsk->thread_group list */
-        if (!likely(pid_alive(tsk)))
-                goto out;
-        t = tsk;
-        do {
-                times->utime += t->utime;
-                times->stime += t->stime;
-                times->sum_exec_runtime += task_sched_runtime(t);
-        } while_each_thread(tsk, t);
-out:
-        rcu_read_unlock();
-}
 static void update_gt_cputime(struct task_cputime *a, struct task_cputime *b)
 {
        if (b->utime > a->utime)
diff --git a/kernel/power/main.c b/kernel/power/main.c
index f458238109cc..1c16f9167de1 100644
--- a/kernel/power/main.c
+++ b/kernel/power/main.c
@@ -59,7 +59,7 @@ static ssize_t pm_async_store(struct kobject *kobj, struct kobj_attribute *attr,
 {
        unsigned long val;
-        if (strict_strtoul(buf, 10, &val))
+        if (kstrtoul(buf, 10, &val))
                return -EINVAL;
        if (val > 1)
diff --git a/kernel/power/process.c b/kernel/power/process.c
index 87da817f9e13..d5a258b60c6f 100644
--- a/kernel/power/process.c
+++ b/kernel/power/process.c
@@ -48,18 +48,7 @@ static int try_to_freeze_tasks(bool user_only)
                        if (p == current || !freeze_task(p))
                                continue;
-                        /*
+                        if (!freezer_should_skip(p))
-                         * Now that we've done set_freeze_flag, don't
-                         * perturb a task in TASK_STOPPED or TASK_TRACED.
-                         * It is "frozen enough".  If the task does wake
-                         * up, it will immediately call try_to_freeze.
-                         *
-                         * Because freeze_task() goes through p's scheduler lock, it's
-                         * guaranteed that TASK_STOPPED/TRACED -> TASK_RUNNING
-                         * transition can't race with task state testing here.
-                         */
-                        if (!task_is_stopped_or_traced(p) &&
-                            !freezer_should_skip(p))
                                todo++;
                } while_each_thread(g, p);
                read_unlock(&tasklist_lock);
diff --git a/kernel/power/qos.c b/kernel/power/qos.c
index 846bd42c7ed1..9322ff7eaad6 100644
--- a/kernel/power/qos.c
+++ b/kernel/power/qos.c
@@ -213,6 +213,69 @@ int pm_qos_update_target(struct pm_qos_constraints *c, struct plist_node *node,
 }
 /**
+ * pm_qos_flags_remove_req - Remove device PM QoS flags request.
+ * @pqf: Device PM QoS flags set to remove the request from.
+ * @req: Request to remove from the set.
+ */
+static void pm_qos_flags_remove_req(struct pm_qos_flags *pqf,
+                                    struct pm_qos_flags_request *req)
+{
+        s32 val = 0;
+        list_del(&req->node);
+        list_for_each_entry(req, &pqf->list, node)
+                val |= req->flags;
+        pqf->effective_flags = val;
+}
+/**
+ * pm_qos_update_flags - Update a set of PM QoS flags.
+ * @pqf: Set of flags to update.
+ * @req: Request to add to the set, to modify, or to remove from the set.
+ * @action: Action to take on the set.
+ * @val: Value of the request to add or modify.
+ *
+ * Update the given set of PM QoS flags and call notifiers if the aggregate
+ * value has changed.  Returns 1 if the aggregate constraint value has changed,
+ * 0 otherwise.
+ */
+bool pm_qos_update_flags(struct pm_qos_flags *pqf,
+                         struct pm_qos_flags_request *req,
+                         enum pm_qos_req_action action, s32 val)
+{
+        unsigned long irqflags;
+        s32 prev_value, curr_value;
+        spin_lock_irqsave(&pm_qos_lock, irqflags);
+        prev_value = list_empty(&pqf->list) ? 0 : pqf->effective_flags;
+        switch (action) {
+        case PM_QOS_REMOVE_REQ:
+                pm_qos_flags_remove_req(pqf, req);
+                break;
+        case PM_QOS_UPDATE_REQ:
+                pm_qos_flags_remove_req(pqf, req);
+        case PM_QOS_ADD_REQ:
+                req->flags = val;
+                INIT_LIST_HEAD(&req->node);
+                list_add_tail(&req->node, &pqf->list);
+                pqf->effective_flags |= val;
+                break;
+        default:
+                /* no action */
+                ;
+        }
+        curr_value = list_empty(&pqf->list) ? 0 : pqf->effective_flags;
+        spin_unlock_irqrestore(&pm_qos_lock, irqflags);
+        return prev_value != curr_value;
+}
+/**
 * pm_qos_request - returns current system wide qos expectation
 * @pm_qos_class: identification of which qos value is requested
 *
@@ -500,7 +563,7 @@ static ssize_t pm_qos_power_write(struct file *filp, const char __user *buf,
                } else {
                        ascii_value[count] = '\0';
                }
-                ret = strict_strtoul(ascii_value, 16, &ulval);
+                ret = kstrtoul(ascii_value, 16, &ulval);
                if (ret) {
                        pr_debug("%s, 0x%lx, 0x%x\n", ascii_value, ulval, ret);
                        return -EINVAL;
diff --git a/kernel/power/swap.c b/kernel/power/swap.c
index 3c9d764eb0d8..7c33ed200410 100644
--- a/kernel/power/swap.c
+++ b/kernel/power/swap.c
@@ -126,7 +126,7 @@ static int swsusp_extents_insert(unsigned long swap_offset)
        /* Figure out where to put the new node */
        while (*new) {
-                ext = container_of(*new, struct swsusp_extent, node);
+                ext = rb_entry(*new, struct swsusp_extent, node);
                parent = *new;
                if (swap_offset < ext->start) {
                        /* Try to merge */
diff --git a/kernel/printk.c b/kernel/printk.c
index 2d607f4d1797..19c0d7bcf24a 100644
--- a/kernel/printk.c
+++ b/kernel/printk.c
@@ -87,6 +87,12 @@ static DEFINE_SEMAPHORE(console_sem);
 struct console *console_drivers;
 EXPORT_SYMBOL_GPL(console_drivers);
+#ifdef CONFIG_LOCKDEP
+static struct lockdep_map console_lock_dep_map = {
+        .name = "console_lock"
+};
+#endif
 /*
 * This is used for debugging the mess that is the VT code by
 * keeping track if we have the console semaphore held. It's
@@ -741,6 +747,21 @@ void __init setup_log_buf(int early)
                free, (free * 100) / __LOG_BUF_LEN);
 }
+static bool __read_mostly ignore_loglevel;
+static int __init ignore_loglevel_setup(char *str)
+{
+        ignore_loglevel = 1;
+        printk(KERN_INFO "debug: ignoring loglevel setting.\n");
+        return 0;
+}
+early_param("ignore_loglevel", ignore_loglevel_setup);
+module_param(ignore_loglevel, bool, S_IRUGO | S_IWUSR);
+MODULE_PARM_DESC(ignore_loglevel, "ignore loglevel setting, to"
+        "print all kernel messages to the console.");
 #ifdef CONFIG_BOOT_PRINTK_DELAY
 static int boot_delay; /* msecs delay after each printk during bootup */
@@ -764,13 +785,15 @@ static int __init boot_delay_setup(char *str)
 }
 __setup("boot_delay=", boot_delay_setup);
-static void boot_delay_msec(void)
+static void boot_delay_msec(int level)
 {
        unsigned long long k;
        unsigned long timeout;
-        if (boot_delay == 0 || system_state != SYSTEM_BOOTING)
+        if ((boot_delay == 0 || system_state != SYSTEM_BOOTING)
+                || (level >= console_loglevel && !ignore_loglevel)) {
                return;
+        }
        k = (unsigned long long)loops_per_msec * boot_delay;
@@ -789,7 +812,7 @@ static void boot_delay_msec(void)
        }
 }
 #else
-static inline void boot_delay_msec(void)
+static inline void boot_delay_msec(int level)
 {
 }
 #endif
@@ -1232,21 +1255,6 @@ SYSCALL_DEFINE3(syslog, int, type, char __user *, buf, int, len)
        return do_syslog(type, buf, len, SYSLOG_FROM_CALL);
 }
-static bool __read_mostly ignore_loglevel;
-static int __init ignore_loglevel_setup(char *str)
-{
-        ignore_loglevel = 1;
-        printk(KERN_INFO "debug: ignoring loglevel setting.\n");
-        return 0;
-}
-early_param("ignore_loglevel", ignore_loglevel_setup);
-module_param(ignore_loglevel, bool, S_IRUGO | S_IWUSR);
-MODULE_PARM_DESC(ignore_loglevel, "ignore loglevel setting, to"
-        "print all kernel messages to the console.");
 /*
 * Call the console drivers, asking them to write out
 * log_buf[start] to log_buf[end - 1].
@@ -1492,7 +1500,7 @@ asmlinkage int vprintk_emit(int facility, int level,
        int this_cpu;
        int printed_len = 0;
-        boot_delay_msec();
+        boot_delay_msec(level);
        printk_delay();
        /* This stops the holder of console_sem just where we want him */
@@ -1908,12 +1916,14 @@ static int __cpuinit console_cpu_notify(struct notifier_block *self,
 */
 void console_lock(void)
 {
-        BUG_ON(in_interrupt());
+        might_sleep();
        down(&console_sem);
        if (console_suspended)
                return;
        console_locked = 1;
        console_may_schedule = 1;
+        mutex_acquire(&console_lock_dep_map, 0, 0, _RET_IP_);
 }
 EXPORT_SYMBOL(console_lock);
@@ -1935,6 +1945,7 @@ int console_trylock(void)
        }
        console_locked = 1;
        console_may_schedule = 0;
+        mutex_acquire(&console_lock_dep_map, 0, 1, _RET_IP_);
        return 1;
 }
 EXPORT_SYMBOL(console_trylock);
@@ -2095,6 +2106,7 @@ skip:
                local_irq_restore(flags);
        }
        console_locked = 0;
+        mutex_release(&console_lock_dep_map, 1, _RET_IP_);
        /* Release the exclusive_console once it is used */
        if (unlikely(exclusive_console))
diff --git a/kernel/profile.c b/kernel/profile.c
index 76b8e77773ee..1f391819c42f 100644
--- a/kernel/profile.c
+++ b/kernel/profile.c
@@ -8,9 +8,10 @@
 *  Scheduler profiling support, Arjan van de Ven and Ingo Molnar,
 *      Red Hat, July 2004
 *  Consolidation of architecture support code for profiling,
- *      William Irwin, Oracle, July 2004
+ *      Nadia Yvette Chambers, Oracle, July 2004
 *  Amortized hit count accounting via per-cpu open-addressed hashtables
- *      to resolve timer interrupt livelocks, William Irwin, Oracle, 2004
+ *      to resolve timer interrupt livelocks, Nadia Yvette Chambers,
+ *      Oracle, 2004
 */
 #include <linux/export.h>
@@ -256,7 +257,7 @@ EXPORT_SYMBOL_GPL(unregister_timer_hook);
 * pagetable hash functions, but uses a full hashtable full of finite
 * collision chains, not just pairs of them.
 *
- * -- wli
+ * -- nyc
 */
 static void __profile_flip_buffers(void *unused)
 {
diff --git a/kernel/ptrace.c b/kernel/ptrace.c
index 7b09b88862cc..1599157336a6 100644
--- a/kernel/ptrace.c
+++ b/kernel/ptrace.c
@@ -463,6 +463,9 @@ void exit_ptrace(struct task_struct *tracer)
                return;
        list_for_each_entry_safe(p, n, &tracer->ptraced, ptrace_entry) {
+                if (unlikely(p->ptrace & PT_EXITKILL))
+                        send_sig_info(SIGKILL, SEND_SIG_FORCED, p);
                if (__ptrace_detach(tracer, p))
                        list_add(&p->ptrace_entry, &ptrace_dead);
        }
diff --git a/kernel/rcu.h b/kernel/rcu.h
index 8ba99cdc6515..20dfba576c2b 100644
--- a/kernel/rcu.h
+++ b/kernel/rcu.h
@@ -109,4 +109,6 @@ static inline bool __rcu_reclaim(char *rn, struct rcu_head *head)
        }
 }
+extern int rcu_expedited;
 #endif /* __LINUX_RCU_H */
diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c
index 29ca1c6da594..a2cf76177b44 100644
--- a/kernel/rcupdate.c
+++ b/kernel/rcupdate.c
@@ -46,12 +46,15 @@
 #include <linux/export.h>
 #include <linux/hardirq.h>
 #include <linux/delay.h>
+#include <linux/module.h>
 #define CREATE_TRACE_POINTS
 #include <trace/events/rcu.h>
 #include "rcu.h"
+module_param(rcu_expedited, int, 0);
 #ifdef CONFIG_PREEMPT_RCU
 /*
diff --git a/kernel/rcutiny.c b/kernel/rcutiny.c
index e4c6a598d6f7..e7dce58f9c2a 100644
--- a/kernel/rcutiny.c
+++ b/kernel/rcutiny.c
@@ -195,7 +195,7 @@ EXPORT_SYMBOL(rcu_is_cpu_idle);
 */
 int rcu_is_cpu_rrupt_from_idle(void)
 {
-        return rcu_dynticks_nesting <= 0;
+        return rcu_dynticks_nesting <= 1;
 }
 /*
diff --git a/kernel/rcutiny_plugin.h b/kernel/rcutiny_plugin.h
index 3d0190282204..f85016a2309b 100644
--- a/kernel/rcutiny_plugin.h
+++ b/kernel/rcutiny_plugin.h
@@ -706,7 +706,10 @@ void synchronize_rcu(void)
                return;
        /* Once we get past the fastpath checks, same code as rcu_barrier(). */
-        rcu_barrier();
+        if (rcu_expedited)
+                synchronize_rcu_expedited();
+        else
+                rcu_barrier();
 }
 EXPORT_SYMBOL_GPL(synchronize_rcu);
diff --git a/kernel/rcutorture.c b/kernel/rcutorture.c
index aaa7b9f3532a..31dea01c85fd 100644
--- a/kernel/rcutorture.c
+++ b/kernel/rcutorture.c
@@ -339,7 +339,6 @@ rcu_stutter_wait(char *title)
 struct rcu_torture_ops {
        void (*init)(void);
-        void (*cleanup)(void);
        int (*readlock)(void);
        void (*read_delay)(struct rcu_random_state *rrsp);
        void (*readunlock)(int idx);
@@ -431,7 +430,6 @@ static void rcu_torture_deferred_free(struct rcu_torture *p)
 static struct rcu_torture_ops rcu_ops = {
        .init           = NULL,
-        .cleanup        = NULL,
        .readlock       = rcu_torture_read_lock,
        .read_delay     = rcu_read_delay,
        .readunlock     = rcu_torture_read_unlock,
@@ -475,7 +473,6 @@ static void rcu_sync_torture_init(void)
 static struct rcu_torture_ops rcu_sync_ops = {
        .init           = rcu_sync_torture_init,
-        .cleanup        = NULL,
        .readlock       = rcu_torture_read_lock,
        .read_delay     = rcu_read_delay,
        .readunlock     = rcu_torture_read_unlock,
@@ -493,7 +490,6 @@ static struct rcu_torture_ops rcu_sync_ops = {
 static struct rcu_torture_ops rcu_expedited_ops = {
        .init           = rcu_sync_torture_init,
-        .cleanup        = NULL,
        .readlock       = rcu_torture_read_lock,
        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
        .readunlock     = rcu_torture_read_unlock,
@@ -536,7 +532,6 @@ static void rcu_bh_torture_deferred_free(struct rcu_torture *p)
 static struct rcu_torture_ops rcu_bh_ops = {
        .init           = NULL,
-        .cleanup        = NULL,
        .readlock       = rcu_bh_torture_read_lock,
        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
        .readunlock     = rcu_bh_torture_read_unlock,
@@ -553,7 +548,6 @@ static struct rcu_torture_ops rcu_bh_ops = {
 static struct rcu_torture_ops rcu_bh_sync_ops = {
        .init           = rcu_sync_torture_init,
-        .cleanup        = NULL,
        .readlock       = rcu_bh_torture_read_lock,
        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
        .readunlock     = rcu_bh_torture_read_unlock,
@@ -570,7 +564,6 @@ static struct rcu_torture_ops rcu_bh_sync_ops = {
 static struct rcu_torture_ops rcu_bh_expedited_ops = {
        .init           = rcu_sync_torture_init,
-        .cleanup        = NULL,
        .readlock       = rcu_bh_torture_read_lock,
        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
        .readunlock     = rcu_bh_torture_read_unlock,
@@ -589,19 +582,7 @@ static struct rcu_torture_ops rcu_bh_expedited_ops = {
 * Definitions for srcu torture testing.
 */
-static struct srcu_struct srcu_ctl;
+DEFINE_STATIC_SRCU(srcu_ctl);
-static void srcu_torture_init(void)
-{
-        init_srcu_struct(&srcu_ctl);
-        rcu_sync_torture_init();
-}
-static void srcu_torture_cleanup(void)
-{
-        synchronize_srcu(&srcu_ctl);
-        cleanup_srcu_struct(&srcu_ctl);
-}
 static int srcu_torture_read_lock(void) __acquires(&srcu_ctl)
 {
@@ -672,8 +653,7 @@ static int srcu_torture_stats(char *page)
 }
 static struct rcu_torture_ops srcu_ops = {
-        .init           = srcu_torture_init,
+        .init           = rcu_sync_torture_init,
-        .cleanup        = srcu_torture_cleanup,
        .readlock       = srcu_torture_read_lock,
        .read_delay     = srcu_read_delay,
        .readunlock     = srcu_torture_read_unlock,
@@ -687,8 +667,7 @@ static struct rcu_torture_ops srcu_ops = {
 };
 static struct rcu_torture_ops srcu_sync_ops = {
-        .init           = srcu_torture_init,
+        .init           = rcu_sync_torture_init,
-        .cleanup        = srcu_torture_cleanup,
        .readlock       = srcu_torture_read_lock,
        .read_delay     = srcu_read_delay,
        .readunlock     = srcu_torture_read_unlock,
@@ -712,8 +691,7 @@ static void srcu_torture_read_unlock_raw(int idx) __releases(&srcu_ctl)
 }
 static struct rcu_torture_ops srcu_raw_ops = {
-        .init           = srcu_torture_init,
+        .init           = rcu_sync_torture_init,
-        .cleanup        = srcu_torture_cleanup,
        .readlock       = srcu_torture_read_lock_raw,
        .read_delay     = srcu_read_delay,
        .readunlock     = srcu_torture_read_unlock_raw,
@@ -727,8 +705,7 @@ static struct rcu_torture_ops srcu_raw_ops = {
 };
 static struct rcu_torture_ops srcu_raw_sync_ops = {
-        .init           = srcu_torture_init,
+        .init           = rcu_sync_torture_init,
-        .cleanup        = srcu_torture_cleanup,
        .readlock       = srcu_torture_read_lock_raw,
        .read_delay     = srcu_read_delay,
        .readunlock     = srcu_torture_read_unlock_raw,
@@ -747,8 +724,7 @@ static void srcu_torture_synchronize_expedited(void)
 }
 static struct rcu_torture_ops srcu_expedited_ops = {
-        .init           = srcu_torture_init,
+        .init           = rcu_sync_torture_init,
-        .cleanup        = srcu_torture_cleanup,
        .readlock       = srcu_torture_read_lock,
        .read_delay     = srcu_read_delay,
        .readunlock     = srcu_torture_read_unlock,
@@ -783,7 +759,6 @@ static void rcu_sched_torture_deferred_free(struct rcu_torture *p)
 static struct rcu_torture_ops sched_ops = {
        .init           = rcu_sync_torture_init,
-        .cleanup        = NULL,
        .readlock       = sched_torture_read_lock,
        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
        .readunlock     = sched_torture_read_unlock,
@@ -799,7 +774,6 @@ static struct rcu_torture_ops sched_ops = {
 static struct rcu_torture_ops sched_sync_ops = {
        .init           = rcu_sync_torture_init,
-        .cleanup        = NULL,
        .readlock       = sched_torture_read_lock,
        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
        .readunlock     = sched_torture_read_unlock,
@@ -814,7 +788,6 @@ static struct rcu_torture_ops sched_sync_ops = {
 static struct rcu_torture_ops sched_expedited_ops = {
        .init           = rcu_sync_torture_init,
-        .cleanup        = NULL,
        .readlock       = sched_torture_read_lock,
        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
        .readunlock     = sched_torture_read_unlock,
@@ -1396,12 +1369,16 @@ rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, char *tag)
                 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d "
                 "test_boost=%d/%d test_boost_interval=%d "
                 "test_boost_duration=%d shutdown_secs=%d "
+                 "stall_cpu=%d stall_cpu_holdoff=%d "
+                 "n_barrier_cbs=%d "
                 "onoff_interval=%d onoff_holdoff=%d\n",
                 torture_type, tag, nrealreaders, nfakewriters,
                 stat_interval, verbose, test_no_idle_hz, shuffle_interval,
                 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter,
                 test_boost, cur_ops->can_boost,
                 test_boost_interval, test_boost_duration, shutdown_secs,
+                 stall_cpu, stall_cpu_holdoff,
+                 n_barrier_cbs,
                 onoff_interval, onoff_holdoff);
 }
@@ -1502,6 +1479,7 @@ rcu_torture_onoff(void *arg)
        unsigned long delta;
        int maxcpu = -1;
        DEFINE_RCU_RANDOM(rand);
+        int ret;
        unsigned long starttime;
        VERBOSE_PRINTK_STRING("rcu_torture_onoff task started");
@@ -1522,7 +1500,13 @@ rcu_torture_onoff(void *arg)
                                         torture_type, cpu);
                        starttime = jiffies;
                        n_offline_attempts++;
-                        if (cpu_down(cpu) == 0) {
+                        ret = cpu_down(cpu);
+                        if (ret) {
+                                if (verbose)
+                                        pr_alert("%s" TORTURE_FLAG
+                                                 "rcu_torture_onoff task: offline %d failed: errno %d\n",
+                                                 torture_type, cpu, ret);
+                        } else {
                                if (verbose)
                                        pr_alert("%s" TORTURE_FLAG
                                                 "rcu_torture_onoff task: offlined %d\n",
@@ -1936,8 +1920,6 @@ rcu_torture_cleanup(void)
        rcu_torture_stats_print();  /* -After- the stats thread is stopped! */
-        if (cur_ops->cleanup)
-                cur_ops->cleanup();
        if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error)
                rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE");
        else if (n_online_successes != n_online_attempts ||
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index 74df86bd9204..e441b77b614e 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -68,9 +68,9 @@ static struct lock_class_key rcu_fqs_class[RCU_NUM_LVLS];
        .level = { &sname##_state.node[0] }, \
        .call = cr, \
        .fqs_state = RCU_GP_IDLE, \
-        .gpnum = -300, \
+        .gpnum = 0UL - 300UL, \
-        .completed = -300, \
+        .completed = 0UL - 300UL, \
-        .onofflock = __RAW_SPIN_LOCK_UNLOCKED(&sname##_state.onofflock), \
+        .orphan_lock = __RAW_SPIN_LOCK_UNLOCKED(&sname##_state.orphan_lock), \
        .orphan_nxttail = &sname##_state.orphan_nxtlist, \
        .orphan_donetail = &sname##_state.orphan_donelist, \
        .barrier_mutex = __MUTEX_INITIALIZER(sname##_state.barrier_mutex), \
@@ -207,18 +207,15 @@ EXPORT_SYMBOL_GPL(rcu_note_context_switch);
 DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
        .dynticks_nesting = DYNTICK_TASK_EXIT_IDLE,
        .dynticks = ATOMIC_INIT(1),
-#if defined(CONFIG_RCU_USER_QS) && !defined(CONFIG_RCU_USER_QS_FORCE)
-        .ignore_user_qs = true,
-#endif
 };
-static int blimit = 10;         /* Maximum callbacks per rcu_do_batch. */
+static long blimit = 10;        /* Maximum callbacks per rcu_do_batch. */
-static int qhimark = 10000;     /* If this many pending, ignore blimit. */
+static long qhimark = 10000;    /* If this many pending, ignore blimit. */
-static int qlowmark = 100;      /* Once only this many pending, use blimit. */
+static long qlowmark = 100;     /* Once only this many pending, use blimit. */
-module_param(blimit, int, 0444);
+module_param(blimit, long, 0444);
-module_param(qhimark, int, 0444);
+module_param(qhimark, long, 0444);
-module_param(qlowmark, int, 0444);
+module_param(qlowmark, long, 0444);
 int rcu_cpu_stall_suppress __read_mostly; /* 1 = suppress stall warnings. */
 int rcu_cpu_stall_timeout __read_mostly = CONFIG_RCU_CPU_STALL_TIMEOUT;
@@ -303,7 +300,8 @@ EXPORT_SYMBOL_GPL(rcu_sched_force_quiescent_state);
 static int
 cpu_has_callbacks_ready_to_invoke(struct rcu_data *rdp)
 {
-        return &rdp->nxtlist != rdp->nxttail[RCU_DONE_TAIL];
+        return &rdp->nxtlist != rdp->nxttail[RCU_DONE_TAIL] &&
+               rdp->nxttail[RCU_DONE_TAIL] != NULL;
 }
 /*
@@ -312,8 +310,11 @@ cpu_has_callbacks_ready_to_invoke(struct rcu_data *rdp)
 static int
 cpu_needs_another_gp(struct rcu_state *rsp, struct rcu_data *rdp)
 {
-        return *rdp->nxttail[RCU_DONE_TAIL +
+        struct rcu_head **ntp;
-                             ACCESS_ONCE(rsp->completed) != rdp->completed] &&
+        ntp = rdp->nxttail[RCU_DONE_TAIL +
+                           (ACCESS_ONCE(rsp->completed) != rdp->completed)];
+        return rdp->nxttail[RCU_DONE_TAIL] && ntp && *ntp &&
               !rcu_gp_in_progress(rsp);
 }
@@ -416,29 +417,7 @@ EXPORT_SYMBOL_GPL(rcu_idle_enter);
 */
 void rcu_user_enter(void)
 {
-        unsigned long flags;
+        rcu_eqs_enter(1);
-        struct rcu_dynticks *rdtp;
-        /*
-         * Some contexts may involve an exception occuring in an irq,
-         * leading to that nesting:
-         * rcu_irq_enter() rcu_user_exit() rcu_user_exit() rcu_irq_exit()
-         * This would mess up the dyntick_nesting count though. And rcu_irq_*()
-         * helpers are enough to protect RCU uses inside the exception. So
-         * just return immediately if we detect we are in an IRQ.
-         */
-        if (in_interrupt())
-                return;
-        WARN_ON_ONCE(!current->mm);
-        local_irq_save(flags);
-        rdtp = &__get_cpu_var(rcu_dynticks);
-        if (!rdtp->ignore_user_qs && !rdtp->in_user) {
-                rdtp->in_user = true;
-                rcu_eqs_enter(true);
-        }
-        local_irq_restore(flags);
 }
 /**
@@ -575,27 +554,7 @@ EXPORT_SYMBOL_GPL(rcu_idle_exit);
 */
 void rcu_user_exit(void)
 {
-        unsigned long flags;
+        rcu_eqs_exit(1);
-        struct rcu_dynticks *rdtp;
-        /*
-         * Some contexts may involve an exception occuring in an irq,
-         * leading to that nesting:
-         * rcu_irq_enter() rcu_user_exit() rcu_user_exit() rcu_irq_exit()
-         * This would mess up the dyntick_nesting count though. And rcu_irq_*()
-         * helpers are enough to protect RCU uses inside the exception. So
-         * just return immediately if we detect we are in an IRQ.
-         */
-        if (in_interrupt())
-                return;
-        local_irq_save(flags);
-        rdtp = &__get_cpu_var(rcu_dynticks);
-        if (rdtp->in_user) {
-                rdtp->in_user = false;
-                rcu_eqs_exit(true);
-        }
-        local_irq_restore(flags);
 }
 /**
@@ -718,21 +677,6 @@ int rcu_is_cpu_idle(void)
 }
 EXPORT_SYMBOL(rcu_is_cpu_idle);
-#ifdef CONFIG_RCU_USER_QS
-void rcu_user_hooks_switch(struct task_struct *prev,
-                           struct task_struct *next)
-{
-        struct rcu_dynticks *rdtp;
-        /* Interrupts are disabled in context switch */
-        rdtp = &__get_cpu_var(rcu_dynticks);
-        if (!rdtp->ignore_user_qs) {
-                clear_tsk_thread_flag(prev, TIF_NOHZ);
-                set_tsk_thread_flag(next, TIF_NOHZ);
-        }
-}
-#endif /* #ifdef CONFIG_RCU_USER_QS */
 #if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU)
 /*
@@ -873,6 +817,29 @@ static void record_gp_stall_check_time(struct rcu_state *rsp)
        rsp->jiffies_stall = jiffies + jiffies_till_stall_check();
 }
+/*
+ * Dump stacks of all tasks running on stalled CPUs.  This is a fallback
+ * for architectures that do not implement trigger_all_cpu_backtrace().
+ * The NMI-triggered stack traces are more accurate because they are
+ * printed by the target CPU.
+ */
+static void rcu_dump_cpu_stacks(struct rcu_state *rsp)
+{
+        int cpu;
+        unsigned long flags;
+        struct rcu_node *rnp;
+        rcu_for_each_leaf_node(rsp, rnp) {
+                raw_spin_lock_irqsave(&rnp->lock, flags);
+                if (rnp->qsmask != 0) {
+                        for (cpu = 0; cpu <= rnp->grphi - rnp->grplo; cpu++)
+                                if (rnp->qsmask & (1UL << cpu))
+                                        dump_cpu_task(rnp->grplo + cpu);
+                }
+                raw_spin_unlock_irqrestore(&rnp->lock, flags);
+        }
+}
 static void print_other_cpu_stall(struct rcu_state *rsp)
 {
        int cpu;
@@ -880,6 +847,7 @@ static void print_other_cpu_stall(struct rcu_state *rsp)
        unsigned long flags;
        int ndetected = 0;
        struct rcu_node *rnp = rcu_get_root(rsp);
+        long totqlen = 0;
        /* Only let one CPU complain about others per time interval. */
@@ -924,12 +892,15 @@ static void print_other_cpu_stall(struct rcu_state *rsp)
        raw_spin_unlock_irqrestore(&rnp->lock, flags);
        print_cpu_stall_info_end();
-        printk(KERN_CONT "(detected by %d, t=%ld jiffies)\n",
+        for_each_possible_cpu(cpu)
-               smp_processor_id(), (long)(jiffies - rsp->gp_start));
+                totqlen += per_cpu_ptr(rsp->rda, cpu)->qlen;
+        pr_cont("(detected by %d, t=%ld jiffies, g=%lu, c=%lu, q=%lu)\n",
+               smp_processor_id(), (long)(jiffies - rsp->gp_start),
+               rsp->gpnum, rsp->completed, totqlen);
        if (ndetected == 0)
                printk(KERN_ERR "INFO: Stall ended before state dump start\n");
        else if (!trigger_all_cpu_backtrace())
-                dump_stack();
+                rcu_dump_cpu_stacks(rsp);
        /* Complain about tasks blocking the grace period. */
@@ -940,8 +911,10 @@ static void print_other_cpu_stall(struct rcu_state *rsp)
 static void print_cpu_stall(struct rcu_state *rsp)
 {
+        int cpu;
        unsigned long flags;
        struct rcu_node *rnp = rcu_get_root(rsp);
+        long totqlen = 0;
        /*
         * OK, time to rat on ourselves...
@@ -952,7 +925,10 @@ static void print_cpu_stall(struct rcu_state *rsp)
        print_cpu_stall_info_begin();
        print_cpu_stall_info(rsp, smp_processor_id());
        print_cpu_stall_info_end();
-        printk(KERN_CONT " (t=%lu jiffies)\n", jiffies - rsp->gp_start);
+        for_each_possible_cpu(cpu)
+                totqlen += per_cpu_ptr(rsp->rda, cpu)->qlen;
+        pr_cont(" (t=%lu jiffies g=%lu c=%lu q=%lu)\n",
+                jiffies - rsp->gp_start, rsp->gpnum, rsp->completed, totqlen);
        if (!trigger_all_cpu_backtrace())
                dump_stack();
@@ -1091,6 +1067,7 @@ static void init_callback_list(struct rcu_data *rdp)
        rdp->nxtlist = NULL;
        for (i = 0; i < RCU_NEXT_SIZE; i++)
                rdp->nxttail[i] = &rdp->nxtlist;
+        init_nocb_callback_list(rdp);
 }
 /*
@@ -1404,15 +1381,37 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
            !cpu_needs_another_gp(rsp, rdp)) {
                /*
                 * Either we have not yet spawned the grace-period
-                 * task or this CPU does not need another grace period.
+                 * task, this CPU does not need another grace period,
+                 * or a grace period is already in progress.
                 * Either way, don't start a new grace period.
                 */
                raw_spin_unlock_irqrestore(&rnp->lock, flags);
                return;
        }
+        /*
+         * Because there is no grace period in progress right now,
+         * any callbacks we have up to this point will be satisfied
+         * by the next grace period.  So promote all callbacks to be
+         * handled after the end of the next grace period.  If the
+         * CPU is not yet aware of the end of the previous grace period,
+         * we need to allow for the callback advancement that will
+         * occur when it does become aware.  Deadlock prevents us from
+         * making it aware at this point: We cannot acquire a leaf
+         * rcu_node ->lock while holding the root rcu_node ->lock.
+         */
+        rdp->nxttail[RCU_NEXT_READY_TAIL] = rdp->nxttail[RCU_NEXT_TAIL];
+        if (rdp->completed == rsp->completed)
+                rdp->nxttail[RCU_WAIT_TAIL] = rdp->nxttail[RCU_NEXT_TAIL];
        rsp->gp_flags = RCU_GP_FLAG_INIT;
-        raw_spin_unlock_irqrestore(&rnp->lock, flags);
+        raw_spin_unlock(&rnp->lock); /* Interrupts remain disabled. */
+        /* Ensure that CPU is aware of completion of last grace period. */
+        rcu_process_gp_end(rsp, rdp);
+        local_irq_restore(flags);
+        /* Wake up rcu_gp_kthread() to start the grace period. */
        wake_up(&rsp->gp_wq);
 }
@@ -1573,16 +1572,20 @@ rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp)
 /*
 * Send the specified CPU's RCU callbacks to the orphanage.  The
 * specified CPU must be offline, and the caller must hold the
- * ->onofflock.
+ * ->orphan_lock.
 */
 static void
 rcu_send_cbs_to_orphanage(int cpu, struct rcu_state *rsp,
                          struct rcu_node *rnp, struct rcu_data *rdp)
 {
+        /* No-CBs CPUs do not have orphanable callbacks. */
+        if (is_nocb_cpu(rdp->cpu))
+                return;
        /*
         * Orphan the callbacks.  First adjust the counts.  This is safe
-         * because ->onofflock excludes _rcu_barrier()'s adoption of
+         * because _rcu_barrier() excludes CPU-hotplug operations, so it
-         * the callbacks, thus no memory barrier is required.
+         * cannot be running now.  Thus no memory barrier is required.
         */
        if (rdp->nxtlist != NULL) {
                rsp->qlen_lazy += rdp->qlen_lazy;
@@ -1623,13 +1626,17 @@ rcu_send_cbs_to_orphanage(int cpu, struct rcu_state *rsp,
 /*
 * Adopt the RCU callbacks from the specified rcu_state structure's
- * orphanage.  The caller must hold the ->onofflock.
+ * orphanage.  The caller must hold the ->orphan_lock.
 */
 static void rcu_adopt_orphan_cbs(struct rcu_state *rsp)
 {
        int i;
        struct rcu_data *rdp = __this_cpu_ptr(rsp->rda);
+        /* No-CBs CPUs are handled specially. */
+        if (rcu_nocb_adopt_orphan_cbs(rsp, rdp))
+                return;
        /* Do the accounting first. */
        rdp->qlen_lazy += rsp->qlen_lazy;
        rdp->qlen += rsp->qlen;
@@ -1702,7 +1709,7 @@ static void rcu_cleanup_dead_cpu(int cpu, struct rcu_state *rsp)
        /* Exclude any attempts to start a new grace period. */
        mutex_lock(&rsp->onoff_mutex);
-        raw_spin_lock_irqsave(&rsp->onofflock, flags);
+        raw_spin_lock_irqsave(&rsp->orphan_lock, flags);
        /* Orphan the dead CPU's callbacks, and adopt them if appropriate. */
        rcu_send_cbs_to_orphanage(cpu, rsp, rnp, rdp);
@@ -1729,10 +1736,10 @@ static void rcu_cleanup_dead_cpu(int cpu, struct rcu_state *rsp)
        /*
         * We still hold the leaf rcu_node structure lock here, and
         * irqs are still disabled.  The reason for this subterfuge is
-         * because invoking rcu_report_unblock_qs_rnp() with ->onofflock
+         * because invoking rcu_report_unblock_qs_rnp() with ->orphan_lock
         * held leads to deadlock.
         */
-        raw_spin_unlock(&rsp->onofflock); /* irqs remain disabled. */
+        raw_spin_unlock(&rsp->orphan_lock); /* irqs remain disabled. */
        rnp = rdp->mynode;
        if (need_report & RCU_OFL_TASKS_NORM_GP)
                rcu_report_unblock_qs_rnp(rnp, flags);
@@ -1769,7 +1776,8 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp)
 {
        unsigned long flags;
        struct rcu_head *next, *list, **tail;
-        int bl, count, count_lazy, i;
+        long bl, count, count_lazy;
+        int i;
        /* If no callbacks are ready, just return.*/
        if (!cpu_has_callbacks_ready_to_invoke(rdp)) {
@@ -2107,9 +2115,15 @@ static void __call_rcu_core(struct rcu_state *rsp, struct rcu_data *rdp,
        }
 }
+/*
+ * Helper function for call_rcu() and friends.  The cpu argument will
+ * normally be -1, indicating "currently running CPU".  It may specify
+ * a CPU only if that CPU is a no-CBs CPU.  Currently, only _rcu_barrier()
+ * is expected to specify a CPU.
+ */
 static void
 __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
-           struct rcu_state *rsp, bool lazy)
+           struct rcu_state *rsp, int cpu, bool lazy)
 {
        unsigned long flags;
        struct rcu_data *rdp;
@@ -2129,9 +2143,14 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
        rdp = this_cpu_ptr(rsp->rda);
        /* Add the callback to our list. */
-        if (unlikely(rdp->nxttail[RCU_NEXT_TAIL] == NULL)) {
+        if (unlikely(rdp->nxttail[RCU_NEXT_TAIL] == NULL) || cpu != -1) {
+                int offline;
+                if (cpu != -1)
+                        rdp = per_cpu_ptr(rsp->rda, cpu);
+                offline = !__call_rcu_nocb(rdp, head, lazy);
+                WARN_ON_ONCE(offline);
                /* _call_rcu() is illegal on offline CPU; leak the callback. */
-                WARN_ON_ONCE(1);
                local_irq_restore(flags);
                return;
        }
@@ -2160,7 +2179,7 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
 */
 void call_rcu_sched(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
 {
-        __call_rcu(head, func, &rcu_sched_state, 0);
+        __call_rcu(head, func, &rcu_sched_state, -1, 0);
 }
 EXPORT_SYMBOL_GPL(call_rcu_sched);
@@ -2169,7 +2188,7 @@ EXPORT_SYMBOL_GPL(call_rcu_sched);
 */
 void call_rcu_bh(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
 {
-        __call_rcu(head, func, &rcu_bh_state, 0);
+        __call_rcu(head, func, &rcu_bh_state, -1, 0);
 }
 EXPORT_SYMBOL_GPL(call_rcu_bh);
@@ -2205,10 +2224,28 @@ static inline int rcu_blocking_is_gp(void)
 * rcu_read_lock_sched().
 *
 * This means that all preempt_disable code sequences, including NMI and
- * hardware-interrupt handlers, in progress on entry will have completed
+ * non-threaded hardware-interrupt handlers, in progress on entry will
- * before this primitive returns.  However, this does not guarantee that
+ * have completed before this primitive returns.  However, this does not
- * softirq handlers will have completed, since in some kernels, these
+ * guarantee that softirq handlers will have completed, since in some
- * handlers can run in process context, and can block.
+ * kernels, these handlers can run in process context, and can block.
+ *
+ * Note that this guarantee implies further memory-ordering guarantees.
+ * On systems with more than one CPU, when synchronize_sched() returns,
+ * each CPU is guaranteed to have executed a full memory barrier since the
+ * end of its last RCU-sched read-side critical section whose beginning
+ * preceded the call to synchronize_sched().  In addition, each CPU having
+ * an RCU read-side critical section that extends beyond the return from
+ * synchronize_sched() is guaranteed to have executed a full memory barrier
+ * after the beginning of synchronize_sched() and before the beginning of
+ * that RCU read-side critical section.  Note that these guarantees include
+ * CPUs that are offline, idle, or executing in user mode, as well as CPUs
+ * that are executing in the kernel.
+ *
+ * Furthermore, if CPU A invoked synchronize_sched(), which returned
+ * to its caller on CPU B, then both CPU A and CPU B are guaranteed
+ * to have executed a full memory barrier during the execution of
+ * synchronize_sched() -- even if CPU A and CPU B are the same CPU (but
+ * again only if the system has more than one CPU).
 *
 * This primitive provides the guarantees made by the (now removed)
 * synchronize_kernel() API.  In contrast, synchronize_rcu() only
@@ -2224,7 +2261,10 @@ void synchronize_sched(void)
                           "Illegal synchronize_sched() in RCU-sched read-side critical section");
        if (rcu_blocking_is_gp())
                return;
-        wait_rcu_gp(call_rcu_sched);
+        if (rcu_expedited)
+                synchronize_sched_expedited();
+        else
+                wait_rcu_gp(call_rcu_sched);
 }
 EXPORT_SYMBOL_GPL(synchronize_sched);
@@ -2236,6 +2276,9 @@ EXPORT_SYMBOL_GPL(synchronize_sched);
 * read-side critical sections have completed.  RCU read-side critical
 * sections are delimited by rcu_read_lock_bh() and rcu_read_unlock_bh(),
 * and may be nested.
+ *
+ * See the description of synchronize_sched() for more detailed information
+ * on memory ordering guarantees.
 */
 void synchronize_rcu_bh(void)
 {
@@ -2245,13 +2288,13 @@ void synchronize_rcu_bh(void)
                           "Illegal synchronize_rcu_bh() in RCU-bh read-side critical section");
        if (rcu_blocking_is_gp())
                return;
-        wait_rcu_gp(call_rcu_bh);
+        if (rcu_expedited)
+                synchronize_rcu_bh_expedited();
+        else
+                wait_rcu_gp(call_rcu_bh);
 }
 EXPORT_SYMBOL_GPL(synchronize_rcu_bh);
-static atomic_t sync_sched_expedited_started = ATOMIC_INIT(0);
-static atomic_t sync_sched_expedited_done = ATOMIC_INIT(0);
 static int synchronize_sched_expedited_cpu_stop(void *data)
 {
        /*
@@ -2308,10 +2351,32 @@ static int synchronize_sched_expedited_cpu_stop(void *data)
 */
 void synchronize_sched_expedited(void)
 {
-        int firstsnap, s, snap, trycount = 0;
+        long firstsnap, s, snap;
+        int trycount = 0;
+        struct rcu_state *rsp = &rcu_sched_state;
+        /*
+         * If we are in danger of counter wrap, just do synchronize_sched().
+         * By allowing sync_sched_expedited_started to advance no more than
+         * ULONG_MAX/8 ahead of sync_sched_expedited_done, we are ensuring
+         * that more than 3.5 billion CPUs would be required to force a
+         * counter wrap on a 32-bit system.  Quite a few more CPUs would of
+         * course be required on a 64-bit system.
+         */
+        if (ULONG_CMP_GE((ulong)atomic_long_read(&rsp->expedited_start),
+                         (ulong)atomic_long_read(&rsp->expedited_done) +
+                         ULONG_MAX / 8)) {
+                synchronize_sched();
+                atomic_long_inc(&rsp->expedited_wrap);
+                return;
+        }
-        /* Note that atomic_inc_return() implies full memory barrier. */
+        /*
-        firstsnap = snap = atomic_inc_return(&sync_sched_expedited_started);
+         * Take a ticket.  Note that atomic_inc_return() implies a
+         * full memory barrier.
+         */
+        snap = atomic_long_inc_return(&rsp->expedited_start);
+        firstsnap = snap;
        get_online_cpus();
        WARN_ON_ONCE(cpu_is_offline(raw_smp_processor_id()));
@@ -2323,48 +2388,65 @@ void synchronize_sched_expedited(void)
                             synchronize_sched_expedited_cpu_stop,
                             NULL) == -EAGAIN) {
                put_online_cpus();
+                atomic_long_inc(&rsp->expedited_tryfail);
+                /* Check to see if someone else did our work for us. */
+                s = atomic_long_read(&rsp->expedited_done);
+                if (ULONG_CMP_GE((ulong)s, (ulong)firstsnap)) {
+                        /* ensure test happens before caller kfree */
+                        smp_mb__before_atomic_inc(); /* ^^^ */
+                        atomic_long_inc(&rsp->expedited_workdone1);
+                        return;
+                }
                /* No joy, try again later.  Or just synchronize_sched(). */
                if (trycount++ < 10) {
                        udelay(trycount * num_online_cpus());
                } else {
-                        synchronize_sched();
+                        wait_rcu_gp(call_rcu_sched);
+                        atomic_long_inc(&rsp->expedited_normal);
                        return;
                }
-                /* Check to see if someone else did our work for us. */
+                /* Recheck to see if someone else did our work for us. */
-                s = atomic_read(&sync_sched_expedited_done);
+                s = atomic_long_read(&rsp->expedited_done);
-                if (UINT_CMP_GE((unsigned)s, (unsigned)firstsnap)) {
+                if (ULONG_CMP_GE((ulong)s, (ulong)firstsnap)) {
-                        smp_mb(); /* ensure test happens before caller kfree */
+                        /* ensure test happens before caller kfree */
+                        smp_mb__before_atomic_inc(); /* ^^^ */
+                        atomic_long_inc(&rsp->expedited_workdone2);
                        return;
                }
                /*
                 * Refetching sync_sched_expedited_started allows later
-                 * callers to piggyback on our grace period.  We subtract
+                 * callers to piggyback on our grace period.  We retry
-                 * 1 to get the same token that the last incrementer got.
+                 * after they started, so our grace period works for them,
-                 * We retry after they started, so our grace period works
+                 * and they started after our first try, so their grace
-                 * for them, and they started after our first try, so their
+                 * period works for us.
-                 * grace period works for us.
                 */
                get_online_cpus();
-                snap = atomic_read(&sync_sched_expedited_started);
+                snap = atomic_long_read(&rsp->expedited_start);
                smp_mb(); /* ensure read is before try_stop_cpus(). */
        }
+        atomic_long_inc(&rsp->expedited_stoppedcpus);
        /*
         * Everyone up to our most recent fetch is covered by our grace
         * period.  Update the counter, but only if our work is still
         * relevant -- which it won't be if someone who started later
-         * than we did beat us to the punch.
+         * than we did already did their update.
         */
        do {
-                s = atomic_read(&sync_sched_expedited_done);
+                atomic_long_inc(&rsp->expedited_done_tries);
-                if (UINT_CMP_GE((unsigned)s, (unsigned)snap)) {
+                s = atomic_long_read(&rsp->expedited_done);
-                        smp_mb(); /* ensure test happens before caller kfree */
+                if (ULONG_CMP_GE((ulong)s, (ulong)snap)) {
+                        /* ensure test happens before caller kfree */
+                        smp_mb__before_atomic_inc(); /* ^^^ */
+                        atomic_long_inc(&rsp->expedited_done_lost);
                        break;
                }
-        } while (atomic_cmpxchg(&sync_sched_expedited_done, s, snap) != s);
+        } while (atomic_long_cmpxchg(&rsp->expedited_done, s, snap) != s);
+        atomic_long_inc(&rsp->expedited_done_exit);
        put_online_cpus();
 }
@@ -2558,9 +2640,17 @@ static void _rcu_barrier(struct rcu_state *rsp)
         * When that callback is invoked, we will know that all of the
         * corresponding CPU's preceding callbacks have been invoked.
         */
-        for_each_online_cpu(cpu) {
+        for_each_possible_cpu(cpu) {
+                if (!cpu_online(cpu) && !is_nocb_cpu(cpu))
+                        continue;
                rdp = per_cpu_ptr(rsp->rda, cpu);
-                if (ACCESS_ONCE(rdp->qlen)) {
+                if (is_nocb_cpu(cpu)) {
+                        _rcu_barrier_trace(rsp, "OnlineNoCB", cpu,
+                                           rsp->n_barrier_done);
+                        atomic_inc(&rsp->barrier_cpu_count);
+                        __call_rcu(&rdp->barrier_head, rcu_barrier_callback,
+                                   rsp, cpu, 0);
+                } else if (ACCESS_ONCE(rdp->qlen)) {
                        _rcu_barrier_trace(rsp, "OnlineQ", cpu,
                                           rsp->n_barrier_done);
                        smp_call_function_single(cpu, rcu_barrier_func, rsp, 1);
@@ -2634,6 +2724,7 @@ rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp)
 #endif
        rdp->cpu = cpu;
        rdp->rsp = rsp;
+        rcu_boot_init_nocb_percpu_data(rdp);
        raw_spin_unlock_irqrestore(&rnp->lock, flags);
 }
@@ -2715,6 +2806,7 @@ static int __cpuinit rcu_cpu_notify(struct notifier_block *self,
        struct rcu_data *rdp = per_cpu_ptr(rcu_state->rda, cpu);
        struct rcu_node *rnp = rdp->mynode;
        struct rcu_state *rsp;
+        int ret = NOTIFY_OK;
        trace_rcu_utilization("Start CPU hotplug");
        switch (action) {
@@ -2728,7 +2820,10 @@ static int __cpuinit rcu_cpu_notify(struct notifier_block *self,
                rcu_boost_kthread_setaffinity(rnp, -1);
                break;
        case CPU_DOWN_PREPARE:
-                rcu_boost_kthread_setaffinity(rnp, cpu);
+                if (nocb_cpu_expendable(cpu))
+                        rcu_boost_kthread_setaffinity(rnp, cpu);
+                else
+                        ret = NOTIFY_BAD;
                break;
        case CPU_DYING:
        case CPU_DYING_FROZEN:
@@ -2752,7 +2847,7 @@ static int __cpuinit rcu_cpu_notify(struct notifier_block *self,
                break;
        }
        trace_rcu_utilization("End CPU hotplug");
-        return NOTIFY_OK;
+        return ret;
 }
 /*
@@ -2772,6 +2867,7 @@ static int __init rcu_spawn_gp_kthread(void)
                raw_spin_lock_irqsave(&rnp->lock, flags);
                rsp->gp_kthread = t;
                raw_spin_unlock_irqrestore(&rnp->lock, flags);
+                rcu_spawn_nocb_kthreads(rsp);
        }
        return 0;
 }
@@ -2967,6 +3063,7 @@ void __init rcu_init(void)
        rcu_init_one(&rcu_sched_state, &rcu_sched_data);
        rcu_init_one(&rcu_bh_state, &rcu_bh_data);
        __rcu_init_preempt();
+        rcu_init_nocb();
         open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
        /*
diff --git a/kernel/rcutree.h b/kernel/rcutree.h
index a240f032848e..4b69291b093d 100644
--- a/kernel/rcutree.h
+++ b/kernel/rcutree.h
@@ -287,6 +287,7 @@ struct rcu_data {
        long            qlen_last_fqs_check;
                                        /* qlen at last check for QS forcing */
        unsigned long   n_cbs_invoked;  /* count of RCU cbs invoked. */
+        unsigned long   n_nocbs_invoked; /* count of no-CBs RCU cbs invoked. */
        unsigned long   n_cbs_orphaned; /* RCU cbs orphaned by dying CPU */
        unsigned long   n_cbs_adopted;  /* RCU cbs adopted from dying CPU */
        unsigned long   n_force_qs_snap;
@@ -317,6 +318,18 @@ struct rcu_data {
        struct rcu_head oom_head;
 #endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */
+        /* 7) Callback offloading. */
+#ifdef CONFIG_RCU_NOCB_CPU
+        struct rcu_head *nocb_head;     /* CBs waiting for kthread. */
+        struct rcu_head **nocb_tail;
+        atomic_long_t nocb_q_count;     /* # CBs waiting for kthread */
+        atomic_long_t nocb_q_count_lazy; /*  (approximate). */
+        int nocb_p_count;               /* # CBs being invoked by kthread */
+        int nocb_p_count_lazy;          /*  (approximate). */
+        wait_queue_head_t nocb_wq;      /* For nocb kthreads to sleep on. */
+        struct task_struct *nocb_kthread;
+#endif /* #ifdef CONFIG_RCU_NOCB_CPU */
        int cpu;
        struct rcu_state *rsp;
 };
@@ -369,6 +382,12 @@ struct rcu_state {
        struct rcu_data __percpu *rda;          /* pointer of percu rcu_data. */
        void (*call)(struct rcu_head *head,     /* call_rcu() flavor. */
                     void (*func)(struct rcu_head *head));
+#ifdef CONFIG_RCU_NOCB_CPU
+        void (*call_remote)(struct rcu_head *head,
+                     void (*func)(struct rcu_head *head));
+                                                /* call_rcu() flavor, but for */
+                                                /*  placing on remote CPU. */
+#endif /* #ifdef CONFIG_RCU_NOCB_CPU */
        /* The following fields are guarded by the root rcu_node's lock. */
@@ -383,9 +402,8 @@ struct rcu_state {
        /* End of fields guarded by root rcu_node's lock. */
-        raw_spinlock_t onofflock ____cacheline_internodealigned_in_smp;
+        raw_spinlock_t orphan_lock ____cacheline_internodealigned_in_smp;
-                                                /* exclude on/offline and */
+                                                /* Protect following fields. */
-                                                /*  starting new GP. */
        struct rcu_head *orphan_nxtlist;        /* Orphaned callbacks that */
                                                /*  need a grace period. */
        struct rcu_head **orphan_nxttail;       /* Tail of above. */
@@ -394,7 +412,7 @@ struct rcu_state {
        struct rcu_head **orphan_donetail;      /* Tail of above. */
        long qlen_lazy;                         /* Number of lazy callbacks. */
        long qlen;                              /* Total number of callbacks. */
-        /* End of fields guarded by onofflock. */
+        /* End of fields guarded by orphan_lock. */
        struct mutex onoff_mutex;               /* Coordinate hotplug & GPs. */
@@ -405,6 +423,18 @@ struct rcu_state {
                                                /*  _rcu_barrier(). */
        /* End of fields guarded by barrier_mutex. */
+        atomic_long_t expedited_start;          /* Starting ticket. */
+        atomic_long_t expedited_done;           /* Done ticket. */
+        atomic_long_t expedited_wrap;           /* # near-wrap incidents. */
+        atomic_long_t expedited_tryfail;        /* # acquisition failures. */
+        atomic_long_t expedited_workdone1;      /* # done by others #1. */
+        atomic_long_t expedited_workdone2;      /* # done by others #2. */
+        atomic_long_t expedited_normal;         /* # fallbacks to normal. */
+        atomic_long_t expedited_stoppedcpus;    /* # successful stop_cpus. */
+        atomic_long_t expedited_done_tries;     /* # tries to update _done. */
+        atomic_long_t expedited_done_lost;      /* # times beaten to _done. */
+        atomic_long_t expedited_done_exit;      /* # times exited _done loop. */
        unsigned long jiffies_force_qs;         /* Time at which to invoke */
                                                /*  force_quiescent_state(). */
        unsigned long n_force_qs;               /* Number of calls to */
@@ -428,6 +458,8 @@ struct rcu_state {
 #define RCU_GP_FLAG_FQS  0x2    /* Need grace-period quiescent-state forcing. */
 extern struct list_head rcu_struct_flavors;
+/* Sequence through rcu_state structures for each RCU flavor. */
 #define for_each_rcu_flavor(rsp) \
        list_for_each_entry((rsp), &rcu_struct_flavors, flavors)
@@ -504,5 +536,32 @@ static void print_cpu_stall_info(struct rcu_state *rsp, int cpu);
 static void print_cpu_stall_info_end(void);
 static void zero_cpu_stall_ticks(struct rcu_data *rdp);
 static void increment_cpu_stall_ticks(void);
+static bool is_nocb_cpu(int cpu);
+static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
+                            bool lazy);
+static bool rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
+                                      struct rcu_data *rdp);
+static bool nocb_cpu_expendable(int cpu);
+static void rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp);
+static void rcu_spawn_nocb_kthreads(struct rcu_state *rsp);
+static void init_nocb_callback_list(struct rcu_data *rdp);
+static void __init rcu_init_nocb(void);
 #endif /* #ifndef RCU_TREE_NONCORE */
+#ifdef CONFIG_RCU_TRACE
+#ifdef CONFIG_RCU_NOCB_CPU
+/* Sum up queue lengths for tracing. */
+static inline void rcu_nocb_q_lengths(struct rcu_data *rdp, long *ql, long *qll)
+{
+        *ql = atomic_long_read(&rdp->nocb_q_count) + rdp->nocb_p_count;
+        *qll = atomic_long_read(&rdp->nocb_q_count_lazy) + rdp->nocb_p_count_lazy;
+}
+#else /* #ifdef CONFIG_RCU_NOCB_CPU */
+static inline void rcu_nocb_q_lengths(struct rcu_data *rdp, long *ql, long *qll)
+{
+        *ql = 0;
+        *qll = 0;
+}
+#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
+#endif /* #ifdef CONFIG_RCU_TRACE */
diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h
index f92115488187..f6e5ec2932b4 100644
--- a/kernel/rcutree_plugin.h
+++ b/kernel/rcutree_plugin.h
@@ -25,6 +25,7 @@
 */
 #include <linux/delay.h>
+#include <linux/gfp.h>
 #include <linux/oom.h>
 #include <linux/smpboot.h>
@@ -36,6 +37,14 @@
 #define RCU_BOOST_PRIO RCU_KTHREAD_PRIO
 #endif
+#ifdef CONFIG_RCU_NOCB_CPU
+static cpumask_var_t rcu_nocb_mask; /* CPUs to have callbacks offloaded. */
+static bool have_rcu_nocb_mask;     /* Was rcu_nocb_mask allocated? */
+static bool rcu_nocb_poll;          /* Offload kthread are to poll. */
+module_param(rcu_nocb_poll, bool, 0444);
+static char __initdata nocb_buf[NR_CPUS * 5];
+#endif /* #ifdef CONFIG_RCU_NOCB_CPU */
 /*
 * Check the RCU kernel configuration parameters and print informative
 * messages about anything out of the ordinary.  If you like #ifdef, you
@@ -76,6 +85,18 @@ static void __init rcu_bootup_announce_oddness(void)
                printk(KERN_INFO "\tExperimental boot-time adjustment of leaf fanout to %d.\n", rcu_fanout_leaf);
        if (nr_cpu_ids != NR_CPUS)
                printk(KERN_INFO "\tRCU restricting CPUs from NR_CPUS=%d to nr_cpu_ids=%d.\n", NR_CPUS, nr_cpu_ids);
+#ifdef CONFIG_RCU_NOCB_CPU
+        if (have_rcu_nocb_mask) {
+                if (cpumask_test_cpu(0, rcu_nocb_mask)) {
+                        cpumask_clear_cpu(0, rcu_nocb_mask);
+                        pr_info("\tCPU 0: illegal no-CBs CPU (cleared).\n");
+                }
+                cpulist_scnprintf(nocb_buf, sizeof(nocb_buf), rcu_nocb_mask);
+                pr_info("\tExperimental no-CBs CPUs: %s.\n", nocb_buf);
+                if (rcu_nocb_poll)
+                        pr_info("\tExperimental polled no-CBs CPUs.\n");
+        }
+#endif /* #ifdef CONFIG_RCU_NOCB_CPU */
 }
 #ifdef CONFIG_TREE_PREEMPT_RCU
@@ -642,7 +663,7 @@ static void rcu_preempt_do_callbacks(void)
 */
 void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
 {
-        __call_rcu(head, func, &rcu_preempt_state, 0);
+        __call_rcu(head, func, &rcu_preempt_state, -1, 0);
 }
 EXPORT_SYMBOL_GPL(call_rcu);
@@ -656,7 +677,7 @@ EXPORT_SYMBOL_GPL(call_rcu);
 void kfree_call_rcu(struct rcu_head *head,
                    void (*func)(struct rcu_head *rcu))
 {
-        __call_rcu(head, func, &rcu_preempt_state, 1);
+        __call_rcu(head, func, &rcu_preempt_state, -1, 1);
 }
 EXPORT_SYMBOL_GPL(kfree_call_rcu);
@@ -670,6 +691,9 @@ EXPORT_SYMBOL_GPL(kfree_call_rcu);
 * concurrently with new RCU read-side critical sections that began while
 * synchronize_rcu() was waiting.  RCU read-side critical sections are
 * delimited by rcu_read_lock() and rcu_read_unlock(), and may be nested.
+ *
+ * See the description of synchronize_sched() for more detailed information
+ * on memory ordering guarantees.
 */
 void synchronize_rcu(void)
 {
@@ -679,7 +703,10 @@ void synchronize_rcu(void)
                           "Illegal synchronize_rcu() in RCU read-side critical section");
        if (!rcu_scheduler_active)
                return;
-        wait_rcu_gp(call_rcu);
+        if (rcu_expedited)
+                synchronize_rcu_expedited();
+        else
+                wait_rcu_gp(call_rcu);
 }
 EXPORT_SYMBOL_GPL(synchronize_rcu);
@@ -757,7 +784,8 @@ static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
 * grace period for the specified rcu_node structure.  If there are no such
 * tasks, report it up the rcu_node hierarchy.
 *
- * Caller must hold sync_rcu_preempt_exp_mutex and rsp->onofflock.
+ * Caller must hold sync_rcu_preempt_exp_mutex and must exclude
+ * CPU hotplug operations.
 */
 static void
 sync_rcu_preempt_exp_init(struct rcu_state *rsp, struct rcu_node *rnp)
@@ -831,7 +859,7 @@ void synchronize_rcu_expedited(void)
                        udelay(trycount * num_online_cpus());
                } else {
                        put_online_cpus();
-                        synchronize_rcu();
+                        wait_rcu_gp(call_rcu);
                        return;
                }
        }
@@ -875,6 +903,11 @@ EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
 /**
 * rcu_barrier - Wait until all in-flight call_rcu() callbacks complete.
+ *
+ * Note that this primitive does not necessarily wait for an RCU grace period
+ * to complete.  For example, if there are no RCU callbacks queued anywhere
+ * in the system, then rcu_barrier() is within its rights to return
+ * immediately, without waiting for anything, much less an RCU grace period.
 */
 void rcu_barrier(void)
 {
@@ -1013,7 +1046,7 @@ static void rcu_preempt_check_callbacks(int cpu)
 void kfree_call_rcu(struct rcu_head *head,
                    void (*func)(struct rcu_head *rcu))
 {
-        __call_rcu(head, func, &rcu_sched_state, 1);
+        __call_rcu(head, func, &rcu_sched_state, -1, 1);
 }
 EXPORT_SYMBOL_GPL(kfree_call_rcu);
@@ -2092,3 +2125,373 @@ static void increment_cpu_stall_ticks(void)
 }
 #endif /* #else #ifdef CONFIG_RCU_CPU_STALL_INFO */
+#ifdef CONFIG_RCU_NOCB_CPU
+/*
+ * Offload callback processing from the boot-time-specified set of CPUs
+ * specified by rcu_nocb_mask.  For each CPU in the set, there is a
+ * kthread created that pulls the callbacks from the corresponding CPU,
+ * waits for a grace period to elapse, and invokes the callbacks.
+ * The no-CBs CPUs do a wake_up() on their kthread when they insert
+ * a callback into any empty list, unless the rcu_nocb_poll boot parameter
+ * has been specified, in which case each kthread actively polls its
+ * CPU.  (Which isn't so great for energy efficiency, but which does
+ * reduce RCU's overhead on that CPU.)
+ *
+ * This is intended to be used in conjunction with Frederic Weisbecker's
+ * adaptive-idle work, which would seriously reduce OS jitter on CPUs
+ * running CPU-bound user-mode computations.
+ *
+ * Offloading of callback processing could also in theory be used as
+ * an energy-efficiency measure because CPUs with no RCU callbacks
+ * queued are more aggressive about entering dyntick-idle mode.
+ */
+/* Parse the boot-time rcu_nocb_mask CPU list from the kernel parameters. */
+static int __init rcu_nocb_setup(char *str)
+{
+        alloc_bootmem_cpumask_var(&rcu_nocb_mask);
+        have_rcu_nocb_mask = true;
+        cpulist_parse(str, rcu_nocb_mask);
+        return 1;
+}
+__setup("rcu_nocbs=", rcu_nocb_setup);
+/* Is the specified CPU a no-CPUs CPU? */
+static bool is_nocb_cpu(int cpu)
+{
+        if (have_rcu_nocb_mask)
+                return cpumask_test_cpu(cpu, rcu_nocb_mask);
+        return false;
+}
+/*
+ * Enqueue the specified string of rcu_head structures onto the specified
+ * CPU's no-CBs lists.  The CPU is specified by rdp, the head of the
+ * string by rhp, and the tail of the string by rhtp.  The non-lazy/lazy
+ * counts are supplied by rhcount and rhcount_lazy.
+ *
+ * If warranted, also wake up the kthread servicing this CPUs queues.
+ */
+static void __call_rcu_nocb_enqueue(struct rcu_data *rdp,
+                                    struct rcu_head *rhp,
+                                    struct rcu_head **rhtp,
+                                    int rhcount, int rhcount_lazy)
+{
+        int len;
+        struct rcu_head **old_rhpp;
+        struct task_struct *t;
+        /* Enqueue the callback on the nocb list and update counts. */
+        old_rhpp = xchg(&rdp->nocb_tail, rhtp);
+        ACCESS_ONCE(*old_rhpp) = rhp;
+        atomic_long_add(rhcount, &rdp->nocb_q_count);
+        atomic_long_add(rhcount_lazy, &rdp->nocb_q_count_lazy);
+        /* If we are not being polled and there is a kthread, awaken it ... */
+        t = ACCESS_ONCE(rdp->nocb_kthread);
+        if (rcu_nocb_poll | !t)
+                return;
+        len = atomic_long_read(&rdp->nocb_q_count);
+        if (old_rhpp == &rdp->nocb_head) {
+                wake_up(&rdp->nocb_wq); /* ... only if queue was empty ... */
+                rdp->qlen_last_fqs_check = 0;
+        } else if (len > rdp->qlen_last_fqs_check + qhimark) {
+                wake_up_process(t); /* ... or if many callbacks queued. */
+                rdp->qlen_last_fqs_check = LONG_MAX / 2;
+        }
+        return;
+}
+/*
+ * This is a helper for __call_rcu(), which invokes this when the normal
+ * callback queue is inoperable.  If this is not a no-CBs CPU, this
+ * function returns failure back to __call_rcu(), which can complain
+ * appropriately.
+ *
+ * Otherwise, this function queues the callback where the corresponding
+ * "rcuo" kthread can find it.
+ */
+static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
+                            bool lazy)
+{
+        if (!is_nocb_cpu(rdp->cpu))
+                return 0;
+        __call_rcu_nocb_enqueue(rdp, rhp, &rhp->next, 1, lazy);
+        return 1;
+}
+/*
+ * Adopt orphaned callbacks on a no-CBs CPU, or return 0 if this is
+ * not a no-CBs CPU.
+ */
+static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
+                                                     struct rcu_data *rdp)
+{
+        long ql = rsp->qlen;
+        long qll = rsp->qlen_lazy;
+        /* If this is not a no-CBs CPU, tell the caller to do it the old way. */
+        if (!is_nocb_cpu(smp_processor_id()))
+                return 0;
+        rsp->qlen = 0;
+        rsp->qlen_lazy = 0;
+        /* First, enqueue the donelist, if any.  This preserves CB ordering. */
+        if (rsp->orphan_donelist != NULL) {
+                __call_rcu_nocb_enqueue(rdp, rsp->orphan_donelist,
+                                        rsp->orphan_donetail, ql, qll);
+                ql = qll = 0;
+                rsp->orphan_donelist = NULL;
+                rsp->orphan_donetail = &rsp->orphan_donelist;
+        }
+        if (rsp->orphan_nxtlist != NULL) {
+                __call_rcu_nocb_enqueue(rdp, rsp->orphan_nxtlist,
+                                        rsp->orphan_nxttail, ql, qll);
+                ql = qll = 0;
+                rsp->orphan_nxtlist = NULL;
+                rsp->orphan_nxttail = &rsp->orphan_nxtlist;
+        }
+        return 1;
+}
+/*
+ * There must be at least one non-no-CBs CPU in operation at any given
+ * time, because no-CBs CPUs are not capable of initiating grace periods
+ * independently.  This function therefore complains if the specified
+ * CPU is the last non-no-CBs CPU, allowing the CPU-hotplug system to
+ * avoid offlining the last such CPU.  (Recursion is a wonderful thing,
+ * but you have to have a base case!)
+ */
+static bool nocb_cpu_expendable(int cpu)
+{
+        cpumask_var_t non_nocb_cpus;
+        int ret;
+        /*
+         * If there are no no-CB CPUs or if this CPU is not a no-CB CPU,
+         * then offlining this CPU is harmless.  Let it happen.
+         */
+        if (!have_rcu_nocb_mask || is_nocb_cpu(cpu))
+                return 1;
+        /* If no memory, play it safe and keep the CPU around. */
+        if (!alloc_cpumask_var(&non_nocb_cpus, GFP_NOIO))
+                return 0;
+        cpumask_andnot(non_nocb_cpus, cpu_online_mask, rcu_nocb_mask);
+        cpumask_clear_cpu(cpu, non_nocb_cpus);
+        ret = !cpumask_empty(non_nocb_cpus);
+        free_cpumask_var(non_nocb_cpus);
+        return ret;
+}
+/*
+ * Helper structure for remote registry of RCU callbacks.
+ * This is needed for when a no-CBs CPU needs to start a grace period.
+ * If it just invokes call_rcu(), the resulting callback will be queued,
+ * which can result in deadlock.
+ */
+struct rcu_head_remote {
+        struct rcu_head *rhp;
+        call_rcu_func_t *crf;
+        void (*func)(struct rcu_head *rhp);
+};
+/*
+ * Register a callback as specified by the rcu_head_remote struct.
+ * This function is intended to be invoked via smp_call_function_single().
+ */
+static void call_rcu_local(void *arg)
+{
+        struct rcu_head_remote *rhrp =
+                container_of(arg, struct rcu_head_remote, rhp);
+        rhrp->crf(rhrp->rhp, rhrp->func);
+}
+/*
+ * Set up an rcu_head_remote structure and the invoke call_rcu_local()
+ * on CPU 0 (which is guaranteed to be a non-no-CBs CPU) via
+ * smp_call_function_single().
+ */
+static void invoke_crf_remote(struct rcu_head *rhp,
+                              void (*func)(struct rcu_head *rhp),
+                              call_rcu_func_t crf)
+{
+        struct rcu_head_remote rhr;
+        rhr.rhp = rhp;
+        rhr.crf = crf;
+        rhr.func = func;
+        smp_call_function_single(0, call_rcu_local, &rhr, 1);
+}
+/*
+ * Helper functions to be passed to wait_rcu_gp(), each of which
+ * invokes invoke_crf_remote() to register a callback appropriately.
+ */
+static void __maybe_unused
+call_rcu_preempt_remote(struct rcu_head *rhp,
+                        void (*func)(struct rcu_head *rhp))
+{
+        invoke_crf_remote(rhp, func, call_rcu);
+}
+static void call_rcu_bh_remote(struct rcu_head *rhp,
+                               void (*func)(struct rcu_head *rhp))
+{
+        invoke_crf_remote(rhp, func, call_rcu_bh);
+}
+static void call_rcu_sched_remote(struct rcu_head *rhp,
+                                  void (*func)(struct rcu_head *rhp))
+{
+        invoke_crf_remote(rhp, func, call_rcu_sched);
+}
+/*
+ * Per-rcu_data kthread, but only for no-CBs CPUs.  Each kthread invokes
+ * callbacks queued by the corresponding no-CBs CPU.
+ */
+static int rcu_nocb_kthread(void *arg)
+{
+        int c, cl;
+        struct rcu_head *list;
+        struct rcu_head *next;
+        struct rcu_head **tail;
+        struct rcu_data *rdp = arg;
+        /* Each pass through this loop invokes one batch of callbacks */
+        for (;;) {
+                /* If not polling, wait for next batch of callbacks. */
+                if (!rcu_nocb_poll)
+                        wait_event(rdp->nocb_wq, rdp->nocb_head);
+                list = ACCESS_ONCE(rdp->nocb_head);
+                if (!list) {
+                        schedule_timeout_interruptible(1);
+                        continue;
+                }
+                /*
+                 * Extract queued callbacks, update counts, and wait
+                 * for a grace period to elapse.
+                 */
+                ACCESS_ONCE(rdp->nocb_head) = NULL;
+                tail = xchg(&rdp->nocb_tail, &rdp->nocb_head);
+                c = atomic_long_xchg(&rdp->nocb_q_count, 0);
+                cl = atomic_long_xchg(&rdp->nocb_q_count_lazy, 0);
+                ACCESS_ONCE(rdp->nocb_p_count) += c;
+                ACCESS_ONCE(rdp->nocb_p_count_lazy) += cl;
+                wait_rcu_gp(rdp->rsp->call_remote);
+                /* Each pass through the following loop invokes a callback. */
+                trace_rcu_batch_start(rdp->rsp->name, cl, c, -1);
+                c = cl = 0;
+                while (list) {
+                        next = list->next;
+                        /* Wait for enqueuing to complete, if needed. */
+                        while (next == NULL && &list->next != tail) {
+                                schedule_timeout_interruptible(1);
+                                next = list->next;
+                        }
+                        debug_rcu_head_unqueue(list);
+                        local_bh_disable();
+                        if (__rcu_reclaim(rdp->rsp->name, list))
+                                cl++;
+                        c++;
+                        local_bh_enable();
+                        list = next;
+                }
+                trace_rcu_batch_end(rdp->rsp->name, c, !!list, 0, 0, 1);
+                ACCESS_ONCE(rdp->nocb_p_count) -= c;
+                ACCESS_ONCE(rdp->nocb_p_count_lazy) -= cl;
+                rdp->n_nocbs_invoked += c;
+        }
+        return 0;
+}
+/* Initialize per-rcu_data variables for no-CBs CPUs. */
+static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
+{
+        rdp->nocb_tail = &rdp->nocb_head;
+        init_waitqueue_head(&rdp->nocb_wq);
+}
+/* Create a kthread for each RCU flavor for each no-CBs CPU. */
+static void __init rcu_spawn_nocb_kthreads(struct rcu_state *rsp)
+{
+        int cpu;
+        struct rcu_data *rdp;
+        struct task_struct *t;
+        if (rcu_nocb_mask == NULL)
+                return;
+        for_each_cpu(cpu, rcu_nocb_mask) {
+                rdp = per_cpu_ptr(rsp->rda, cpu);
+                t = kthread_run(rcu_nocb_kthread, rdp, "rcuo%d", cpu);
+                BUG_ON(IS_ERR(t));
+                ACCESS_ONCE(rdp->nocb_kthread) = t;
+        }
+}
+/* Prevent __call_rcu() from enqueuing callbacks on no-CBs CPUs */
+static void init_nocb_callback_list(struct rcu_data *rdp)
+{
+        if (rcu_nocb_mask == NULL ||
+            !cpumask_test_cpu(rdp->cpu, rcu_nocb_mask))
+                return;
+        rdp->nxttail[RCU_NEXT_TAIL] = NULL;
+}
+/* Initialize the ->call_remote fields in the rcu_state structures. */
+static void __init rcu_init_nocb(void)
+{
+#ifdef CONFIG_PREEMPT_RCU
+        rcu_preempt_state.call_remote = call_rcu_preempt_remote;
+#endif /* #ifdef CONFIG_PREEMPT_RCU */
+        rcu_bh_state.call_remote = call_rcu_bh_remote;
+        rcu_sched_state.call_remote = call_rcu_sched_remote;
+}
+#else /* #ifdef CONFIG_RCU_NOCB_CPU */
+static bool is_nocb_cpu(int cpu)
+{
+        return false;
+}
+static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
+                            bool lazy)
+{
+        return 0;
+}
+static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
+                                                     struct rcu_data *rdp)
+{
+        return 0;
+}
+static bool nocb_cpu_expendable(int cpu)
+{
+        return 1;
+}
+static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
+{
+}
+static void __init rcu_spawn_nocb_kthreads(struct rcu_state *rsp)
+{
+}
+static void init_nocb_callback_list(struct rcu_data *rdp)
+{
+}
+static void __init rcu_init_nocb(void)
+{
+}
+#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
diff --git a/kernel/rcutree_trace.c b/kernel/rcutree_trace.c
index 693513bc50e6..0d095dcaa670 100644
--- a/kernel/rcutree_trace.c
+++ b/kernel/rcutree_trace.c
@@ -46,29 +46,58 @@
 #define RCU_TREE_NONCORE
 #include "rcutree.h"
-static int show_rcubarrier(struct seq_file *m, void *unused)
+#define ulong2long(a) (*(long *)(&(a)))
+static int r_open(struct inode *inode, struct file *file,
+                                        const struct seq_operations *op)
 {
-        struct rcu_state *rsp;
+        int ret = seq_open(file, op);
+        if (!ret) {
+                struct seq_file *m = (struct seq_file *)file->private_data;
+                m->private = inode->i_private;
+        }
+        return ret;
+}
+static void *r_start(struct seq_file *m, loff_t *pos)
+{
+        struct rcu_state *rsp = (struct rcu_state *)m->private;
+        *pos = cpumask_next(*pos - 1, cpu_possible_mask);
+        if ((*pos) < nr_cpu_ids)
+                return per_cpu_ptr(rsp->rda, *pos);
+        return NULL;
+}
-        for_each_rcu_flavor(rsp)
+static void *r_next(struct seq_file *m, void *v, loff_t *pos)
-                seq_printf(m, "%s: bcc: %d nbd: %lu\n",
+{
-                           rsp->name,
+        (*pos)++;
-                           atomic_read(&rsp->barrier_cpu_count),
+        return r_start(m, pos);
-                           rsp->n_barrier_done);
+}
+static void r_stop(struct seq_file *m, void *v)
+{
+}
+static int show_rcubarrier(struct seq_file *m, void *v)
+{
+        struct rcu_state *rsp = (struct rcu_state *)m->private;
+        seq_printf(m, "bcc: %d nbd: %lu\n",
+                   atomic_read(&rsp->barrier_cpu_count),
+                   rsp->n_barrier_done);
        return 0;
 }
 static int rcubarrier_open(struct inode *inode, struct file *file)
 {
-        return single_open(file, show_rcubarrier, NULL);
+        return single_open(file, show_rcubarrier, inode->i_private);
 }
 static const struct file_operations rcubarrier_fops = {
        .owner = THIS_MODULE,
        .open = rcubarrier_open,
        .read = seq_read,
-        .llseek = seq_lseek,
+        .llseek = no_llseek,
-        .release = single_release,
+        .release = seq_release,
 };
 #ifdef CONFIG_RCU_BOOST
@@ -84,12 +113,14 @@ static char convert_kthread_status(unsigned int kthread_status)
 static void print_one_rcu_data(struct seq_file *m, struct rcu_data *rdp)
 {
+        long ql, qll;
        if (!rdp->beenonline)
                return;
-        seq_printf(m, "%3d%cc=%lu g=%lu pq=%d qp=%d",
+        seq_printf(m, "%3d%cc=%ld g=%ld pq=%d qp=%d",
                   rdp->cpu,
                   cpu_is_offline(rdp->cpu) ? '!' : ' ',
-                   rdp->completed, rdp->gpnum,
+                   ulong2long(rdp->completed), ulong2long(rdp->gpnum),
                   rdp->passed_quiesce, rdp->qs_pending);
        seq_printf(m, " dt=%d/%llx/%d df=%lu",
                   atomic_read(&rdp->dynticks->dynticks),
@@ -97,8 +128,11 @@ static void print_one_rcu_data(struct seq_file *m, struct rcu_data *rdp)
                   rdp->dynticks->dynticks_nmi_nesting,
                   rdp->dynticks_fqs);
        seq_printf(m, " of=%lu", rdp->offline_fqs);
+        rcu_nocb_q_lengths(rdp, &ql, &qll);
+        qll += rdp->qlen_lazy;
+        ql += rdp->qlen;
        seq_printf(m, " ql=%ld/%ld qs=%c%c%c%c",
-                   rdp->qlen_lazy, rdp->qlen,
+                   qll, ql,
                   ".N"[rdp->nxttail[RCU_NEXT_READY_TAIL] !=
                        rdp->nxttail[RCU_NEXT_TAIL]],
                   ".R"[rdp->nxttail[RCU_WAIT_TAIL] !=
@@ -114,101 +148,67 @@ static void print_one_rcu_data(struct seq_file *m, struct rcu_data *rdp)
                   per_cpu(rcu_cpu_kthread_loops, rdp->cpu) & 0xffff);
 #endif /* #ifdef CONFIG_RCU_BOOST */
        seq_printf(m, " b=%ld", rdp->blimit);
-        seq_printf(m, " ci=%lu co=%lu ca=%lu\n",
+        seq_printf(m, " ci=%lu nci=%lu co=%lu ca=%lu\n",
-                   rdp->n_cbs_invoked, rdp->n_cbs_orphaned, rdp->n_cbs_adopted);
+                   rdp->n_cbs_invoked, rdp->n_nocbs_invoked,
+                   rdp->n_cbs_orphaned, rdp->n_cbs_adopted);
 }
-static int show_rcudata(struct seq_file *m, void *unused)
+static int show_rcudata(struct seq_file *m, void *v)
 {
-        int cpu;
+        print_one_rcu_data(m, (struct rcu_data *)v);
-        struct rcu_state *rsp;
-        for_each_rcu_flavor(rsp) {
-                seq_printf(m, "%s:\n", rsp->name);
-                for_each_possible_cpu(cpu)
-                        print_one_rcu_data(m, per_cpu_ptr(rsp->rda, cpu));
-        }
        return 0;
 }
+static const struct seq_operations rcudate_op = {
+        .start = r_start,
+        .next  = r_next,
+        .stop  = r_stop,
+        .show  = show_rcudata,
+};
 static int rcudata_open(struct inode *inode, struct file *file)
 {
-        return single_open(file, show_rcudata, NULL);
+        return r_open(inode, file, &rcudate_op);
 }
 static const struct file_operations rcudata_fops = {
        .owner = THIS_MODULE,
        .open = rcudata_open,
        .read = seq_read,
-        .llseek = seq_lseek,
+        .llseek = no_llseek,
-        .release = single_release,
+        .release = seq_release,
 };
-static void print_one_rcu_data_csv(struct seq_file *m, struct rcu_data *rdp)
+static int show_rcuexp(struct seq_file *m, void *v)
-{
-        if (!rdp->beenonline)
-                return;
-        seq_printf(m, "%d,%s,%lu,%lu,%d,%d",
-                   rdp->cpu,
-                   cpu_is_offline(rdp->cpu) ? "\"N\"" : "\"Y\"",
-                   rdp->completed, rdp->gpnum,
-                   rdp->passed_quiesce, rdp->qs_pending);
-        seq_printf(m, ",%d,%llx,%d,%lu",
-                   atomic_read(&rdp->dynticks->dynticks),
-                   rdp->dynticks->dynticks_nesting,
-                   rdp->dynticks->dynticks_nmi_nesting,
-                   rdp->dynticks_fqs);
-        seq_printf(m, ",%lu", rdp->offline_fqs);
-        seq_printf(m, ",%ld,%ld,\"%c%c%c%c\"", rdp->qlen_lazy, rdp->qlen,
-                   ".N"[rdp->nxttail[RCU_NEXT_READY_TAIL] !=
-                        rdp->nxttail[RCU_NEXT_TAIL]],
-                   ".R"[rdp->nxttail[RCU_WAIT_TAIL] !=
-                        rdp->nxttail[RCU_NEXT_READY_TAIL]],
-                   ".W"[rdp->nxttail[RCU_DONE_TAIL] !=
-                        rdp->nxttail[RCU_WAIT_TAIL]],
-                   ".D"[&rdp->nxtlist != rdp->nxttail[RCU_DONE_TAIL]]);
-#ifdef CONFIG_RCU_BOOST
-        seq_printf(m, ",%d,\"%c\"",
-                   per_cpu(rcu_cpu_has_work, rdp->cpu),
-                   convert_kthread_status(per_cpu(rcu_cpu_kthread_status,
-                                          rdp->cpu)));
-#endif /* #ifdef CONFIG_RCU_BOOST */
-        seq_printf(m, ",%ld", rdp->blimit);
-        seq_printf(m, ",%lu,%lu,%lu\n",
-                   rdp->n_cbs_invoked, rdp->n_cbs_orphaned, rdp->n_cbs_adopted);
-}
-static int show_rcudata_csv(struct seq_file *m, void *unused)
 {
-        int cpu;
+        struct rcu_state *rsp = (struct rcu_state *)m->private;
-        struct rcu_state *rsp;
+        seq_printf(m, "s=%lu d=%lu w=%lu tf=%lu wd1=%lu wd2=%lu n=%lu sc=%lu dt=%lu dl=%lu dx=%lu\n",
-        seq_puts(m, "\"CPU\",\"Online?\",\"c\",\"g\",\"pq\",\"pq\",");
+                   atomic_long_read(&rsp->expedited_start),
-        seq_puts(m, "\"dt\",\"dt nesting\",\"dt NMI nesting\",\"df\",");
+                   atomic_long_read(&rsp->expedited_done),
-        seq_puts(m, "\"of\",\"qll\",\"ql\",\"qs\"");
+                   atomic_long_read(&rsp->expedited_wrap),
-#ifdef CONFIG_RCU_BOOST
+                   atomic_long_read(&rsp->expedited_tryfail),
-        seq_puts(m, "\"kt\",\"ktl\"");
+                   atomic_long_read(&rsp->expedited_workdone1),
-#endif /* #ifdef CONFIG_RCU_BOOST */
+                   atomic_long_read(&rsp->expedited_workdone2),
-        seq_puts(m, ",\"b\",\"ci\",\"co\",\"ca\"\n");
+                   atomic_long_read(&rsp->expedited_normal),
-        for_each_rcu_flavor(rsp) {
+                   atomic_long_read(&rsp->expedited_stoppedcpus),
-                seq_printf(m, "\"%s:\"\n", rsp->name);
+                   atomic_long_read(&rsp->expedited_done_tries),
-                for_each_possible_cpu(cpu)
+                   atomic_long_read(&rsp->expedited_done_lost),
-                        print_one_rcu_data_csv(m, per_cpu_ptr(rsp->rda, cpu));
+                   atomic_long_read(&rsp->expedited_done_exit));
-        }
        return 0;
 }
-static int rcudata_csv_open(struct inode *inode, struct file *file)
+static int rcuexp_open(struct inode *inode, struct file *file)
 {
-        return single_open(file, show_rcudata_csv, NULL);
+        return single_open(file, show_rcuexp, inode->i_private);
 }
-static const struct file_operations rcudata_csv_fops = {
+static const struct file_operations rcuexp_fops = {
        .owner = THIS_MODULE,
-        .open = rcudata_csv_open,
+        .open = rcuexp_open,
        .read = seq_read,
-        .llseek = seq_lseek,
+        .llseek = no_llseek,
-        .release = single_release,
+        .release = seq_release,
 };
 #ifdef CONFIG_RCU_BOOST
@@ -254,27 +254,11 @@ static const struct file_operations rcu_node_boost_fops = {
        .owner = THIS_MODULE,
        .open = rcu_node_boost_open,
        .read = seq_read,
-        .llseek = seq_lseek,
+        .llseek = no_llseek,
        .release = single_release,
 };
-/*
+#endif /* #ifdef CONFIG_RCU_BOOST */
- * Create the rcuboost debugfs entry.  Standard error return.
- */
-static int rcu_boost_trace_create_file(struct dentry *rcudir)
-{
-        return !debugfs_create_file("rcuboost", 0444, rcudir, NULL,
-                                    &rcu_node_boost_fops);
-}
-#else /* #ifdef CONFIG_RCU_BOOST */
-static int rcu_boost_trace_create_file(struct dentry *rcudir)
-{
-        return 0;  /* There cannot be an error if we didn't create it! */
-}
-#endif /* #else #ifdef CONFIG_RCU_BOOST */
 static void print_one_rcu_state(struct seq_file *m, struct rcu_state *rsp)
 {
@@ -283,8 +267,9 @@ static void print_one_rcu_state(struct seq_file *m, struct rcu_state *rsp)
        struct rcu_node *rnp;
        gpnum = rsp->gpnum;
-        seq_printf(m, "%s: c=%lu g=%lu s=%d jfq=%ld j=%x ",
+        seq_printf(m, "c=%ld g=%ld s=%d jfq=%ld j=%x ",
-                   rsp->name, rsp->completed, gpnum, rsp->fqs_state,
+                   ulong2long(rsp->completed), ulong2long(gpnum),
+                   rsp->fqs_state,
                   (long)(rsp->jiffies_force_qs - jiffies),
                   (int)(jiffies & 0xffff));
        seq_printf(m, "nfqs=%lu/nfqsng=%lu(%lu) fqlh=%lu oqlen=%ld/%ld\n",
@@ -306,26 +291,24 @@ static void print_one_rcu_state(struct seq_file *m, struct rcu_state *rsp)
        seq_puts(m, "\n");
 }
-static int show_rcuhier(struct seq_file *m, void *unused)
+static int show_rcuhier(struct seq_file *m, void *v)
 {
-        struct rcu_state *rsp;
+        struct rcu_state *rsp = (struct rcu_state *)m->private;
+        print_one_rcu_state(m, rsp);
-        for_each_rcu_flavor(rsp)
-                print_one_rcu_state(m, rsp);
        return 0;
 }
 static int rcuhier_open(struct inode *inode, struct file *file)
 {
-        return single_open(file, show_rcuhier, NULL);
+        return single_open(file, show_rcuhier, inode->i_private);
 }
 static const struct file_operations rcuhier_fops = {
        .owner = THIS_MODULE,
        .open = rcuhier_open,
        .read = seq_read,
-        .llseek = seq_lseek,
+        .llseek = no_llseek,
-        .release = single_release,
+        .release = seq_release,
 };
 static void show_one_rcugp(struct seq_file *m, struct rcu_state *rsp)
@@ -338,42 +321,42 @@ static void show_one_rcugp(struct seq_file *m, struct rcu_state *rsp)
        struct rcu_node *rnp = &rsp->node[0];
        raw_spin_lock_irqsave(&rnp->lock, flags);
-        completed = rsp->completed;
+        completed = ACCESS_ONCE(rsp->completed);
-        gpnum = rsp->gpnum;
+        gpnum = ACCESS_ONCE(rsp->gpnum);
-        if (rsp->completed == rsp->gpnum)
+        if (completed == gpnum)
                gpage = 0;
        else
                gpage = jiffies - rsp->gp_start;
        gpmax = rsp->gp_max;
        raw_spin_unlock_irqrestore(&rnp->lock, flags);
-        seq_printf(m, "%s: completed=%ld  gpnum=%lu  age=%ld  max=%ld\n",
+        seq_printf(m, "completed=%ld  gpnum=%ld  age=%ld  max=%ld\n",
-                   rsp->name, completed, gpnum, gpage, gpmax);
+                   ulong2long(completed), ulong2long(gpnum), gpage, gpmax);
 }
-static int show_rcugp(struct seq_file *m, void *unused)
+static int show_rcugp(struct seq_file *m, void *v)
 {
-        struct rcu_state *rsp;
+        struct rcu_state *rsp = (struct rcu_state *)m->private;
+        show_one_rcugp(m, rsp);
-        for_each_rcu_flavor(rsp)
-                show_one_rcugp(m, rsp);
        return 0;
 }
 static int rcugp_open(struct inode *inode, struct file *file)
 {
-        return single_open(file, show_rcugp, NULL);
+        return single_open(file, show_rcugp, inode->i_private);
 }
 static const struct file_operations rcugp_fops = {
        .owner = THIS_MODULE,
        .open = rcugp_open,
        .read = seq_read,
-        .llseek = seq_lseek,
+        .llseek = no_llseek,
-        .release = single_release,
+        .release = seq_release,
 };
 static void print_one_rcu_pending(struct seq_file *m, struct rcu_data *rdp)
 {
+        if (!rdp->beenonline)
+                return;
        seq_printf(m, "%3d%cnp=%ld ",
                   rdp->cpu,
                   cpu_is_offline(rdp->cpu) ? '!' : ' ',
@@ -389,34 +372,30 @@ static void print_one_rcu_pending(struct seq_file *m, struct rcu_data *rdp)
                   rdp->n_rp_need_nothing);
 }
-static int show_rcu_pending(struct seq_file *m, void *unused)
+static int show_rcu_pending(struct seq_file *m, void *v)
 {
-        int cpu;
+        print_one_rcu_pending(m, (struct rcu_data *)v);
-        struct rcu_data *rdp;
-        struct rcu_state *rsp;
-        for_each_rcu_flavor(rsp) {
-                seq_printf(m, "%s:\n", rsp->name);
-                for_each_possible_cpu(cpu) {
-                        rdp = per_cpu_ptr(rsp->rda, cpu);
-                        if (rdp->beenonline)
-                                print_one_rcu_pending(m, rdp);
-                }
-        }
        return 0;
 }
+static const struct seq_operations rcu_pending_op = {
+        .start = r_start,
+        .next  = r_next,
+        .stop  = r_stop,
+        .show  = show_rcu_pending,
+};
 static int rcu_pending_open(struct inode *inode, struct file *file)
 {
-        return single_open(file, show_rcu_pending, NULL);
+        return r_open(inode, file, &rcu_pending_op);
 }
 static const struct file_operations rcu_pending_fops = {
        .owner = THIS_MODULE,
        .open = rcu_pending_open,
        .read = seq_read,
-        .llseek = seq_lseek,
+        .llseek = no_llseek,
-        .release = single_release,
+        .release = seq_release,
 };
 static int show_rcutorture(struct seq_file *m, void *unused)
@@ -446,43 +425,58 @@ static struct dentry *rcudir;
 static int __init rcutree_trace_init(void)
 {
+        struct rcu_state *rsp;
        struct dentry *retval;
+        struct dentry *rspdir;
        rcudir = debugfs_create_dir("rcu", NULL);
        if (!rcudir)
                goto free_out;
-        retval = debugfs_create_file("rcubarrier", 0444, rcudir,
+        for_each_rcu_flavor(rsp) {
-                                                NULL, &rcubarrier_fops);
+                rspdir = debugfs_create_dir(rsp->name, rcudir);
-        if (!retval)
+                if (!rspdir)
-                goto free_out;
+                        goto free_out;
-        retval = debugfs_create_file("rcudata", 0444, rcudir,
+                retval = debugfs_create_file("rcudata", 0444,
-                                                NULL, &rcudata_fops);
+                                rspdir, rsp, &rcudata_fops);
-        if (!retval)
+                if (!retval)
-                goto free_out;
+                        goto free_out;
-        retval = debugfs_create_file("rcudata.csv", 0444, rcudir,
+                retval = debugfs_create_file("rcuexp", 0444,
-                                                NULL, &rcudata_csv_fops);
+                                rspdir, rsp, &rcuexp_fops);
-        if (!retval)
+                if (!retval)
-                goto free_out;
+                        goto free_out;
-        if (rcu_boost_trace_create_file(rcudir))
+                retval = debugfs_create_file("rcu_pending", 0444,
-                goto free_out;
+                                rspdir, rsp, &rcu_pending_fops);
+                if (!retval)
+                        goto free_out;
+                retval = debugfs_create_file("rcubarrier", 0444,
+                                rspdir, rsp, &rcubarrier_fops);
+                if (!retval)
+                        goto free_out;
-        retval = debugfs_create_file("rcugp", 0444, rcudir, NULL, &rcugp_fops);
+#ifdef CONFIG_RCU_BOOST
-        if (!retval)
+                if (rsp == &rcu_preempt_state) {
-                goto free_out;
+                        retval = debugfs_create_file("rcuboost", 0444,
+                                rspdir, NULL, &rcu_node_boost_fops);
+                        if (!retval)
+                                goto free_out;
+                }
+#endif
-        retval = debugfs_create_file("rcuhier", 0444, rcudir,
+                retval = debugfs_create_file("rcugp", 0444,
-                                                NULL, &rcuhier_fops);
+                                rspdir, rsp, &rcugp_fops);
-        if (!retval)
+                if (!retval)
-                goto free_out;
+                        goto free_out;
-        retval = debugfs_create_file("rcu_pending", 0444, rcudir,
+                retval = debugfs_create_file("rcuhier", 0444,
-                                                NULL, &rcu_pending_fops);
+                                rspdir, rsp, &rcuhier_fops);
-        if (!retval)
+                if (!retval)
-                goto free_out;
+                        goto free_out;
+        }
        retval = debugfs_create_file("rcutorture", 0444, rcudir,
                                                NULL, &rcutorture_fops);
diff --git a/kernel/res_counter.c b/kernel/res_counter.c
index ad581aa2369a..3920d593e63c 100644
--- a/kernel/res_counter.c
+++ b/kernel/res_counter.c
@@ -192,25 +192,3 @@ int res_counter_memparse_write_strategy(const char *buf,
        *res = PAGE_ALIGN(*res);
        return 0;
 }
-int res_counter_write(struct res_counter *counter, int member,
-                      const char *buf, write_strategy_fn write_strategy)
-{
-        char *end;
-        unsigned long flags;
-        unsigned long long tmp, *val;
-        if (write_strategy) {
-                if (write_strategy(buf, &tmp))
-                        return -EINVAL;
-        } else {
-                tmp = simple_strtoull(buf, &end, 10);
-                if (*end != '\0')
-                        return -EINVAL;
-        }
-        spin_lock_irqsave(&counter->lock, flags);
-        val = res_counter_member(counter, member);
-        *val = tmp;
-        spin_unlock_irqrestore(&counter->lock, flags);
-        return 0;
-}
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 2f5eb1838b3e..257002c13bb0 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -72,6 +72,7 @@
 #include <linux/slab.h>
 #include <linux/init_task.h>
 #include <linux/binfmts.h>
+#include <linux/context_tracking.h>
 #include <asm/switch_to.h>
 #include <asm/tlb.h>
@@ -192,23 +193,10 @@ static void sched_feat_disable(int i) { };
 static void sched_feat_enable(int i) { };
 #endif /* HAVE_JUMP_LABEL */
-static ssize_t
+static int sched_feat_set(char *cmp)
-sched_feat_write(struct file *filp, const char __user *ubuf,
-                size_t cnt, loff_t *ppos)
 {
-        char buf[64];
-        char *cmp;
-        int neg = 0;
        int i;
+        int neg = 0;
-        if (cnt > 63)
-                cnt = 63;
-        if (copy_from_user(&buf, ubuf, cnt))
-                return -EFAULT;
-        buf[cnt] = 0;
-        cmp = strstrip(buf);
        if (strncmp(cmp, "NO_", 3) == 0) {
                neg = 1;
@@ -228,6 +216,27 @@ sched_feat_write(struct file *filp, const char __user *ubuf,
                }
        }
+        return i;
+}
+static ssize_t
+sched_feat_write(struct file *filp, const char __user *ubuf,
+                size_t cnt, loff_t *ppos)
+{
+        char buf[64];
+        char *cmp;
+        int i;
+        if (cnt > 63)
+                cnt = 63;
+        if (copy_from_user(&buf, ubuf, cnt))
+                return -EFAULT;
+        buf[cnt] = 0;
+        cmp = strstrip(buf);
+        i = sched_feat_set(cmp);
        if (i == __SCHED_FEAT_NR)
                return -EINVAL;
@@ -922,6 +931,13 @@ void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
                rq->skip_clock_update = 1;
 }
+static ATOMIC_NOTIFIER_HEAD(task_migration_notifier);
+void register_task_migration_notifier(struct notifier_block *n)
+{
+        atomic_notifier_chain_register(&task_migration_notifier, n);
+}
 #ifdef CONFIG_SMP
 void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
 {
@@ -952,8 +968,18 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
        trace_sched_migrate_task(p, new_cpu);
        if (task_cpu(p) != new_cpu) {
+                struct task_migration_notifier tmn;
+                if (p->sched_class->migrate_task_rq)
+                        p->sched_class->migrate_task_rq(p, new_cpu);
                p->se.nr_migrations++;
                perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, NULL, 0);
+                tmn.task = p;
+                tmn.from_cpu = task_cpu(p);
+                tmn.to_cpu = new_cpu;
+                atomic_notifier_call_chain(&task_migration_notifier, 0, &tmn);
        }
        __set_task_cpu(p, new_cpu);
@@ -1524,6 +1550,15 @@ static void __sched_fork(struct task_struct *p)
        p->se.vruntime                  = 0;
        INIT_LIST_HEAD(&p->se.group_node);
+/*
+ * Load-tracking only depends on SMP, FAIR_GROUP_SCHED dependency below may be
+ * removed when useful for applications beyond shares distribution (e.g.
+ * load-balance).
+ */
+#if defined(CONFIG_SMP) && defined(CONFIG_FAIR_GROUP_SCHED)
+        p->se.avg.runnable_avg_period = 0;
+        p->se.avg.runnable_avg_sum = 0;
+#endif
 #ifdef CONFIG_SCHEDSTATS
        memset(&p->se.statistics, 0, sizeof(p->se.statistics));
 #endif
@@ -1533,8 +1568,41 @@ static void __sched_fork(struct task_struct *p)
 #ifdef CONFIG_PREEMPT_NOTIFIERS
        INIT_HLIST_HEAD(&p->preempt_notifiers);
 #endif
+#ifdef CONFIG_NUMA_BALANCING
+        if (p->mm && atomic_read(&p->mm->mm_users) == 1) {
+                p->mm->numa_next_scan = jiffies;
+                p->mm->numa_next_reset = jiffies;
+                p->mm->numa_scan_seq = 0;
+        }
+        p->node_stamp = 0ULL;
+        p->numa_scan_seq = p->mm ? p->mm->numa_scan_seq : 0;
+        p->numa_migrate_seq = p->mm ? p->mm->numa_scan_seq - 1 : 0;
+        p->numa_scan_period = sysctl_numa_balancing_scan_delay;
+        p->numa_work.next = &p->numa_work;
+#endif /* CONFIG_NUMA_BALANCING */
 }
+#ifdef CONFIG_NUMA_BALANCING
+#ifdef CONFIG_SCHED_DEBUG
+void set_numabalancing_state(bool enabled)
+{
+        if (enabled)
+                sched_feat_set("NUMA");
+        else
+                sched_feat_set("NO_NUMA");
+}
+#else
+__read_mostly bool numabalancing_enabled;
+void set_numabalancing_state(bool enabled)
+{
+        numabalancing_enabled = enabled;
+}
+#endif /* CONFIG_SCHED_DEBUG */
+#endif /* CONFIG_NUMA_BALANCING */
 /*
 * fork()/clone()-time setup:
 */
@@ -1886,8 +1954,8 @@ context_switch(struct rq *rq, struct task_struct *prev,
        spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
 #endif
+        context_tracking_task_switch(prev, next);
        /* Here we just switch the register state and the stack. */
-        rcu_switch(prev, next);
        switch_to(prev, next, prev);
        barrier();
@@ -2911,7 +2979,7 @@ asmlinkage void __sched schedule(void)
 }
 EXPORT_SYMBOL(schedule);
-#ifdef CONFIG_RCU_USER_QS
+#ifdef CONFIG_CONTEXT_TRACKING
 asmlinkage void __sched schedule_user(void)
 {
        /*
@@ -2920,9 +2988,9 @@ asmlinkage void __sched schedule_user(void)
         * we haven't yet exited the RCU idle mode. Do it here manually until
         * we find a better solution.
         */
-        rcu_user_exit();
+        user_exit();
        schedule();
-        rcu_user_enter();
+        user_enter();
 }
 #endif
@@ -3027,7 +3095,7 @@ asmlinkage void __sched preempt_schedule_irq(void)
        /* Catch callers which need to be fixed */
        BUG_ON(ti->preempt_count || !irqs_disabled());
-        rcu_user_exit();
+        user_exit();
        do {
                add_preempt_count(PREEMPT_ACTIVE);
                local_irq_enable();
@@ -4480,6 +4548,7 @@ static const char stat_nam[] = TASK_STATE_TO_CHAR_STR;
 void sched_show_task(struct task_struct *p)
 {
        unsigned long free = 0;
+        int ppid;
        unsigned state;
        state = p->state ? __ffs(p->state) + 1 : 0;
@@ -4499,8 +4568,11 @@ void sched_show_task(struct task_struct *p)
 #ifdef CONFIG_DEBUG_STACK_USAGE
        free = stack_not_used(p);
 #endif
+        rcu_read_lock();
+        ppid = task_pid_nr(rcu_dereference(p->real_parent));
+        rcu_read_unlock();
        printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free,
-                task_pid_nr(p), task_pid_nr(rcu_dereference(p->real_parent)),
+                task_pid_nr(p), ppid,
                (unsigned long)task_thread_info(p)->flags);
        show_stack(p, NULL);
@@ -7474,7 +7546,7 @@ static inline struct task_group *cgroup_tg(struct cgroup *cgrp)
                            struct task_group, css);
 }
-static struct cgroup_subsys_state *cpu_cgroup_create(struct cgroup *cgrp)
+static struct cgroup_subsys_state *cpu_cgroup_css_alloc(struct cgroup *cgrp)
 {
        struct task_group *tg, *parent;
@@ -7491,7 +7563,7 @@ static struct cgroup_subsys_state *cpu_cgroup_create(struct cgroup *cgrp)
        return &tg->css;
 }
-static void cpu_cgroup_destroy(struct cgroup *cgrp)
+static void cpu_cgroup_css_free(struct cgroup *cgrp)
 {
        struct task_group *tg = cgroup_tg(cgrp);
@@ -7851,8 +7923,8 @@ static struct cftype cpu_files[] = {
 struct cgroup_subsys cpu_cgroup_subsys = {
        .name           = "cpu",
-        .create         = cpu_cgroup_create,
+        .css_alloc      = cpu_cgroup_css_alloc,
-        .destroy        = cpu_cgroup_destroy,
+        .css_free       = cpu_cgroup_css_free,
        .can_attach     = cpu_cgroup_can_attach,
        .attach         = cpu_cgroup_attach,
        .exit           = cpu_cgroup_exit,
@@ -7875,7 +7947,7 @@ struct cgroup_subsys cpu_cgroup_subsys = {
 struct cpuacct root_cpuacct;
 /* create a new cpu accounting group */
-static struct cgroup_subsys_state *cpuacct_create(struct cgroup *cgrp)
+static struct cgroup_subsys_state *cpuacct_css_alloc(struct cgroup *cgrp)
 {
        struct cpuacct *ca;
@@ -7905,7 +7977,7 @@ out:
 }
 /* destroy an existing cpu accounting group */
-static void cpuacct_destroy(struct cgroup *cgrp)
+static void cpuacct_css_free(struct cgroup *cgrp)
 {
        struct cpuacct *ca = cgroup_ca(cgrp);
@@ -8076,9 +8148,15 @@ void cpuacct_charge(struct task_struct *tsk, u64 cputime)
 struct cgroup_subsys cpuacct_subsys = {
        .name = "cpuacct",
-        .create = cpuacct_create,
+        .css_alloc = cpuacct_css_alloc,
-        .destroy = cpuacct_destroy,
+        .css_free = cpuacct_css_free,
        .subsys_id = cpuacct_subsys_id,
        .base_cftypes = files,
 };
 #endif  /* CONFIG_CGROUP_CPUACCT */
+void dump_cpu_task(int cpu)
+{
+        pr_info("Task dump for CPU %d:\n", cpu);
+        sched_show_task(cpu_curr(cpu));
+}
diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c
index 81b763ba58a6..293b202fcf79 100644
--- a/kernel/sched/cputime.c
+++ b/kernel/sched/cputime.c
@@ -43,7 +43,7 @@ DEFINE_PER_CPU(seqcount_t, irq_time_seq);
 * Called before incrementing preempt_count on {soft,}irq_enter
 * and before decrementing preempt_count on {soft,}irq_exit.
 */
-void vtime_account(struct task_struct *curr)
+void irqtime_account_irq(struct task_struct *curr)
 {
        unsigned long flags;
        s64 delta;
@@ -73,7 +73,7 @@ void vtime_account(struct task_struct *curr)
        irq_time_write_end();
        local_irq_restore(flags);
 }
-EXPORT_SYMBOL_GPL(vtime_account);
+EXPORT_SYMBOL_GPL(irqtime_account_irq);
 static int irqtime_account_hi_update(void)
 {
@@ -288,6 +288,34 @@ static __always_inline bool steal_account_process_tick(void)
        return false;
 }
+/*
+ * Accumulate raw cputime values of dead tasks (sig->[us]time) and live
+ * tasks (sum on group iteration) belonging to @tsk's group.
+ */
+void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times)
+{
+        struct signal_struct *sig = tsk->signal;
+        struct task_struct *t;
+        times->utime = sig->utime;
+        times->stime = sig->stime;
+        times->sum_exec_runtime = sig->sum_sched_runtime;
+        rcu_read_lock();
+        /* make sure we can trust tsk->thread_group list */
+        if (!likely(pid_alive(tsk)))
+                goto out;
+        t = tsk;
+        do {
+                times->utime += t->utime;
+                times->stime += t->stime;
+                times->sum_exec_runtime += task_sched_runtime(t);
+        } while_each_thread(tsk, t);
+out:
+        rcu_read_unlock();
+}
 #ifndef CONFIG_VIRT_CPU_ACCOUNTING
 #ifdef CONFIG_IRQ_TIME_ACCOUNTING
@@ -417,13 +445,13 @@ void account_idle_ticks(unsigned long ticks)
 * Use precise platform statistics if available:
 */
 #ifdef CONFIG_VIRT_CPU_ACCOUNTING
-void task_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
+void task_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st)
 {
        *ut = p->utime;
        *st = p->stime;
 }
-void thread_group_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
+void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st)
 {
        struct task_cputime cputime;
@@ -433,6 +461,29 @@ void thread_group_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
        *st = cputime.stime;
 }
+void vtime_account_system_irqsafe(struct task_struct *tsk)
+{
+        unsigned long flags;
+        local_irq_save(flags);
+        vtime_account_system(tsk);
+        local_irq_restore(flags);
+}
+EXPORT_SYMBOL_GPL(vtime_account_system_irqsafe);
+#ifndef __ARCH_HAS_VTIME_TASK_SWITCH
+void vtime_task_switch(struct task_struct *prev)
+{
+        if (is_idle_task(prev))
+                vtime_account_idle(prev);
+        else
+                vtime_account_system(prev);
+        vtime_account_user(prev);
+        arch_vtime_task_switch(prev);
+}
+#endif
 /*
 * Archs that account the whole time spent in the idle task
 * (outside irq) as idle time can rely on this and just implement
@@ -444,16 +495,10 @@ void thread_group_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
 #ifndef __ARCH_HAS_VTIME_ACCOUNT
 void vtime_account(struct task_struct *tsk)
 {
-        unsigned long flags;
-        local_irq_save(flags);
        if (in_interrupt() || !is_idle_task(tsk))
                vtime_account_system(tsk);
        else
                vtime_account_idle(tsk);
-        local_irq_restore(flags);
 }
 EXPORT_SYMBOL_GPL(vtime_account);
 #endif /* __ARCH_HAS_VTIME_ACCOUNT */
@@ -478,14 +523,30 @@ static cputime_t scale_utime(cputime_t utime, cputime_t rtime, cputime_t total)
        return (__force cputime_t) temp;
 }
-void task_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
+/*
+ * Adjust tick based cputime random precision against scheduler
+ * runtime accounting.
+ */
+static void cputime_adjust(struct task_cputime *curr,
+                           struct cputime *prev,
+                           cputime_t *ut, cputime_t *st)
 {
-        cputime_t rtime, utime = p->utime, total = utime + p->stime;
+        cputime_t rtime, utime, total;
+        utime = curr->utime;
+        total = utime + curr->stime;
        /*
-         * Use CFS's precise accounting:
+         * Tick based cputime accounting depend on random scheduling
+         * timeslices of a task to be interrupted or not by the timer.
+         * Depending on these circumstances, the number of these interrupts
+         * may be over or under-optimistic, matching the real user and system
+         * cputime with a variable precision.
+         *
+         * Fix this by scaling these tick based values against the total
+         * runtime accounted by the CFS scheduler.
         */
-        rtime = nsecs_to_cputime(p->se.sum_exec_runtime);
+        rtime = nsecs_to_cputime(curr->sum_exec_runtime);
        if (total)
                utime = scale_utime(utime, rtime, total);
@@ -493,38 +554,36 @@ void task_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
                utime = rtime;
        /*
-         * Compare with previous values, to keep monotonicity:
+         * If the tick based count grows faster than the scheduler one,
+         * the result of the scaling may go backward.
+         * Let's enforce monotonicity.
         */
-        p->prev_utime = max(p->prev_utime, utime);
+        prev->utime = max(prev->utime, utime);
-        p->prev_stime = max(p->prev_stime, rtime - p->prev_utime);
+        prev->stime = max(prev->stime, rtime - prev->utime);
-        *ut = p->prev_utime;
+        *ut = prev->utime;
-        *st = p->prev_stime;
+        *st = prev->stime;
+}
+void task_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st)
+{
+        struct task_cputime cputime = {
+                .utime = p->utime,
+                .stime = p->stime,
+                .sum_exec_runtime = p->se.sum_exec_runtime,
+        };
+        cputime_adjust(&cputime, &p->prev_cputime, ut, st);
 }
 /*
 * Must be called with siglock held.
 */
-void thread_group_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
+void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st)
 {
-        struct signal_struct *sig = p->signal;
        struct task_cputime cputime;
-        cputime_t rtime, utime, total;
        thread_group_cputime(p, &cputime);
+        cputime_adjust(&cputime, &p->signal->prev_cputime, ut, st);
-        total = cputime.utime + cputime.stime;
-        rtime = nsecs_to_cputime(cputime.sum_exec_runtime);
-        if (total)
-                utime = scale_utime(cputime.utime, rtime, total);
-        else
-                utime = rtime;
-        sig->prev_utime = max(sig->prev_utime, utime);
-        sig->prev_stime = max(sig->prev_stime, rtime - sig->prev_utime);
-        *ut = sig->prev_utime;
-        *st = sig->prev_stime;
 }
 #endif
diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
index 6f79596e0ea9..2cd3c1b4e582 100644
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -61,14 +61,20 @@ static unsigned long nsec_low(unsigned long long nsec)
 static void print_cfs_group_stats(struct seq_file *m, int cpu, struct task_group *tg)
 {
        struct sched_entity *se = tg->se[cpu];
-        if (!se)
-                return;
 #define P(F) \
        SEQ_printf(m, "  .%-30s: %lld\n", #F, (long long)F)
 #define PN(F) \
        SEQ_printf(m, "  .%-30s: %lld.%06ld\n", #F, SPLIT_NS((long long)F))
+        if (!se) {
+                struct sched_avg *avg = &cpu_rq(cpu)->avg;
+                P(avg->runnable_avg_sum);
+                P(avg->runnable_avg_period);
+                return;
+        }
        PN(se->exec_start);
        PN(se->vruntime);
        PN(se->sum_exec_runtime);
@@ -85,6 +91,12 @@ static void print_cfs_group_stats(struct seq_file *m, int cpu, struct task_group
        P(se->statistics.wait_count);
 #endif
        P(se->load.weight);
+#ifdef CONFIG_SMP
+        P(se->avg.runnable_avg_sum);
+        P(se->avg.runnable_avg_period);
+        P(se->avg.load_avg_contrib);
+        P(se->avg.decay_count);
+#endif
 #undef PN
 #undef P
 }
@@ -206,14 +218,18 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
        SEQ_printf(m, "  .%-30s: %ld\n", "load", cfs_rq->load.weight);
 #ifdef CONFIG_FAIR_GROUP_SCHED
 #ifdef CONFIG_SMP
-        SEQ_printf(m, "  .%-30s: %Ld.%06ld\n", "load_avg",
+        SEQ_printf(m, "  .%-30s: %lld\n", "runnable_load_avg",
-                        SPLIT_NS(cfs_rq->load_avg));
+                        cfs_rq->runnable_load_avg);
-        SEQ_printf(m, "  .%-30s: %Ld.%06ld\n", "load_period",
+        SEQ_printf(m, "  .%-30s: %lld\n", "blocked_load_avg",
-                        SPLIT_NS(cfs_rq->load_period));
+                        cfs_rq->blocked_load_avg);
-        SEQ_printf(m, "  .%-30s: %ld\n", "load_contrib",
+        SEQ_printf(m, "  .%-30s: %ld\n", "tg_load_avg",
-                        cfs_rq->load_contribution);
+                        atomic64_read(&cfs_rq->tg->load_avg));
-        SEQ_printf(m, "  .%-30s: %d\n", "load_tg",
+        SEQ_printf(m, "  .%-30s: %lld\n", "tg_load_contrib",
-                        atomic_read(&cfs_rq->tg->load_weight));
+                        cfs_rq->tg_load_contrib);
+        SEQ_printf(m, "  .%-30s: %d\n", "tg_runnable_contrib",
+                        cfs_rq->tg_runnable_contrib);
+        SEQ_printf(m, "  .%-30s: %d\n", "tg->runnable_avg",
+                        atomic_read(&cfs_rq->tg->runnable_avg));
 #endif
        print_cfs_group_stats(m, cpu, cfs_rq->tg);
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 6b800a14b990..4603d6cb9e25 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -26,6 +26,9 @@
 #include <linux/slab.h>
 #include <linux/profile.h>
 #include <linux/interrupt.h>
+#include <linux/mempolicy.h>
+#include <linux/migrate.h>
+#include <linux/task_work.h>
 #include <trace/events/sched.h>
@@ -259,6 +262,9 @@ static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp)
        return grp->my_q;
 }
+static void update_cfs_rq_blocked_load(struct cfs_rq *cfs_rq,
+                                       int force_update);
 static inline void list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
 {
        if (!cfs_rq->on_list) {
@@ -278,6 +284,8 @@ static inline void list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
                }
                cfs_rq->on_list = 1;
+                /* We should have no load, but we need to update last_decay. */
+                update_cfs_rq_blocked_load(cfs_rq, 0);
        }
 }
@@ -653,9 +661,6 @@ static u64 sched_vslice(struct cfs_rq *cfs_rq, struct sched_entity *se)
        return calc_delta_fair(sched_slice(cfs_rq, se), se);
 }
-static void update_cfs_load(struct cfs_rq *cfs_rq, int global_update);
-static void update_cfs_shares(struct cfs_rq *cfs_rq);
 /*
 * Update the current task's runtime statistics. Skip current tasks that
 * are not in our scheduling class.
@@ -675,10 +680,6 @@ __update_curr(struct cfs_rq *cfs_rq, struct sched_entity *curr,
        curr->vruntime += delta_exec_weighted;
        update_min_vruntime(cfs_rq);
-#if defined CONFIG_SMP && defined CONFIG_FAIR_GROUP_SCHED
-        cfs_rq->load_unacc_exec_time += delta_exec;
-#endif
 }
 static void update_curr(struct cfs_rq *cfs_rq)
@@ -776,6 +777,227 @@ update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
 * Scheduling class queueing methods:
 */
+#ifdef CONFIG_NUMA_BALANCING
+/*
+ * numa task sample period in ms
+ */
+unsigned int sysctl_numa_balancing_scan_period_min = 100;
+unsigned int sysctl_numa_balancing_scan_period_max = 100*50;
+unsigned int sysctl_numa_balancing_scan_period_reset = 100*600;
+/* Portion of address space to scan in MB */
+unsigned int sysctl_numa_balancing_scan_size = 256;
+/* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */
+unsigned int sysctl_numa_balancing_scan_delay = 1000;
+static void task_numa_placement(struct task_struct *p)
+{
+        int seq = ACCESS_ONCE(p->mm->numa_scan_seq);
+        if (p->numa_scan_seq == seq)
+                return;
+        p->numa_scan_seq = seq;
+        /* FIXME: Scheduling placement policy hints go here */
+}
+/*
+ * Got a PROT_NONE fault for a page on @node.
+ */
+void task_numa_fault(int node, int pages, bool migrated)
+{
+        struct task_struct *p = current;
+        if (!sched_feat_numa(NUMA))
+                return;
+        /* FIXME: Allocate task-specific structure for placement policy here */
+        /*
+         * If pages are properly placed (did not migrate) then scan slower.
+         * This is reset periodically in case of phase changes
+         */
+        if (!migrated)
+                p->numa_scan_period = min(sysctl_numa_balancing_scan_period_max,
+                        p->numa_scan_period + jiffies_to_msecs(10));
+        task_numa_placement(p);
+}
+static void reset_ptenuma_scan(struct task_struct *p)
+{
+        ACCESS_ONCE(p->mm->numa_scan_seq)++;
+        p->mm->numa_scan_offset = 0;
+}
+/*
+ * The expensive part of numa migration is done from task_work context.
+ * Triggered from task_tick_numa().
+ */
+void task_numa_work(struct callback_head *work)
+{
+        unsigned long migrate, next_scan, now = jiffies;
+        struct task_struct *p = current;
+        struct mm_struct *mm = p->mm;
+        struct vm_area_struct *vma;
+        unsigned long start, end;
+        long pages;
+        WARN_ON_ONCE(p != container_of(work, struct task_struct, numa_work));
+        work->next = work; /* protect against double add */
+        /*
+         * Who cares about NUMA placement when they're dying.
+         *
+         * NOTE: make sure not to dereference p->mm before this check,
+         * exit_task_work() happens _after_ exit_mm() so we could be called
+         * without p->mm even though we still had it when we enqueued this
+         * work.
+         */
+        if (p->flags & PF_EXITING)
+                return;
+        /*
+         * We do not care about task placement until a task runs on a node
+         * other than the first one used by the address space. This is
+         * largely because migrations are driven by what CPU the task
+         * is running on. If it's never scheduled on another node, it'll
+         * not migrate so why bother trapping the fault.
+         */
+        if (mm->first_nid == NUMA_PTE_SCAN_INIT)
+                mm->first_nid = numa_node_id();
+        if (mm->first_nid != NUMA_PTE_SCAN_ACTIVE) {
+                /* Are we running on a new node yet? */
+                if (numa_node_id() == mm->first_nid &&
+                    !sched_feat_numa(NUMA_FORCE))
+                        return;
+                mm->first_nid = NUMA_PTE_SCAN_ACTIVE;
+        }
+        /*
+         * Reset the scan period if enough time has gone by. Objective is that
+         * scanning will be reduced if pages are properly placed. As tasks
+         * can enter different phases this needs to be re-examined. Lacking
+         * proper tracking of reference behaviour, this blunt hammer is used.
+         */
+        migrate = mm->numa_next_reset;
+        if (time_after(now, migrate)) {
+                p->numa_scan_period = sysctl_numa_balancing_scan_period_min;
+                next_scan = now + msecs_to_jiffies(sysctl_numa_balancing_scan_period_reset);
+                xchg(&mm->numa_next_reset, next_scan);
+        }
+        /*
+         * Enforce maximal scan/migration frequency..
+         */
+        migrate = mm->numa_next_scan;
+        if (time_before(now, migrate))
+                return;
+        if (p->numa_scan_period == 0)
+                p->numa_scan_period = sysctl_numa_balancing_scan_period_min;
+        next_scan = now + msecs_to_jiffies(p->numa_scan_period);
+        if (cmpxchg(&mm->numa_next_scan, migrate, next_scan) != migrate)
+                return;
+        /*
+         * Do not set pte_numa if the current running node is rate-limited.
+         * This loses statistics on the fault but if we are unwilling to
+         * migrate to this node, it is less likely we can do useful work
+         */
+        if (migrate_ratelimited(numa_node_id()))
+                return;
+        start = mm->numa_scan_offset;
+        pages = sysctl_numa_balancing_scan_size;
+        pages <<= 20 - PAGE_SHIFT; /* MB in pages */
+        if (!pages)
+                return;
+        down_read(&mm->mmap_sem);
+        vma = find_vma(mm, start);
+        if (!vma) {
+                reset_ptenuma_scan(p);
+                start = 0;
+                vma = mm->mmap;
+        }
+        for (; vma; vma = vma->vm_next) {
+                if (!vma_migratable(vma))
+                        continue;
+                /* Skip small VMAs. They are not likely to be of relevance */
+                if (vma->vm_end - vma->vm_start < HPAGE_SIZE)
+                        continue;
+                do {
+                        start = max(start, vma->vm_start);
+                        end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE);
+                        end = min(end, vma->vm_end);
+                        pages -= change_prot_numa(vma, start, end);
+                        start = end;
+                        if (pages <= 0)
+                                goto out;
+                } while (end != vma->vm_end);
+        }
+out:
+        /*
+         * It is possible to reach the end of the VMA list but the last few VMAs are
+         * not guaranteed to the vma_migratable. If they are not, we would find the
+         * !migratable VMA on the next scan but not reset the scanner to the start
+         * so check it now.
+         */
+        if (vma)
+                mm->numa_scan_offset = start;
+        else
+                reset_ptenuma_scan(p);
+        up_read(&mm->mmap_sem);
+}
+/*
+ * Drive the periodic memory faults..
+ */
+void task_tick_numa(struct rq *rq, struct task_struct *curr)
+{
+        struct callback_head *work = &curr->numa_work;
+        u64 period, now;
+        /*
+         * We don't care about NUMA placement if we don't have memory.
+         */
+        if (!curr->mm || (curr->flags & PF_EXITING) || work->next != work)
+                return;
+        /*
+         * Using runtime rather than walltime has the dual advantage that
+         * we (mostly) drive the selection from busy threads and that the
+         * task needs to have done some actual work before we bother with
+         * NUMA placement.
+         */
+        now = curr->se.sum_exec_runtime;
+        period = (u64)curr->numa_scan_period * NSEC_PER_MSEC;
+        if (now - curr->node_stamp > period) {
+                if (!curr->node_stamp)
+                        curr->numa_scan_period = sysctl_numa_balancing_scan_period_min;
+                curr->node_stamp = now;
+                if (!time_before(jiffies, curr->mm->numa_next_scan)) {
+                        init_task_work(work, task_numa_work); /* TODO: move this into sched_fork() */
+                        task_work_add(curr, work, true);
+                }
+        }
+}
+#else
+static void task_tick_numa(struct rq *rq, struct task_struct *curr)
+{
+}
+#endif /* CONFIG_NUMA_BALANCING */
 static void
 account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
@@ -801,72 +1023,7 @@ account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
 }
 #ifdef CONFIG_FAIR_GROUP_SCHED
-/* we need this in update_cfs_load and load-balance functions below */
-static inline int throttled_hierarchy(struct cfs_rq *cfs_rq);
 # ifdef CONFIG_SMP
-static void update_cfs_rq_load_contribution(struct cfs_rq *cfs_rq,
-                                            int global_update)
-{
-        struct task_group *tg = cfs_rq->tg;
-        long load_avg;
-        load_avg = div64_u64(cfs_rq->load_avg, cfs_rq->load_period+1);
-        load_avg -= cfs_rq->load_contribution;
-        if (global_update || abs(load_avg) > cfs_rq->load_contribution / 8) {
-                atomic_add(load_avg, &tg->load_weight);
-                cfs_rq->load_contribution += load_avg;
-        }
-}
-static void update_cfs_load(struct cfs_rq *cfs_rq, int global_update)
-{
-        u64 period = sysctl_sched_shares_window;
-        u64 now, delta;
-        unsigned long load = cfs_rq->load.weight;
-        if (cfs_rq->tg == &root_task_group || throttled_hierarchy(cfs_rq))
-                return;
-        now = rq_of(cfs_rq)->clock_task;
-        delta = now - cfs_rq->load_stamp;
-        /* truncate load history at 4 idle periods */
-        if (cfs_rq->load_stamp > cfs_rq->load_last &&
-            now - cfs_rq->load_last > 4 * period) {
-                cfs_rq->load_period = 0;
-                cfs_rq->load_avg = 0;
-                delta = period - 1;
-        }
-        cfs_rq->load_stamp = now;
-        cfs_rq->load_unacc_exec_time = 0;
-        cfs_rq->load_period += delta;
-        if (load) {
-                cfs_rq->load_last = now;
-                cfs_rq->load_avg += delta * load;
-        }
-        /* consider updating load contribution on each fold or truncate */
-        if (global_update || cfs_rq->load_period > period
-            || !cfs_rq->load_period)
-                update_cfs_rq_load_contribution(cfs_rq, global_update);
-        while (cfs_rq->load_period > period) {
-                /*
-                 * Inline assembly required to prevent the compiler
-                 * optimising this loop into a divmod call.
-                 * See __iter_div_u64_rem() for another example of this.
-                 */
-                asm("" : "+rm" (cfs_rq->load_period));
-                cfs_rq->load_period /= 2;
-                cfs_rq->load_avg /= 2;
-        }
-        if (!cfs_rq->curr && !cfs_rq->nr_running && !cfs_rq->load_avg)
-                list_del_leaf_cfs_rq(cfs_rq);
-}
 static inline long calc_tg_weight(struct task_group *tg, struct cfs_rq *cfs_rq)
 {
        long tg_weight;
@@ -876,8 +1033,8 @@ static inline long calc_tg_weight(struct task_group *tg, struct cfs_rq *cfs_rq)
         * to gain a more accurate current total weight. See
         * update_cfs_rq_load_contribution().
         */
-        tg_weight = atomic_read(&tg->load_weight);
+        tg_weight = atomic64_read(&tg->load_avg);
-        tg_weight -= cfs_rq->load_contribution;
+        tg_weight -= cfs_rq->tg_load_contrib;
        tg_weight += cfs_rq->load.weight;
        return tg_weight;
@@ -901,27 +1058,11 @@ static long calc_cfs_shares(struct cfs_rq *cfs_rq, struct task_group *tg)
        return shares;
 }
-static void update_entity_shares_tick(struct cfs_rq *cfs_rq)
-{
-        if (cfs_rq->load_unacc_exec_time > sysctl_sched_shares_window) {
-                update_cfs_load(cfs_rq, 0);
-                update_cfs_shares(cfs_rq);
-        }
-}
 # else /* CONFIG_SMP */
-static void update_cfs_load(struct cfs_rq *cfs_rq, int global_update)
-{
-}
 static inline long calc_cfs_shares(struct cfs_rq *cfs_rq, struct task_group *tg)
 {
        return tg->shares;
 }
-static inline void update_entity_shares_tick(struct cfs_rq *cfs_rq)
-{
-}
 # endif /* CONFIG_SMP */
 static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se,
                            unsigned long weight)
@@ -939,6 +1080,8 @@ static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se,
                account_entity_enqueue(cfs_rq, se);
 }
+static inline int throttled_hierarchy(struct cfs_rq *cfs_rq);
 static void update_cfs_shares(struct cfs_rq *cfs_rq)
 {
        struct task_group *tg;
@@ -958,18 +1101,477 @@ static void update_cfs_shares(struct cfs_rq *cfs_rq)
        reweight_entity(cfs_rq_of(se), se, shares);
 }
 #else /* CONFIG_FAIR_GROUP_SCHED */
-static void update_cfs_load(struct cfs_rq *cfs_rq, int global_update)
+static inline void update_cfs_shares(struct cfs_rq *cfs_rq)
 {
 }
+#endif /* CONFIG_FAIR_GROUP_SCHED */
-static inline void update_cfs_shares(struct cfs_rq *cfs_rq)
+/* Only depends on SMP, FAIR_GROUP_SCHED may be removed when useful in lb */
+#if defined(CONFIG_SMP) && defined(CONFIG_FAIR_GROUP_SCHED)
+/*
+ * We choose a half-life close to 1 scheduling period.
+ * Note: The tables below are dependent on this value.
+ */
+#define LOAD_AVG_PERIOD 32
+#define LOAD_AVG_MAX 47742 /* maximum possible load avg */
+#define LOAD_AVG_MAX_N 345 /* number of full periods to produce LOAD_MAX_AVG */
+/* Precomputed fixed inverse multiplies for multiplication by y^n */
+static const u32 runnable_avg_yN_inv[] = {
+        0xffffffff, 0xfa83b2da, 0xf5257d14, 0xefe4b99a, 0xeac0c6e6, 0xe5b906e6,
+        0xe0ccdeeb, 0xdbfbb796, 0xd744fcc9, 0xd2a81d91, 0xce248c14, 0xc9b9bd85,
+        0xc5672a10, 0xc12c4cc9, 0xbd08a39e, 0xb8fbaf46, 0xb504f333, 0xb123f581,
+        0xad583ee9, 0xa9a15ab4, 0xa5fed6a9, 0xa2704302, 0x9ef5325f, 0x9b8d39b9,
+        0x9837f050, 0x94f4efa8, 0x91c3d373, 0x8ea4398a, 0x8b95c1e3, 0x88980e80,
+        0x85aac367, 0x82cd8698,
+};
+/*
+ * Precomputed \Sum y^k { 1<=k<=n }.  These are floor(true_value) to prevent
+ * over-estimates when re-combining.
+ */
+static const u32 runnable_avg_yN_sum[] = {
+            0, 1002, 1982, 2941, 3880, 4798, 5697, 6576, 7437, 8279, 9103,
+         9909,10698,11470,12226,12966,13690,14398,15091,15769,16433,17082,
+        17718,18340,18949,19545,20128,20698,21256,21802,22336,22859,23371,
+};
+/*
+ * Approximate:
+ *   val * y^n,    where y^32 ~= 0.5 (~1 scheduling period)
+ */
+static __always_inline u64 decay_load(u64 val, u64 n)
+{
+        unsigned int local_n;
+        if (!n)
+                return val;
+        else if (unlikely(n > LOAD_AVG_PERIOD * 63))
+                return 0;
+        /* after bounds checking we can collapse to 32-bit */
+        local_n = n;
+        /*
+         * As y^PERIOD = 1/2, we can combine
+         *    y^n = 1/2^(n/PERIOD) * k^(n%PERIOD)
+         * With a look-up table which covers k^n (n<PERIOD)
+         *
+         * To achieve constant time decay_load.
+         */
+        if (unlikely(local_n >= LOAD_AVG_PERIOD)) {
+                val >>= local_n / LOAD_AVG_PERIOD;
+                local_n %= LOAD_AVG_PERIOD;
+        }
+        val *= runnable_avg_yN_inv[local_n];
+        /* We don't use SRR here since we always want to round down. */
+        return val >> 32;
+}
+/*
+ * For updates fully spanning n periods, the contribution to runnable
+ * average will be: \Sum 1024*y^n
+ *
+ * We can compute this reasonably efficiently by combining:
+ *   y^PERIOD = 1/2 with precomputed \Sum 1024*y^n {for  n <PERIOD}
+ */
+static u32 __compute_runnable_contrib(u64 n)
 {
+        u32 contrib = 0;
+        if (likely(n <= LOAD_AVG_PERIOD))
+                return runnable_avg_yN_sum[n];
+        else if (unlikely(n >= LOAD_AVG_MAX_N))
+                return LOAD_AVG_MAX;
+        /* Compute \Sum k^n combining precomputed values for k^i, \Sum k^j */
+        do {
+                contrib /= 2; /* y^LOAD_AVG_PERIOD = 1/2 */
+                contrib += runnable_avg_yN_sum[LOAD_AVG_PERIOD];
+                n -= LOAD_AVG_PERIOD;
+        } while (n > LOAD_AVG_PERIOD);
+        contrib = decay_load(contrib, n);
+        return contrib + runnable_avg_yN_sum[n];
+}
+/*
+ * We can represent the historical contribution to runnable average as the
+ * coefficients of a geometric series.  To do this we sub-divide our runnable
+ * history into segments of approximately 1ms (1024us); label the segment that
+ * occurred N-ms ago p_N, with p_0 corresponding to the current period, e.g.
+ *
+ * [<- 1024us ->|<- 1024us ->|<- 1024us ->| ...
+ *      p0            p1           p2
+ *     (now)       (~1ms ago)  (~2ms ago)
+ *
+ * Let u_i denote the fraction of p_i that the entity was runnable.
+ *
+ * We then designate the fractions u_i as our co-efficients, yielding the
+ * following representation of historical load:
+ *   u_0 + u_1*y + u_2*y^2 + u_3*y^3 + ...
+ *
+ * We choose y based on the with of a reasonably scheduling period, fixing:
+ *   y^32 = 0.5
+ *
+ * This means that the contribution to load ~32ms ago (u_32) will be weighted
+ * approximately half as much as the contribution to load within the last ms
+ * (u_0).
+ *
+ * When a period "rolls over" and we have new u_0`, multiplying the previous
+ * sum again by y is sufficient to update:
+ *   load_avg = u_0` + y*(u_0 + u_1*y + u_2*y^2 + ... )
+ *            = u_0 + u_1*y + u_2*y^2 + ... [re-labeling u_i --> u_{i+1}]
+ */
+static __always_inline int __update_entity_runnable_avg(u64 now,
+                                                        struct sched_avg *sa,
+                                                        int runnable)
+{
+        u64 delta, periods;
+        u32 runnable_contrib;
+        int delta_w, decayed = 0;
+        delta = now - sa->last_runnable_update;
+        /*
+         * This should only happen when time goes backwards, which it
+         * unfortunately does during sched clock init when we swap over to TSC.
+         */
+        if ((s64)delta < 0) {
+                sa->last_runnable_update = now;
+                return 0;
+        }
+        /*
+         * Use 1024ns as the unit of measurement since it's a reasonable
+         * approximation of 1us and fast to compute.
+         */
+        delta >>= 10;
+        if (!delta)
+                return 0;
+        sa->last_runnable_update = now;
+        /* delta_w is the amount already accumulated against our next period */
+        delta_w = sa->runnable_avg_period % 1024;
+        if (delta + delta_w >= 1024) {
+                /* period roll-over */
+                decayed = 1;
+                /*
+                 * Now that we know we're crossing a period boundary, figure
+                 * out how much from delta we need to complete the current
+                 * period and accrue it.
+                 */
+                delta_w = 1024 - delta_w;
+                if (runnable)
+                        sa->runnable_avg_sum += delta_w;
+                sa->runnable_avg_period += delta_w;
+                delta -= delta_w;
+                /* Figure out how many additional periods this update spans */
+                periods = delta / 1024;
+                delta %= 1024;
+                sa->runnable_avg_sum = decay_load(sa->runnable_avg_sum,
+                                                  periods + 1);
+                sa->runnable_avg_period = decay_load(sa->runnable_avg_period,
+                                                     periods + 1);
+                /* Efficiently calculate \sum (1..n_period) 1024*y^i */
+                runnable_contrib = __compute_runnable_contrib(periods);
+                if (runnable)
+                        sa->runnable_avg_sum += runnable_contrib;
+                sa->runnable_avg_period += runnable_contrib;
+        }
+        /* Remainder of delta accrued against u_0` */
+        if (runnable)
+                sa->runnable_avg_sum += delta;
+        sa->runnable_avg_period += delta;
+        return decayed;
 }
-static inline void update_entity_shares_tick(struct cfs_rq *cfs_rq)
+/* Synchronize an entity's decay with its parenting cfs_rq.*/
+static inline u64 __synchronize_entity_decay(struct sched_entity *se)
 {
+        struct cfs_rq *cfs_rq = cfs_rq_of(se);
+        u64 decays = atomic64_read(&cfs_rq->decay_counter);
+        decays -= se->avg.decay_count;
+        if (!decays)
+                return 0;
+        se->avg.load_avg_contrib = decay_load(se->avg.load_avg_contrib, decays);
+        se->avg.decay_count = 0;
+        return decays;
+}
+#ifdef CONFIG_FAIR_GROUP_SCHED
+static inline void __update_cfs_rq_tg_load_contrib(struct cfs_rq *cfs_rq,
+                                                 int force_update)
+{
+        struct task_group *tg = cfs_rq->tg;
+        s64 tg_contrib;
+        tg_contrib = cfs_rq->runnable_load_avg + cfs_rq->blocked_load_avg;
+        tg_contrib -= cfs_rq->tg_load_contrib;
+        if (force_update || abs64(tg_contrib) > cfs_rq->tg_load_contrib / 8) {
+                atomic64_add(tg_contrib, &tg->load_avg);
+                cfs_rq->tg_load_contrib += tg_contrib;
+        }
+}
+/*
+ * Aggregate cfs_rq runnable averages into an equivalent task_group
+ * representation for computing load contributions.
+ */
+static inline void __update_tg_runnable_avg(struct sched_avg *sa,
+                                                  struct cfs_rq *cfs_rq)
+{
+        struct task_group *tg = cfs_rq->tg;
+        long contrib;
+        /* The fraction of a cpu used by this cfs_rq */
+        contrib = div_u64(sa->runnable_avg_sum << NICE_0_SHIFT,
+                          sa->runnable_avg_period + 1);
+        contrib -= cfs_rq->tg_runnable_contrib;
+        if (abs(contrib) > cfs_rq->tg_runnable_contrib / 64) {
+                atomic_add(contrib, &tg->runnable_avg);
+                cfs_rq->tg_runnable_contrib += contrib;
+        }
+}
+static inline void __update_group_entity_contrib(struct sched_entity *se)
+{
+        struct cfs_rq *cfs_rq = group_cfs_rq(se);
+        struct task_group *tg = cfs_rq->tg;
+        int runnable_avg;
+        u64 contrib;
+        contrib = cfs_rq->tg_load_contrib * tg->shares;
+        se->avg.load_avg_contrib = div64_u64(contrib,
+                                             atomic64_read(&tg->load_avg) + 1);
+        /*
+         * For group entities we need to compute a correction term in the case
+         * that they are consuming <1 cpu so that we would contribute the same
+         * load as a task of equal weight.
+         *
+         * Explicitly co-ordinating this measurement would be expensive, but
+         * fortunately the sum of each cpus contribution forms a usable
+         * lower-bound on the true value.
+         *
+         * Consider the aggregate of 2 contributions.  Either they are disjoint
+         * (and the sum represents true value) or they are disjoint and we are
+         * understating by the aggregate of their overlap.
+         *
+         * Extending this to N cpus, for a given overlap, the maximum amount we
+         * understand is then n_i(n_i+1)/2 * w_i where n_i is the number of
+         * cpus that overlap for this interval and w_i is the interval width.
+         *
+         * On a small machine; the first term is well-bounded which bounds the
+         * total error since w_i is a subset of the period.  Whereas on a
+         * larger machine, while this first term can be larger, if w_i is the
+         * of consequential size guaranteed to see n_i*w_i quickly converge to
+         * our upper bound of 1-cpu.
+         */
+        runnable_avg = atomic_read(&tg->runnable_avg);
+        if (runnable_avg < NICE_0_LOAD) {
+                se->avg.load_avg_contrib *= runnable_avg;
+                se->avg.load_avg_contrib >>= NICE_0_SHIFT;
+        }
+}
+#else
+static inline void __update_cfs_rq_tg_load_contrib(struct cfs_rq *cfs_rq,
+                                                 int force_update) {}
+static inline void __update_tg_runnable_avg(struct sched_avg *sa,
+                                                  struct cfs_rq *cfs_rq) {}
+static inline void __update_group_entity_contrib(struct sched_entity *se) {}
+#endif
+static inline void __update_task_entity_contrib(struct sched_entity *se)
+{
+        u32 contrib;
+        /* avoid overflowing a 32-bit type w/ SCHED_LOAD_SCALE */
+        contrib = se->avg.runnable_avg_sum * scale_load_down(se->load.weight);
+        contrib /= (se->avg.runnable_avg_period + 1);
+        se->avg.load_avg_contrib = scale_load(contrib);
 }
-#endif /* CONFIG_FAIR_GROUP_SCHED */
+/* Compute the current contribution to load_avg by se, return any delta */
+static long __update_entity_load_avg_contrib(struct sched_entity *se)
+{
+        long old_contrib = se->avg.load_avg_contrib;
+        if (entity_is_task(se)) {
+                __update_task_entity_contrib(se);
+        } else {
+                __update_tg_runnable_avg(&se->avg, group_cfs_rq(se));
+                __update_group_entity_contrib(se);
+        }
+        return se->avg.load_avg_contrib - old_contrib;
+}
+static inline void subtract_blocked_load_contrib(struct cfs_rq *cfs_rq,
+                                                 long load_contrib)
+{
+        if (likely(load_contrib < cfs_rq->blocked_load_avg))
+                cfs_rq->blocked_load_avg -= load_contrib;
+        else
+                cfs_rq->blocked_load_avg = 0;
+}
+static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq);
+/* Update a sched_entity's runnable average */
+static inline void update_entity_load_avg(struct sched_entity *se,
+                                          int update_cfs_rq)
+{
+        struct cfs_rq *cfs_rq = cfs_rq_of(se);
+        long contrib_delta;
+        u64 now;
+        /*
+         * For a group entity we need to use their owned cfs_rq_clock_task() in
+         * case they are the parent of a throttled hierarchy.
+         */
+        if (entity_is_task(se))
+                now = cfs_rq_clock_task(cfs_rq);
+        else
+                now = cfs_rq_clock_task(group_cfs_rq(se));
+        if (!__update_entity_runnable_avg(now, &se->avg, se->on_rq))
+                return;
+        contrib_delta = __update_entity_load_avg_contrib(se);
+        if (!update_cfs_rq)
+                return;
+        if (se->on_rq)
+                cfs_rq->runnable_load_avg += contrib_delta;
+        else
+                subtract_blocked_load_contrib(cfs_rq, -contrib_delta);
+}
+/*
+ * Decay the load contributed by all blocked children and account this so that
+ * their contribution may appropriately discounted when they wake up.
+ */
+static void update_cfs_rq_blocked_load(struct cfs_rq *cfs_rq, int force_update)
+{
+        u64 now = cfs_rq_clock_task(cfs_rq) >> 20;
+        u64 decays;
+        decays = now - cfs_rq->last_decay;
+        if (!decays && !force_update)
+                return;
+        if (atomic64_read(&cfs_rq->removed_load)) {
+                u64 removed_load = atomic64_xchg(&cfs_rq->removed_load, 0);
+                subtract_blocked_load_contrib(cfs_rq, removed_load);
+        }
+        if (decays) {
+                cfs_rq->blocked_load_avg = decay_load(cfs_rq->blocked_load_avg,
+                                                      decays);
+                atomic64_add(decays, &cfs_rq->decay_counter);
+                cfs_rq->last_decay = now;
+        }
+        __update_cfs_rq_tg_load_contrib(cfs_rq, force_update);
+}
+static inline void update_rq_runnable_avg(struct rq *rq, int runnable)
+{
+        __update_entity_runnable_avg(rq->clock_task, &rq->avg, runnable);
+        __update_tg_runnable_avg(&rq->avg, &rq->cfs);
+}
+/* Add the load generated by se into cfs_rq's child load-average */
+static inline void enqueue_entity_load_avg(struct cfs_rq *cfs_rq,
+                                                  struct sched_entity *se,
+                                                  int wakeup)
+{
+        /*
+         * We track migrations using entity decay_count <= 0, on a wake-up
+         * migration we use a negative decay count to track the remote decays
+         * accumulated while sleeping.
+         */
+        if (unlikely(se->avg.decay_count <= 0)) {
+                se->avg.last_runnable_update = rq_of(cfs_rq)->clock_task;
+                if (se->avg.decay_count) {
+                        /*
+                         * In a wake-up migration we have to approximate the
+                         * time sleeping.  This is because we can't synchronize
+                         * clock_task between the two cpus, and it is not
+                         * guaranteed to be read-safe.  Instead, we can
+                         * approximate this using our carried decays, which are
+                         * explicitly atomically readable.
+                         */
+                        se->avg.last_runnable_update -= (-se->avg.decay_count)
+                                                        << 20;
+                        update_entity_load_avg(se, 0);
+                        /* Indicate that we're now synchronized and on-rq */
+                        se->avg.decay_count = 0;
+                }
+                wakeup = 0;
+        } else {
+                __synchronize_entity_decay(se);
+        }
+        /* migrated tasks did not contribute to our blocked load */
+        if (wakeup) {
+                subtract_blocked_load_contrib(cfs_rq, se->avg.load_avg_contrib);
+                update_entity_load_avg(se, 0);
+        }
+        cfs_rq->runnable_load_avg += se->avg.load_avg_contrib;
+        /* we force update consideration on load-balancer moves */
+        update_cfs_rq_blocked_load(cfs_rq, !wakeup);
+}
+/*
+ * Remove se's load from this cfs_rq child load-average, if the entity is
+ * transitioning to a blocked state we track its projected decay using
+ * blocked_load_avg.
+ */
+static inline void dequeue_entity_load_avg(struct cfs_rq *cfs_rq,
+                                                  struct sched_entity *se,
+                                                  int sleep)
+{
+        update_entity_load_avg(se, 1);
+        /* we force update consideration on load-balancer moves */
+        update_cfs_rq_blocked_load(cfs_rq, !sleep);
+        cfs_rq->runnable_load_avg -= se->avg.load_avg_contrib;
+        if (sleep) {
+                cfs_rq->blocked_load_avg += se->avg.load_avg_contrib;
+                se->avg.decay_count = atomic64_read(&cfs_rq->decay_counter);
+        } /* migrations, e.g. sleep=0 leave decay_count == 0 */
+}
+#else
+static inline void update_entity_load_avg(struct sched_entity *se,
+                                          int update_cfs_rq) {}
+static inline void update_rq_runnable_avg(struct rq *rq, int runnable) {}
+static inline void enqueue_entity_load_avg(struct cfs_rq *cfs_rq,
+                                           struct sched_entity *se,
+                                           int wakeup) {}
+static inline void dequeue_entity_load_avg(struct cfs_rq *cfs_rq,
+                                           struct sched_entity *se,
+                                           int sleep) {}
+static inline void update_cfs_rq_blocked_load(struct cfs_rq *cfs_rq,
+                                              int force_update) {}
+#endif
 static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
@@ -1096,7 +1698,7 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
         * Update run-time statistics of the 'current'.
         */
        update_curr(cfs_rq);
-        update_cfs_load(cfs_rq, 0);
+        enqueue_entity_load_avg(cfs_rq, se, flags & ENQUEUE_WAKEUP);
        account_entity_enqueue(cfs_rq, se);
        update_cfs_shares(cfs_rq);
@@ -1171,6 +1773,7 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
         * Update run-time statistics of the 'current'.
         */
        update_curr(cfs_rq);
+        dequeue_entity_load_avg(cfs_rq, se, flags & DEQUEUE_SLEEP);
        update_stats_dequeue(cfs_rq, se);
        if (flags & DEQUEUE_SLEEP) {
@@ -1191,7 +1794,6 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
        if (se != cfs_rq->curr)
                __dequeue_entity(cfs_rq, se);
        se->on_rq = 0;
-        update_cfs_load(cfs_rq, 0);
        account_entity_dequeue(cfs_rq, se);
        /*
@@ -1340,6 +1942,8 @@ static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev)
                update_stats_wait_start(cfs_rq, prev);
                /* Put 'current' back into the tree. */
                __enqueue_entity(cfs_rq, prev);
+                /* in !on_rq case, update occurred at dequeue */
+                update_entity_load_avg(prev, 1);
        }
        cfs_rq->curr = NULL;
 }
@@ -1353,9 +1957,10 @@ entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued)
        update_curr(cfs_rq);
        /*
-         * Update share accounting for long-running entities.
+         * Ensure that runnable average is periodically updated.
         */
-        update_entity_shares_tick(cfs_rq);
+        update_entity_load_avg(curr, 1);
+        update_cfs_rq_blocked_load(cfs_rq, 1);
 #ifdef CONFIG_SCHED_HRTICK
        /*
@@ -1448,6 +2053,15 @@ static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
        return &tg->cfs_bandwidth;
 }
+/* rq->task_clock normalized against any time this cfs_rq has spent throttled */
+static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq)
+{
+        if (unlikely(cfs_rq->throttle_count))
+                return cfs_rq->throttled_clock_task;
+        return rq_of(cfs_rq)->clock_task - cfs_rq->throttled_clock_task_time;
+}
 /* returns 0 on failure to allocate runtime */
 static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq)
 {
@@ -1592,14 +2206,9 @@ static int tg_unthrottle_up(struct task_group *tg, void *data)
        cfs_rq->throttle_count--;
 #ifdef CONFIG_SMP
        if (!cfs_rq->throttle_count) {
-                u64 delta = rq->clock_task - cfs_rq->load_stamp;
+                /* adjust cfs_rq_clock_task() */
+                cfs_rq->throttled_clock_task_time += rq->clock_task -
-                /* leaving throttled state, advance shares averaging windows */
+                                             cfs_rq->throttled_clock_task;
-                cfs_rq->load_stamp += delta;
-                cfs_rq->load_last += delta;
-                /* update entity weight now that we are on_rq again */
-                update_cfs_shares(cfs_rq);
        }
 #endif
@@ -1611,9 +2220,9 @@ static int tg_throttle_down(struct task_group *tg, void *data)
        struct rq *rq = data;
        struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
-        /* group is entering throttled state, record last load */
+        /* group is entering throttled state, stop time */
        if (!cfs_rq->throttle_count)
-                update_cfs_load(cfs_rq, 0);
+                cfs_rq->throttled_clock_task = rq->clock_task;
        cfs_rq->throttle_count++;
        return 0;
@@ -1628,7 +2237,7 @@ static void throttle_cfs_rq(struct cfs_rq *cfs_rq)
        se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))];
-        /* account load preceding throttle */
+        /* freeze hierarchy runnable averages while throttled */
        rcu_read_lock();
        walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq);
        rcu_read_unlock();
@@ -1652,7 +2261,7 @@ static void throttle_cfs_rq(struct cfs_rq *cfs_rq)
                rq->nr_running -= task_delta;
        cfs_rq->throttled = 1;
-        cfs_rq->throttled_timestamp = rq->clock;
+        cfs_rq->throttled_clock = rq->clock;
        raw_spin_lock(&cfs_b->lock);
        list_add_tail_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq);
        raw_spin_unlock(&cfs_b->lock);
@@ -1670,10 +2279,9 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
        cfs_rq->throttled = 0;
        raw_spin_lock(&cfs_b->lock);
-        cfs_b->throttled_time += rq->clock - cfs_rq->throttled_timestamp;
+        cfs_b->throttled_time += rq->clock - cfs_rq->throttled_clock;
        list_del_rcu(&cfs_rq->throttled_list);
        raw_spin_unlock(&cfs_b->lock);
-        cfs_rq->throttled_timestamp = 0;
        update_rq_clock(rq);
        /* update hierarchical throttle state */
@@ -2073,8 +2681,13 @@ static void unthrottle_offline_cfs_rqs(struct rq *rq)
 }
 #else /* CONFIG_CFS_BANDWIDTH */
-static __always_inline
+static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq)
-void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, unsigned long delta_exec) {}
+{
+        return rq_of(cfs_rq)->clock_task;
+}
+static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
+                                     unsigned long delta_exec) {}
 static void check_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
 static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {}
 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
@@ -2207,12 +2820,14 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
                if (cfs_rq_throttled(cfs_rq))
                        break;
-                update_cfs_load(cfs_rq, 0);
                update_cfs_shares(cfs_rq);
+                update_entity_load_avg(se, 1);
        }
-        if (!se)
+        if (!se) {
+                update_rq_runnable_avg(rq, rq->nr_running);
                inc_nr_running(rq);
+        }
        hrtick_update(rq);
 }
@@ -2266,12 +2881,14 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
                if (cfs_rq_throttled(cfs_rq))
                        break;
-                update_cfs_load(cfs_rq, 0);
                update_cfs_shares(cfs_rq);
+                update_entity_load_avg(se, 1);
        }
-        if (!se)
+        if (!se) {
                dec_nr_running(rq);
+                update_rq_runnable_avg(rq, 1);
+        }
        hrtick_update(rq);
 }
@@ -2781,6 +3398,37 @@ unlock:
        return new_cpu;
 }
+/*
+ * Load-tracking only depends on SMP, FAIR_GROUP_SCHED dependency below may be
+ * removed when useful for applications beyond shares distribution (e.g.
+ * load-balance).
+ */
+#ifdef CONFIG_FAIR_GROUP_SCHED
+/*
+ * Called immediately before a task is migrated to a new cpu; task_cpu(p) and
+ * cfs_rq_of(p) references at time of call are still valid and identify the
+ * previous cpu.  However, the caller only guarantees p->pi_lock is held; no
+ * other assumptions, including the state of rq->lock, should be made.
+ */
+static void
+migrate_task_rq_fair(struct task_struct *p, int next_cpu)
+{
+        struct sched_entity *se = &p->se;
+        struct cfs_rq *cfs_rq = cfs_rq_of(se);
+        /*
+         * Load tracking: accumulate removed load so that it can be processed
+         * when we next update owning cfs_rq under rq->lock.  Tasks contribute
+         * to blocked load iff they have a positive decay-count.  It can never
+         * be negative here since on-rq tasks have decay-count == 0.
+         */
+        if (se->avg.decay_count) {
+                se->avg.decay_count = -__synchronize_entity_decay(se);
+                atomic64_add(se->avg.load_avg_contrib, &cfs_rq->removed_load);
+        }
+}
+#endif
 #endif /* CONFIG_SMP */
 static unsigned long
@@ -2907,7 +3555,7 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_
         * Batch and idle tasks do not preempt non-idle tasks (their preemption
         * is driven by the tick):
         */
-        if (unlikely(p->policy != SCHED_NORMAL))
+        if (unlikely(p->policy != SCHED_NORMAL) || !sched_feat(WAKEUP_PREEMPTION))
                return;
        find_matching_se(&se, &pse);
@@ -3033,8 +3681,122 @@ static bool yield_to_task_fair(struct rq *rq, struct task_struct *p, bool preemp
 #ifdef CONFIG_SMP
 /**************************************************
- * Fair scheduling class load-balancing methods:
+ * Fair scheduling class load-balancing methods.
- */
+ *
+ * BASICS
+ *
+ * The purpose of load-balancing is to achieve the same basic fairness the
+ * per-cpu scheduler provides, namely provide a proportional amount of compute
+ * time to each task. This is expressed in the following equation:
+ *
+ *   W_i,n/P_i == W_j,n/P_j for all i,j                               (1)
+ *
+ * Where W_i,n is the n-th weight average for cpu i. The instantaneous weight
+ * W_i,0 is defined as:
+ *
+ *   W_i,0 = \Sum_j w_i,j                                             (2)
+ *
+ * Where w_i,j is the weight of the j-th runnable task on cpu i. This weight
+ * is derived from the nice value as per prio_to_weight[].
+ *
+ * The weight average is an exponential decay average of the instantaneous
+ * weight:
+ *
+ *   W'_i,n = (2^n - 1) / 2^n * W_i,n + 1 / 2^n * W_i,0               (3)
+ *
+ * P_i is the cpu power (or compute capacity) of cpu i, typically it is the
+ * fraction of 'recent' time available for SCHED_OTHER task execution. But it
+ * can also include other factors [XXX].
+ *
+ * To achieve this balance we define a measure of imbalance which follows
+ * directly from (1):
+ *
+ *   imb_i,j = max{ avg(W/P), W_i/P_i } - min{ avg(W/P), W_j/P_j }    (4)
+ *
+ * We them move tasks around to minimize the imbalance. In the continuous
+ * function space it is obvious this converges, in the discrete case we get
+ * a few fun cases generally called infeasible weight scenarios.
+ *
+ * [XXX expand on:
+ *     - infeasible weights;
+ *     - local vs global optima in the discrete case. ]
+ *
+ *
+ * SCHED DOMAINS
+ *
+ * In order to solve the imbalance equation (4), and avoid the obvious O(n^2)
+ * for all i,j solution, we create a tree of cpus that follows the hardware
+ * topology where each level pairs two lower groups (or better). This results
+ * in O(log n) layers. Furthermore we reduce the number of cpus going up the
+ * tree to only the first of the previous level and we decrease the frequency
+ * of load-balance at each level inv. proportional to the number of cpus in
+ * the groups.
+ *
+ * This yields:
+ *
+ *     log_2 n     1     n
+ *   \Sum       { --- * --- * 2^i } = O(n)                            (5)
+ *     i = 0      2^i   2^i
+ *                               `- size of each group
+ *         |         |     `- number of cpus doing load-balance
+ *         |         `- freq
+ *         `- sum over all levels
+ *
+ * Coupled with a limit on how many tasks we can migrate every balance pass,
+ * this makes (5) the runtime complexity of the balancer.
+ *
+ * An important property here is that each CPU is still (indirectly) connected
+ * to every other cpu in at most O(log n) steps:
+ *
+ * The adjacency matrix of the resulting graph is given by:
+ *
+ *             log_2 n     
+ *   A_i,j = \Union     (i % 2^k == 0) && i / 2^(k+1) == j / 2^(k+1)  (6)
+ *             k = 0
+ *
+ * And you'll find that:
+ *
+ *   A^(log_2 n)_i,j != 0  for all i,j                                (7)
+ *
+ * Showing there's indeed a path between every cpu in at most O(log n) steps.
+ * The task movement gives a factor of O(m), giving a convergence complexity
+ * of:
+ *
+ *   O(nm log n),  n := nr_cpus, m := nr_tasks                        (8)
+ *
+ *
+ * WORK CONSERVING
+ *
+ * In order to avoid CPUs going idle while there's still work to do, new idle
+ * balancing is more aggressive and has the newly idle cpu iterate up the domain
+ * tree itself instead of relying on other CPUs to bring it work.
+ *
+ * This adds some complexity to both (5) and (8) but it reduces the total idle
+ * time.
+ *
+ * [XXX more?]
+ *
+ *
+ * CGROUPS
+ *
+ * Cgroups make a horror show out of (2), instead of a simple sum we get:
+ *
+ *                                s_k,i
+ *   W_i,0 = \Sum_j \Prod_k w_k * -----                               (9)
+ *                                 S_k
+ *
+ * Where
+ *
+ *   s_k,i = \Sum_j w_i,j,k  and  S_k = \Sum_i s_k,i                 (10)
+ *
+ * w_i,j,k is the weight of the j-th runnable task in the k-th cgroup on cpu i.
+ *
+ * The big problem is S_k, its a global sum needed to compute a local (W_i)
+ * property.
+ *
+ * [XXX write more on how we solve this.. _after_ merging pjt's patches that
+ *      rewrite all of this once again.]
+ */ 
 static unsigned long __read_mostly max_load_balance_interval = HZ/10;
@@ -3300,52 +4062,58 @@ next:
 /*
 * update tg->load_weight by folding this cpu's load_avg
 */
-static int update_shares_cpu(struct task_group *tg, int cpu)
+static void __update_blocked_averages_cpu(struct task_group *tg, int cpu)
 {
-        struct cfs_rq *cfs_rq;
+        struct sched_entity *se = tg->se[cpu];
-        unsigned long flags;
+        struct cfs_rq *cfs_rq = tg->cfs_rq[cpu];
-        struct rq *rq;
-        if (!tg->se[cpu])
-                return 0;
-        rq = cpu_rq(cpu);
-        cfs_rq = tg->cfs_rq[cpu];
-        raw_spin_lock_irqsave(&rq->lock, flags);
-        update_rq_clock(rq);
-        update_cfs_load(cfs_rq, 1);
-        /*
+        /* throttled entities do not contribute to load */
-         * We need to update shares after updating tg->load_weight in
+        if (throttled_hierarchy(cfs_rq))
-         * order to adjust the weight of groups with long running tasks.
+                return;
-         */
-        update_cfs_shares(cfs_rq);
-        raw_spin_unlock_irqrestore(&rq->lock, flags);
+        update_cfs_rq_blocked_load(cfs_rq, 1);
-        return 0;
+        if (se) {
+                update_entity_load_avg(se, 1);
+                /*
+                 * We pivot on our runnable average having decayed to zero for
+                 * list removal.  This generally implies that all our children
+                 * have also been removed (modulo rounding error or bandwidth
+                 * control); however, such cases are rare and we can fix these
+                 * at enqueue.
+                 *
+                 * TODO: fix up out-of-order children on enqueue.
+                 */
+                if (!se->avg.runnable_avg_sum && !cfs_rq->nr_running)
+                        list_del_leaf_cfs_rq(cfs_rq);
+        } else {
+                struct rq *rq = rq_of(cfs_rq);
+                update_rq_runnable_avg(rq, rq->nr_running);
+        }
 }
-static void update_shares(int cpu)
+static void update_blocked_averages(int cpu)
 {
-        struct cfs_rq *cfs_rq;
        struct rq *rq = cpu_rq(cpu);
+        struct cfs_rq *cfs_rq;
+        unsigned long flags;
-        rcu_read_lock();
+        raw_spin_lock_irqsave(&rq->lock, flags);
+        update_rq_clock(rq);
        /*
         * Iterates the task_group tree in a bottom up fashion, see
         * list_add_leaf_cfs_rq() for details.
         */
        for_each_leaf_cfs_rq(rq, cfs_rq) {
-                /* throttled entities do not contribute to load */
+                /*
-                if (throttled_hierarchy(cfs_rq))
+                 * Note: We may want to consider periodically releasing
-                        continue;
+                 * rq->lock about these updates so that creating many task
+                 * groups does not result in continually extending hold time.
-                update_shares_cpu(cfs_rq->tg, cpu);
+                 */
+                __update_blocked_averages_cpu(cfs_rq->tg, rq->cpu);
        }
-        rcu_read_unlock();
+        raw_spin_unlock_irqrestore(&rq->lock, flags);
 }
 /*
@@ -3397,7 +4165,7 @@ static unsigned long task_h_load(struct task_struct *p)
        return load;
 }
 #else
-static inline void update_shares(int cpu)
+static inline void update_blocked_averages(int cpu)
 {
 }
@@ -4457,12 +5225,14 @@ void idle_balance(int this_cpu, struct rq *this_rq)
        if (this_rq->avg_idle < sysctl_sched_migration_cost)
                return;
+        update_rq_runnable_avg(this_rq, 1);
        /*
         * Drop the rq->lock, but keep IRQ/preempt disabled.
         */
        raw_spin_unlock(&this_rq->lock);
-        update_shares(this_cpu);
+        update_blocked_averages(this_cpu);
        rcu_read_lock();
        for_each_domain(this_cpu, sd) {
                unsigned long interval;
@@ -4717,7 +5487,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
        int update_next_balance = 0;
        int need_serialize;
-        update_shares(cpu);
+        update_blocked_averages(cpu);
        rcu_read_lock();
        for_each_domain(cpu, sd) {
@@ -4954,6 +5724,11 @@ static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued)
                cfs_rq = cfs_rq_of(se);
                entity_tick(cfs_rq, se, queued);
        }
+        if (sched_feat_numa(NUMA))
+                task_tick_numa(rq, curr);
+        update_rq_runnable_avg(rq, 1);
 }
 /*
@@ -5046,6 +5821,20 @@ static void switched_from_fair(struct rq *rq, struct task_struct *p)
                place_entity(cfs_rq, se, 0);
                se->vruntime -= cfs_rq->min_vruntime;
        }
+#if defined(CONFIG_FAIR_GROUP_SCHED) && defined(CONFIG_SMP)
+        /*
+        * Remove our load from contribution when we leave sched_fair
+        * and ensure we don't carry in an old decay_count if we
+        * switch back.
+        */
+        if (p->se.avg.decay_count) {
+                struct cfs_rq *cfs_rq = cfs_rq_of(&p->se);
+                __synchronize_entity_decay(&p->se);
+                subtract_blocked_load_contrib(cfs_rq,
+                                p->se.avg.load_avg_contrib);
+        }
+#endif
 }
 /*
@@ -5092,11 +5881,16 @@ void init_cfs_rq(struct cfs_rq *cfs_rq)
 #ifndef CONFIG_64BIT
        cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime;
 #endif
+#if defined(CONFIG_FAIR_GROUP_SCHED) && defined(CONFIG_SMP)
+        atomic64_set(&cfs_rq->decay_counter, 1);
+        atomic64_set(&cfs_rq->removed_load, 0);
+#endif
 }
 #ifdef CONFIG_FAIR_GROUP_SCHED
 static void task_move_group_fair(struct task_struct *p, int on_rq)
 {
+        struct cfs_rq *cfs_rq;
        /*
         * If the task was not on the rq at the time of this cgroup movement
         * it must have been asleep, sleeping tasks keep their ->vruntime
@@ -5128,8 +5922,19 @@ static void task_move_group_fair(struct task_struct *p, int on_rq)
        if (!on_rq)
                p->se.vruntime -= cfs_rq_of(&p->se)->min_vruntime;
        set_task_rq(p, task_cpu(p));
-        if (!on_rq)
+        if (!on_rq) {
-                p->se.vruntime += cfs_rq_of(&p->se)->min_vruntime;
+                cfs_rq = cfs_rq_of(&p->se);
+                p->se.vruntime += cfs_rq->min_vruntime;
+#ifdef CONFIG_SMP
+                /*
+                 * migrate_task_rq_fair() will have removed our previous
+                 * contribution, but we must synchronize for ongoing future
+                 * decay.
+                 */
+                p->se.avg.decay_count = atomic64_read(&cfs_rq->decay_counter);
+                cfs_rq->blocked_load_avg += p->se.avg.load_avg_contrib;
+#endif
+        }
 }
 void free_fair_sched_group(struct task_group *tg)
@@ -5214,10 +6019,6 @@ void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
        cfs_rq->tg = tg;
        cfs_rq->rq = rq;
-#ifdef CONFIG_SMP
-        /* allow initial update_cfs_load() to truncate */
-        cfs_rq->load_stamp = 1;
-#endif
        init_cfs_rq_runtime(cfs_rq);
        tg->cfs_rq[cpu] = cfs_rq;
@@ -5319,7 +6120,9 @@ const struct sched_class fair_sched_class = {
 #ifdef CONFIG_SMP
        .select_task_rq         = select_task_rq_fair,
+#ifdef CONFIG_FAIR_GROUP_SCHED
+        .migrate_task_rq        = migrate_task_rq_fair,
+#endif
        .rq_online              = rq_online_fair,
        .rq_offline             = rq_offline_fair,
diff --git a/kernel/sched/features.h b/kernel/sched/features.h
index eebefcad7027..1ad1d2b5395f 100644
--- a/kernel/sched/features.h
+++ b/kernel/sched/features.h
@@ -32,6 +32,11 @@ SCHED_FEAT(LAST_BUDDY, true)
 SCHED_FEAT(CACHE_HOT_BUDDY, true)
 /*
+ * Allow wakeup-time preemption of the current task:
+ */
+SCHED_FEAT(WAKEUP_PREEMPTION, true)
+/*
 * Use arch dependent cpu power functions
 */
 SCHED_FEAT(ARCH_POWER, true)
@@ -61,3 +66,14 @@ SCHED_FEAT(TTWU_QUEUE, true)
 SCHED_FEAT(FORCE_SD_OVERLAP, false)
 SCHED_FEAT(RT_RUNTIME_SHARE, true)
 SCHED_FEAT(LB_MIN, false)
+/*
+ * Apply the automatic NUMA scheduling policy. Enabled automatically
+ * at runtime if running on a NUMA machine. Can be controlled via
+ * numa_balancing=. Allow PTE scanning to be forced on UMA machines
+ * for debugging the core machinery.
+ */
+#ifdef CONFIG_NUMA_BALANCING
+SCHED_FEAT(NUMA,        false)
+SCHED_FEAT(NUMA_FORCE,  false)
+#endif
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 7a7db09cfabc..fc886441436a 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -112,6 +112,8 @@ struct task_group {
        unsigned long shares;
        atomic_t load_weight;
+        atomic64_t load_avg;
+        atomic_t runnable_avg;
 #endif
 #ifdef CONFIG_RT_GROUP_SCHED
@@ -222,22 +224,29 @@ struct cfs_rq {
        unsigned int nr_spread_over;
 #endif
+#ifdef CONFIG_SMP
+/*
+ * Load-tracking only depends on SMP, FAIR_GROUP_SCHED dependency below may be
+ * removed when useful for applications beyond shares distribution (e.g.
+ * load-balance).
+ */
 #ifdef CONFIG_FAIR_GROUP_SCHED
-        struct rq *rq;  /* cpu runqueue to which this cfs_rq is attached */
        /*
-         * leaf cfs_rqs are those that hold tasks (lowest schedulable entity in
+         * CFS Load tracking
-         * a hierarchy). Non-leaf lrqs hold other higher schedulable entities
+         * Under CFS, load is tracked on a per-entity basis and aggregated up.
-         * (like users, containers etc.)
+         * This allows for the description of both thread and group usage (in
-         *
+         * the FAIR_GROUP_SCHED case).
-         * leaf_cfs_rq_list ties together list of leaf cfs_rq's in a cpu. This
-         * list is used during load balance.
         */
-        int on_list;
+        u64 runnable_load_avg, blocked_load_avg;
-        struct list_head leaf_cfs_rq_list;
+        atomic64_t decay_counter, removed_load;
-        struct task_group *tg;  /* group that "owns" this runqueue */
+        u64 last_decay;
+#endif /* CONFIG_FAIR_GROUP_SCHED */
+/* These always depend on CONFIG_FAIR_GROUP_SCHED */
+#ifdef CONFIG_FAIR_GROUP_SCHED
+        u32 tg_runnable_contrib;
+        u64 tg_load_contrib;
+#endif /* CONFIG_FAIR_GROUP_SCHED */
-#ifdef CONFIG_SMP
        /*
         *   h_load = weight * f(tg)
         *
@@ -245,26 +254,30 @@ struct cfs_rq {
         * this group.
         */
        unsigned long h_load;
+#endif /* CONFIG_SMP */
+#ifdef CONFIG_FAIR_GROUP_SCHED
+        struct rq *rq;  /* cpu runqueue to which this cfs_rq is attached */
        /*
-         * Maintaining per-cpu shares distribution for group scheduling
+         * leaf cfs_rqs are those that hold tasks (lowest schedulable entity in
+         * a hierarchy). Non-leaf lrqs hold other higher schedulable entities
+         * (like users, containers etc.)
         *
-         * load_stamp is the last time we updated the load average
+         * leaf_cfs_rq_list ties together list of leaf cfs_rq's in a cpu. This
-         * load_last is the last time we updated the load average and saw load
+         * list is used during load balance.
-         * load_unacc_exec_time is currently unaccounted execution time
         */
-        u64 load_avg;
+        int on_list;
-        u64 load_period;
+        struct list_head leaf_cfs_rq_list;
-        u64 load_stamp, load_last, load_unacc_exec_time;
+        struct task_group *tg;  /* group that "owns" this runqueue */
-        unsigned long load_contribution;
-#endif /* CONFIG_SMP */
 #ifdef CONFIG_CFS_BANDWIDTH
        int runtime_enabled;
        u64 runtime_expires;
        s64 runtime_remaining;
-        u64 throttled_timestamp;
+        u64 throttled_clock, throttled_clock_task;
+        u64 throttled_clock_task_time;
        int throttled, throttle_count;
        struct list_head throttled_list;
 #endif /* CONFIG_CFS_BANDWIDTH */
@@ -467,6 +480,8 @@ struct rq {
 #ifdef CONFIG_SMP
        struct llist_head wake_list;
 #endif
+        struct sched_avg avg;
 };
 static inline int cpu_of(struct rq *rq)
@@ -648,6 +663,18 @@ extern struct static_key sched_feat_keys[__SCHED_FEAT_NR];
 #define sched_feat(x) (sysctl_sched_features & (1UL << __SCHED_FEAT_##x))
 #endif /* SCHED_DEBUG && HAVE_JUMP_LABEL */
+#ifdef CONFIG_NUMA_BALANCING
+#define sched_feat_numa(x) sched_feat(x)
+#ifdef CONFIG_SCHED_DEBUG
+#define numabalancing_enabled sched_feat_numa(NUMA)
+#else
+extern bool numabalancing_enabled;
+#endif /* CONFIG_SCHED_DEBUG */
+#else
+#define sched_feat_numa(x) (0)
+#define numabalancing_enabled (0)
+#endif /* CONFIG_NUMA_BALANCING */
 static inline u64 global_rt_period(void)
 {
        return (u64)sysctl_sched_rt_period * NSEC_PER_USEC;
@@ -1212,4 +1239,3 @@ static inline u64 irq_time_read(int cpu)
 }
 #endif /* CONFIG_64BIT */
 #endif /* CONFIG_IRQ_TIME_ACCOUNTING */
diff --git a/kernel/seccomp.c b/kernel/seccomp.c
index ee376beedaf9..5af44b593770 100644
--- a/kernel/seccomp.c
+++ b/kernel/seccomp.c
@@ -396,25 +396,29 @@ int __secure_computing(int this_syscall)
 #ifdef CONFIG_SECCOMP_FILTER
        case SECCOMP_MODE_FILTER: {
                int data;
+                struct pt_regs *regs = task_pt_regs(current);
                ret = seccomp_run_filters(this_syscall);
                data = ret & SECCOMP_RET_DATA;
                ret &= SECCOMP_RET_ACTION;
                switch (ret) {
                case SECCOMP_RET_ERRNO:
                        /* Set the low-order 16-bits as a errno. */
-                        syscall_set_return_value(current, task_pt_regs(current),
+                        syscall_set_return_value(current, regs,
                                                 -data, 0);
                        goto skip;
                case SECCOMP_RET_TRAP:
                        /* Show the handler the original registers. */
-                        syscall_rollback(current, task_pt_regs(current));
+                        syscall_rollback(current, regs);
                        /* Let the filter pass back 16 bits of data. */
                        seccomp_send_sigsys(this_syscall, data);
                        goto skip;
                case SECCOMP_RET_TRACE:
                        /* Skip these calls if there is no tracer. */
-                        if (!ptrace_event_enabled(current, PTRACE_EVENT_SECCOMP))
+                        if (!ptrace_event_enabled(current, PTRACE_EVENT_SECCOMP)) {
+                                syscall_set_return_value(current, regs,
+                                                         -ENOSYS, 0);
                                goto skip;
+                        }
                        /* Allow the BPF to provide the event message */
                        ptrace_event(PTRACE_EVENT_SECCOMP, data);
                        /*
@@ -425,6 +429,9 @@ int __secure_computing(int this_syscall)
                         */
                        if (fatal_signal_pending(current))
                                break;
+                        if (syscall_get_nr(current, regs) < 0)
+                                goto skip;  /* Explicit request to skip. */
                        return 0;
                case SECCOMP_RET_ALLOW:
                        return 0;
diff --git a/kernel/signal.c b/kernel/signal.c
index b2445d86f226..580a91e63471 100644
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -1159,8 +1159,9 @@ static int send_signal(int sig, struct siginfo *info, struct task_struct *t,
        return __send_signal(sig, info, t, group, from_ancestor_ns);
 }
-static void print_fatal_signal(struct pt_regs *regs, int signr)
+static void print_fatal_signal(int signr)
 {
+        struct pt_regs *regs = signal_pt_regs();
        printk("%s/%d: potentially unexpected fatal signal %d.\n",
                current->comm, task_pid_nr(current), signr);
@@ -1908,7 +1909,7 @@ static void ptrace_stop(int exit_code, int why, int clear_code, siginfo_t *info)
                preempt_disable();
                read_unlock(&tasklist_lock);
                preempt_enable_no_resched();
-                schedule();
+                freezable_schedule();
        } else {
                /*
                 * By the time we got the lock, our tracer went away.
@@ -1930,13 +1931,6 @@ static void ptrace_stop(int exit_code, int why, int clear_code, siginfo_t *info)
        }
        /*
-         * While in TASK_TRACED, we were considered "frozen enough".
-         * Now that we woke up, it's crucial if we're supposed to be
-         * frozen that we freeze now before running anything substantial.
-         */
-        try_to_freeze();
-        /*
         * We are back.  Now reacquire the siglock before touching
         * last_siginfo, so that we are sure to have synchronized with
         * any signal-sending on another CPU that wants to examine it.
@@ -2092,7 +2086,7 @@ static bool do_signal_stop(int signr)
                }
                /* Now we don't run again until woken by SIGCONT or SIGKILL */
-                schedule();
+                freezable_schedule();
                return true;
        } else {
                /*
@@ -2138,10 +2132,9 @@ static void do_jobctl_trap(void)
        }
 }
-static int ptrace_signal(int signr, siginfo_t *info,
+static int ptrace_signal(int signr, siginfo_t *info)
-                         struct pt_regs *regs, void *cookie)
 {
-        ptrace_signal_deliver(regs, cookie);
+        ptrace_signal_deliver();
        /*
         * We do not check sig_kernel_stop(signr) but set this marker
         * unconditionally because we do not know whether debugger will
@@ -2200,15 +2193,14 @@ int get_signal_to_deliver(siginfo_t *info, struct k_sigaction *return_ka,
        if (unlikely(uprobe_deny_signal()))
                return 0;
-relock:
        /*
-         * We'll jump back here after any time we were stopped in TASK_STOPPED.
+         * Do this once, we can't return to user-mode if freezing() == T.
-         * While in TASK_STOPPED, we were considered "frozen enough".
+         * do_signal_stop() and ptrace_stop() do freezable_schedule() and
-         * Now that we woke up, it's crucial if we're supposed to be
+         * thus do not need another check after return.
-         * frozen that we freeze now before running anything substantial.
         */
        try_to_freeze();
+relock:
        spin_lock_irq(&sighand->siglock);
        /*
         * Every stopped thread goes here after wakeup. Check to see if
@@ -2265,8 +2257,7 @@ relock:
                        break; /* will return 0 */
                if (unlikely(current->ptrace) && signr != SIGKILL) {
-                        signr = ptrace_signal(signr, info,
+                        signr = ptrace_signal(signr, info);
-                                              regs, cookie);
                        if (!signr)
                                continue;
                }
@@ -2351,7 +2342,7 @@ relock:
                if (sig_kernel_coredump(signr)) {
                        if (print_fatal_signals)
-                                print_fatal_signal(regs, info->si_signo);
+                                print_fatal_signal(info->si_signo);
                        /*
                         * If it was able to dump core, this kills all
                         * other threads in the group and synchronizes with
@@ -2360,7 +2351,7 @@ relock:
                         * first and our do_group_exit call below will use
                         * that value and ignore the one we pass it.
                         */
-                        do_coredump(info, regs);
+                        do_coredump(info);
                }
                /*
diff --git a/kernel/softirq.c b/kernel/softirq.c
index cc96bdc0c2c9..ed567babe789 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -221,7 +221,7 @@ asmlinkage void __do_softirq(void)
        current->flags &= ~PF_MEMALLOC;
        pending = local_softirq_pending();
-        vtime_account(current);
+        vtime_account_irq_enter(current);
        __local_bh_disable((unsigned long)__builtin_return_address(0),
                                SOFTIRQ_OFFSET);
@@ -272,7 +272,7 @@ restart:
        lockdep_softirq_exit();
-        vtime_account(current);
+        vtime_account_irq_exit(current);
        __local_bh_enable(SOFTIRQ_OFFSET);
        tsk_restore_flags(current, old_flags, PF_MEMALLOC);
 }
@@ -341,7 +341,7 @@ static inline void invoke_softirq(void)
 */
 void irq_exit(void)
 {
-        vtime_account(current);
+        vtime_account_irq_exit(current);
        trace_hardirq_exit();
        sub_preempt_count(IRQ_EXIT_OFFSET);
        if (!in_interrupt() && local_softirq_pending())
diff --git a/kernel/srcu.c b/kernel/srcu.c
index 97c465ebd844..2b859828cdc3 100644
--- a/kernel/srcu.c
+++ b/kernel/srcu.c
@@ -16,8 +16,10 @@
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 *
 * Copyright (C) IBM Corporation, 2006
+ * Copyright (C) Fujitsu, 2012
 *
 * Author: Paul McKenney <paulmck@us.ibm.com>
+ *         Lai Jiangshan <laijs@cn.fujitsu.com>
 *
 * For detailed explanation of Read-Copy Update mechanism see -
 *              Documentation/RCU/ *.txt
@@ -34,6 +36,10 @@
 #include <linux/delay.h>
 #include <linux/srcu.h>
+#include <trace/events/rcu.h>
+#include "rcu.h"
 /*
 * Initialize an rcu_batch structure to empty.
 */
@@ -92,9 +98,6 @@ static inline void rcu_batch_move(struct rcu_batch *to, struct rcu_batch *from)
        }
 }
-/* single-thread state-machine */
-static void process_srcu(struct work_struct *work);
 static int init_srcu_struct_fields(struct srcu_struct *sp)
 {
        sp->completed = 0;
@@ -464,7 +467,9 @@ static void __synchronize_srcu(struct srcu_struct *sp, int trycount)
 */
 void synchronize_srcu(struct srcu_struct *sp)
 {
-        __synchronize_srcu(sp, SYNCHRONIZE_SRCU_TRYCOUNT);
+        __synchronize_srcu(sp, rcu_expedited
+                           ? SYNCHRONIZE_SRCU_EXP_TRYCOUNT
+                           : SYNCHRONIZE_SRCU_TRYCOUNT);
 }
 EXPORT_SYMBOL_GPL(synchronize_srcu);
@@ -637,7 +642,7 @@ static void srcu_reschedule(struct srcu_struct *sp)
 /*
 * This is the work-queue function that handles SRCU grace periods.
 */
-static void process_srcu(struct work_struct *work)
+void process_srcu(struct work_struct *work)
 {
        struct srcu_struct *sp;
@@ -648,3 +653,4 @@ static void process_srcu(struct work_struct *work)
        srcu_invoke_callbacks(sp);
        srcu_reschedule(sp);
 }
+EXPORT_SYMBOL_GPL(process_srcu);
diff --git a/kernel/sys.c b/kernel/sys.c
index e6e0ece5f6a0..265b37690421 100644
--- a/kernel/sys.c
+++ b/kernel/sys.c
@@ -1046,7 +1046,7 @@ void do_sys_times(struct tms *tms)
        cputime_t tgutime, tgstime, cutime, cstime;
        spin_lock_irq(&current->sighand->siglock);
-        thread_group_times(current, &tgutime, &tgstime);
+        thread_group_cputime_adjusted(current, &tgutime, &tgstime);
        cutime = current->signal->cutime;
        cstime = current->signal->cstime;
        spin_unlock_irq(&current->sighand->siglock);
@@ -1704,7 +1704,7 @@ static void k_getrusage(struct task_struct *p, int who, struct rusage *r)
        utime = stime = 0;
        if (who == RUSAGE_THREAD) {
-                task_times(current, &utime, &stime);
+                task_cputime_adjusted(current, &utime, &stime);
                accumulate_thread_rusage(p, r);
                maxrss = p->signal->maxrss;
                goto out;
@@ -1730,7 +1730,7 @@ static void k_getrusage(struct task_struct *p, int who, struct rusage *r)
                                break;
                case RUSAGE_SELF:
-                        thread_group_times(p, &tgutime, &tgstime);
+                        thread_group_cputime_adjusted(p, &tgutime, &tgstime);
                        utime += tgutime;
                        stime += tgstime;
                        r->ru_nvcsw += p->signal->nvcsw;
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index 26f65eaa01f9..c88878db491e 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -256,9 +256,11 @@ static int min_sched_granularity_ns = 100000;		/* 100 usecs */
 static int max_sched_granularity_ns = NSEC_PER_SEC;     /* 1 second */
 static int min_wakeup_granularity_ns;                   /* 0 usecs */
 static int max_wakeup_granularity_ns = NSEC_PER_SEC;    /* 1 second */
+#ifdef CONFIG_SMP
 static int min_sched_tunable_scaling = SCHED_TUNABLESCALING_NONE;
 static int max_sched_tunable_scaling = SCHED_TUNABLESCALING_END-1;
-#endif
+#endif /* CONFIG_SMP */
+#endif /* CONFIG_SCHED_DEBUG */
 #ifdef CONFIG_COMPACTION
 static int min_extfrag_threshold;
@@ -301,6 +303,7 @@ static struct ctl_table kern_table[] = {
                .extra1         = &min_wakeup_granularity_ns,
                .extra2         = &max_wakeup_granularity_ns,
        },
+#ifdef CONFIG_SMP
        {
                .procname       = "sched_tunable_scaling",
                .data           = &sysctl_sched_tunable_scaling,
@@ -347,7 +350,45 @@ static struct ctl_table kern_table[] = {
                .extra1         = &zero,
                .extra2         = &one,
        },
-#endif
+#endif /* CONFIG_SMP */
+#ifdef CONFIG_NUMA_BALANCING
+        {
+                .procname       = "numa_balancing_scan_delay_ms",
+                .data           = &sysctl_numa_balancing_scan_delay,
+                .maxlen         = sizeof(unsigned int),
+                .mode           = 0644,
+                .proc_handler   = proc_dointvec,
+        },
+        {
+                .procname       = "numa_balancing_scan_period_min_ms",
+                .data           = &sysctl_numa_balancing_scan_period_min,
+                .maxlen         = sizeof(unsigned int),
+                .mode           = 0644,
+                .proc_handler   = proc_dointvec,
+        },
+        {
+                .procname       = "numa_balancing_scan_period_reset",
+                .data           = &sysctl_numa_balancing_scan_period_reset,
+                .maxlen         = sizeof(unsigned int),
+                .mode           = 0644,
+                .proc_handler   = proc_dointvec,
+        },
+        {
+                .procname       = "numa_balancing_scan_period_max_ms",
+                .data           = &sysctl_numa_balancing_scan_period_max,
+                .maxlen         = sizeof(unsigned int),
+                .mode           = 0644,
+                .proc_handler   = proc_dointvec,
+        },
+        {
+                .procname       = "numa_balancing_scan_size_mb",
+                .data           = &sysctl_numa_balancing_scan_size,
+                .maxlen         = sizeof(unsigned int),
+                .mode           = 0644,
+                .proc_handler   = proc_dointvec,
+        },
+#endif /* CONFIG_NUMA_BALANCING */
+#endif /* CONFIG_SCHED_DEBUG */
        {
                .procname       = "sched_rt_period_us",
                .data           = &sysctl_sched_rt_period,
@@ -565,7 +606,7 @@ static struct ctl_table kern_table[] = {
                .extra2         = &one,
        },
 #endif
-#ifdef CONFIG_HOTPLUG
        {
                .procname       = "hotplug",
                .data           = &uevent_helper,
@@ -573,7 +614,7 @@ static struct ctl_table kern_table[] = {
                .mode           = 0644,
                .proc_handler   = proc_dostring,
        },
-#endif
 #ifdef CONFIG_CHR_DEV_SG
        {
                .procname       = "sg-big-buff",
diff --git a/kernel/time/Makefile b/kernel/time/Makefile
index e2fd74b8e8c2..ff7d9d2ab504 100644
--- a/kernel/time/Makefile
+++ b/kernel/time/Makefile
@@ -1,4 +1,4 @@
-obj-y += timekeeping.o ntp.o clocksource.o jiffies.o timer_list.o timecompare.o
+obj-y += timekeeping.o ntp.o clocksource.o jiffies.o timer_list.o
 obj-y += timeconv.o posix-clock.o alarmtimer.o
 obj-$(CONFIG_GENERIC_CLOCKEVENTS_BUILD)         += clockevents.o
diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c
index 6629bf7b5285..7a925ba456fb 100644
--- a/kernel/time/jiffies.c
+++ b/kernel/time/jiffies.c
@@ -58,7 +58,7 @@ static cycle_t jiffies_read(struct clocksource *cs)
        return (cycle_t) jiffies;
 }
-struct clocksource clocksource_jiffies = {
+static struct clocksource clocksource_jiffies = {
        .name           = "jiffies",
        .rating         = 1, /* lowest valid rating*/
        .read           = jiffies_read,
@@ -67,6 +67,8 @@ struct clocksource clocksource_jiffies = {
        .shift          = JIFFIES_SHIFT,
 };
+__cacheline_aligned_in_smp DEFINE_SEQLOCK(jiffies_lock);
 #if (BITS_PER_LONG < 64)
 u64 get_jiffies_64(void)
 {
@@ -74,9 +76,9 @@ u64 get_jiffies_64(void)
        u64 ret;
        do {
-                seq = read_seqbegin(&xtime_lock);
+                seq = read_seqbegin(&jiffies_lock);
                ret = jiffies_64;
-        } while (read_seqretry(&xtime_lock, seq));
+        } while (read_seqretry(&jiffies_lock, seq));
        return ret;
 }
 EXPORT_SYMBOL(get_jiffies_64);
diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c
index da6c9ecad4e4..b1600a6973f4 100644
--- a/kernel/time/tick-common.c
+++ b/kernel/time/tick-common.c
@@ -63,13 +63,13 @@ int tick_is_oneshot_available(void)
 static void tick_periodic(int cpu)
 {
        if (tick_do_timer_cpu == cpu) {
-                write_seqlock(&xtime_lock);
+                write_seqlock(&jiffies_lock);
                /* Keep track of the next tick event */
                tick_next_period = ktime_add(tick_next_period, tick_period);
                do_timer(1);
-                write_sequnlock(&xtime_lock);
+                write_sequnlock(&jiffies_lock);
        }
        update_process_times(user_mode(get_irq_regs()));
@@ -130,9 +130,9 @@ void tick_setup_periodic(struct clock_event_device *dev, int broadcast)
                ktime_t next;
                do {
-                        seq = read_seqbegin(&xtime_lock);
+                        seq = read_seqbegin(&jiffies_lock);
                        next = tick_next_period;
-                } while (read_seqretry(&xtime_lock, seq));
+                } while (read_seqretry(&jiffies_lock, seq));
                clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT);
diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h
index 4e265b901fed..cf3e59ed6dc0 100644
--- a/kernel/time/tick-internal.h
+++ b/kernel/time/tick-internal.h
@@ -141,4 +141,3 @@ static inline int tick_device_is_functional(struct clock_event_device *dev)
 #endif
 extern void do_timer(unsigned long ticks);
-extern seqlock_t xtime_lock;
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index a40260885265..d58e552d9fd1 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -31,7 +31,7 @@
 static DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched);
 /*
- * The time, when the last jiffy update happened. Protected by xtime_lock.
+ * The time, when the last jiffy update happened. Protected by jiffies_lock.
 */
 static ktime_t last_jiffies_update;
@@ -49,14 +49,14 @@ static void tick_do_update_jiffies64(ktime_t now)
        ktime_t delta;
        /*
-         * Do a quick check without holding xtime_lock:
+         * Do a quick check without holding jiffies_lock:
         */
        delta = ktime_sub(now, last_jiffies_update);
        if (delta.tv64 < tick_period.tv64)
                return;
-        /* Reevalute with xtime_lock held */
+        /* Reevalute with jiffies_lock held */
-        write_seqlock(&xtime_lock);
+        write_seqlock(&jiffies_lock);
        delta = ktime_sub(now, last_jiffies_update);
        if (delta.tv64 >= tick_period.tv64) {
@@ -79,7 +79,7 @@ static void tick_do_update_jiffies64(ktime_t now)
                /* Keep the tick_next_period variable up to date */
                tick_next_period = ktime_add(last_jiffies_update, tick_period);
        }
-        write_sequnlock(&xtime_lock);
+        write_sequnlock(&jiffies_lock);
 }
 /*
@@ -89,15 +89,58 @@ static ktime_t tick_init_jiffy_update(void)
 {
        ktime_t period;
-        write_seqlock(&xtime_lock);
+        write_seqlock(&jiffies_lock);
        /* Did we start the jiffies update yet ? */
        if (last_jiffies_update.tv64 == 0)
                last_jiffies_update = tick_next_period;
        period = last_jiffies_update;
-        write_sequnlock(&xtime_lock);
+        write_sequnlock(&jiffies_lock);
        return period;
 }
+static void tick_sched_do_timer(ktime_t now)
+{
+        int cpu = smp_processor_id();
+#ifdef CONFIG_NO_HZ
+        /*
+         * Check if the do_timer duty was dropped. We don't care about
+         * concurrency: This happens only when the cpu in charge went
+         * into a long sleep. If two cpus happen to assign themself to
+         * this duty, then the jiffies update is still serialized by
+         * jiffies_lock.
+         */
+        if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE))
+                tick_do_timer_cpu = cpu;
+#endif
+        /* Check, if the jiffies need an update */
+        if (tick_do_timer_cpu == cpu)
+                tick_do_update_jiffies64(now);
+}
+static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs)
+{
+#ifdef CONFIG_NO_HZ
+        /*
+         * When we are idle and the tick is stopped, we have to touch
+         * the watchdog as we might not schedule for a really long
+         * time. This happens on complete idle SMP systems while
+         * waiting on the login prompt. We also increment the "start of
+         * idle" jiffy stamp so the idle accounting adjustment we do
+         * when we go busy again does not account too much ticks.
+         */
+        if (ts->tick_stopped) {
+                touch_softlockup_watchdog();
+                if (is_idle_task(current))
+                        ts->idle_jiffies++;
+        }
+#endif
+        update_process_times(user_mode(regs));
+        profile_tick(CPU_PROFILING);
+}
 /*
 * NOHZ - aka dynamic tick functionality
 */
@@ -282,11 +325,11 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
        /* Read jiffies and the time when jiffies were updated last */
        do {
-                seq = read_seqbegin(&xtime_lock);
+                seq = read_seqbegin(&jiffies_lock);
                last_update = last_jiffies_update;
                last_jiffies = jiffies;
                time_delta = timekeeping_max_deferment();
-        } while (read_seqretry(&xtime_lock, seq));
+        } while (read_seqretry(&jiffies_lock, seq));
        if (rcu_needs_cpu(cpu, &rcu_delta_jiffies) || printk_needs_cpu(cpu) ||
            arch_needs_cpu(cpu)) {
@@ -526,6 +569,8 @@ void tick_nohz_irq_exit(void)
        if (!ts->inidle)
                return;
+        /* Cancel the timer because CPU already waken up from the C-states*/
+        menu_hrtimer_cancel();
        __tick_nohz_idle_enter(ts);
 }
@@ -621,6 +666,8 @@ void tick_nohz_idle_exit(void)
        ts->inidle = 0;
+        /* Cancel the timer because CPU already waken up from the C-states*/
+        menu_hrtimer_cancel();
        if (ts->idle_active || ts->tick_stopped)
                now = ktime_get();
@@ -648,40 +695,12 @@ static void tick_nohz_handler(struct clock_event_device *dev)
 {
        struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
        struct pt_regs *regs = get_irq_regs();
-        int cpu = smp_processor_id();
        ktime_t now = ktime_get();
        dev->next_event.tv64 = KTIME_MAX;
-        /*
+        tick_sched_do_timer(now);
-         * Check if the do_timer duty was dropped. We don't care about
+        tick_sched_handle(ts, regs);
-         * concurrency: This happens only when the cpu in charge went
-         * into a long sleep. If two cpus happen to assign themself to
-         * this duty, then the jiffies update is still serialized by
-         * xtime_lock.
-         */
-        if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE))
-                tick_do_timer_cpu = cpu;
-        /* Check, if the jiffies need an update */
-        if (tick_do_timer_cpu == cpu)
-                tick_do_update_jiffies64(now);
-        /*
-         * When we are idle and the tick is stopped, we have to touch
-         * the watchdog as we might not schedule for a really long
-         * time. This happens on complete idle SMP systems while
-         * waiting on the login prompt. We also increment the "start
-         * of idle" jiffy stamp so the idle accounting adjustment we
-         * do when we go busy again does not account too much ticks.
-         */
-        if (ts->tick_stopped) {
-                touch_softlockup_watchdog();
-                ts->idle_jiffies++;
-        }
-        update_process_times(user_mode(regs));
-        profile_tick(CPU_PROFILING);
        while (tick_nohz_reprogram(ts, now)) {
                now = ktime_get();
@@ -794,7 +813,7 @@ void tick_check_idle(int cpu)
 #ifdef CONFIG_HIGH_RES_TIMERS
 /*
 * We rearm the timer until we get disabled by the idle code.
- * Called with interrupts disabled and timer->base->cpu_base->lock held.
+ * Called with interrupts disabled.
 */
 static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
 {
@@ -802,45 +821,15 @@ static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
                container_of(timer, struct tick_sched, sched_timer);
        struct pt_regs *regs = get_irq_regs();
        ktime_t now = ktime_get();
-        int cpu = smp_processor_id();
-#ifdef CONFIG_NO_HZ
+        tick_sched_do_timer(now);
-        /*
-         * Check if the do_timer duty was dropped. We don't care about
-         * concurrency: This happens only when the cpu in charge went
-         * into a long sleep. If two cpus happen to assign themself to
-         * this duty, then the jiffies update is still serialized by
-         * xtime_lock.
-         */
-        if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE))
-                tick_do_timer_cpu = cpu;
-#endif
-        /* Check, if the jiffies need an update */
-        if (tick_do_timer_cpu == cpu)
-                tick_do_update_jiffies64(now);
        /*
         * Do not call, when we are not in irq context and have
         * no valid regs pointer
         */
-        if (regs) {
+        if (regs)
-                /*
+                tick_sched_handle(ts, regs);
-                 * When we are idle and the tick is stopped, we have to touch
-                 * the watchdog as we might not schedule for a really long
-                 * time. This happens on complete idle SMP systems while
-                 * waiting on the login prompt. We also increment the "start of
-                 * idle" jiffy stamp so the idle accounting adjustment we do
-                 * when we go busy again does not account too much ticks.
-                 */
-                if (ts->tick_stopped) {
-                        touch_softlockup_watchdog();
-                        if (is_idle_task(current))
-                                ts->idle_jiffies++;
-                }
-                update_process_times(user_mode(regs));
-                profile_tick(CPU_PROFILING);
-        }
        hrtimer_forward(timer, now, tick_period);
@@ -874,7 +863,7 @@ void tick_setup_sched_timer(void)
        /* Get the next period (per cpu) */
        hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update());
-        /* Offset the tick to avert xtime_lock contention. */
+        /* Offset the tick to avert jiffies_lock contention. */
        if (sched_skew_tick) {
                u64 offset = ktime_to_ns(tick_period) >> 1;
                do_div(offset, num_possible_cpus());
diff --git a/kernel/time/timecompare.c b/kernel/time/timecompare.c
deleted file mode 100644
index a9ae369925ce..000000000000
--- a/kernel/time/timecompare.c
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Copyright (C) 2009 Intel Corporation.
- * Author: Patrick Ohly <patrick.ohly@intel.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- */
-#include <linux/timecompare.h>
-#include <linux/module.h>
-#include <linux/slab.h>
-#include <linux/math64.h>
-#include <linux/kernel.h>
-/*
- * fixed point arithmetic scale factor for skew
- *
- * Usually one would measure skew in ppb (parts per billion, 1e9), but
- * using a factor of 2 simplifies the math.
- */
-#define TIMECOMPARE_SKEW_RESOLUTION (((s64)1)<<30)
-ktime_t timecompare_transform(struct timecompare *sync,
-                              u64 source_tstamp)
-{
-        u64 nsec;
-        nsec = source_tstamp + sync->offset;
-        nsec += (s64)(source_tstamp - sync->last_update) * sync->skew /
-                TIMECOMPARE_SKEW_RESOLUTION;
-        return ns_to_ktime(nsec);
-}
-EXPORT_SYMBOL_GPL(timecompare_transform);
-int timecompare_offset(struct timecompare *sync,
-                       s64 *offset,
-                       u64 *source_tstamp)
-{
-        u64 start_source = 0, end_source = 0;
-        struct {
-                s64 offset;
-                s64 duration_target;
-        } buffer[10], sample, *samples;
-        int counter = 0, i;
-        int used;
-        int index;
-        int num_samples = sync->num_samples;
-        if (num_samples > ARRAY_SIZE(buffer)) {
-                samples = kmalloc(sizeof(*samples) * num_samples, GFP_ATOMIC);
-                if (!samples) {
-                        samples = buffer;
-                        num_samples = ARRAY_SIZE(buffer);
-                }
-        } else {
-                samples = buffer;
-        }
-        /* run until we have enough valid samples, but do not try forever */
-        i = 0;
-        counter = 0;
-        while (1) {
-                u64 ts;
-                ktime_t start, end;
-                start = sync->target();
-                ts = timecounter_read(sync->source);
-                end = sync->target();
-                if (!i)
-                        start_source = ts;
-                /* ignore negative durations */
-                sample.duration_target = ktime_to_ns(ktime_sub(end, start));
-                if (sample.duration_target >= 0) {
-                        /*
-                         * assume symetric delay to and from source:
-                         * average target time corresponds to measured
-                         * source time
-                         */
-                        sample.offset =
-                                (ktime_to_ns(end) + ktime_to_ns(start)) / 2 -
-                                ts;
-                        /* simple insertion sort based on duration */
-                        index = counter - 1;
-                        while (index >= 0) {
-                                if (samples[index].duration_target <
-                                    sample.duration_target)
-                                        break;
-                                samples[index + 1] = samples[index];
-                                index--;
-                        }
-                        samples[index + 1] = sample;
-                        counter++;
-                }
-                i++;
-                if (counter >= num_samples || i >= 100000) {
-                        end_source = ts;
-                        break;
-                }
-        }
-        *source_tstamp = (end_source + start_source) / 2;
-        /* remove outliers by only using 75% of the samples */
-        used = counter * 3 / 4;
-        if (!used)
-                used = counter;
-        if (used) {
-                /* calculate average */
-                s64 off = 0;
-                for (index = 0; index < used; index++)
-                        off += samples[index].offset;
-                *offset = div_s64(off, used);
-        }
-        if (samples && samples != buffer)
-                kfree(samples);
-        return used;
-}
-EXPORT_SYMBOL_GPL(timecompare_offset);
-void __timecompare_update(struct timecompare *sync,
-                          u64 source_tstamp)
-{
-        s64 offset;
-        u64 average_time;
-        if (!timecompare_offset(sync, &offset, &average_time))
-                return;
-        if (!sync->last_update) {
-                sync->last_update = average_time;
-                sync->offset = offset;
-                sync->skew = 0;
-        } else {
-                s64 delta_nsec = average_time - sync->last_update;
-                /* avoid division by negative or small deltas */
-                if (delta_nsec >= 10000) {
-                        s64 delta_offset_nsec = offset - sync->offset;
-                        s64 skew; /* delta_offset_nsec *
-                                     TIMECOMPARE_SKEW_RESOLUTION /
-                                     delta_nsec */
-                        u64 divisor;
-                        /* div_s64() is limited to 32 bit divisor */
-                        skew = delta_offset_nsec * TIMECOMPARE_SKEW_RESOLUTION;
-                        divisor = delta_nsec;
-                        while (unlikely(divisor >= ((s64)1) << 32)) {
-                                /* divide both by 2; beware, right shift
-                                   of negative value has undefined
-                                   behavior and can only be used for
-                                   the positive divisor */
-                                skew = div_s64(skew, 2);
-                                divisor >>= 1;
-                        }
-                        skew = div_s64(skew, divisor);
-                        /*
-                         * Calculate new overall skew as 4/16 the
-                         * old value and 12/16 the new one. This is
-                         * a rather arbitrary tradeoff between
-                         * only using the latest measurement (0/16 and
-                         * 16/16) and even more weight on past measurements.
-                         */
-#define TIMECOMPARE_NEW_SKEW_PER_16 12
-                        sync->skew =
-                                div_s64((16 - TIMECOMPARE_NEW_SKEW_PER_16) *
-                                        sync->skew +
-                                        TIMECOMPARE_NEW_SKEW_PER_16 * skew,
-                                        16);
-                        sync->last_update = average_time;
-                        sync->offset = offset;
-                }
-        }
-}
-EXPORT_SYMBOL_GPL(__timecompare_update);
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index e424970bb562..cbc6acb0db3f 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -21,16 +21,11 @@
 #include <linux/time.h>
 #include <linux/tick.h>
 #include <linux/stop_machine.h>
+#include <linux/pvclock_gtod.h>
 static struct timekeeper timekeeper;
-/*
- * This read-write spinlock protects us from races in SMP while
- * playing with xtime.
- */
-__cacheline_aligned_in_smp DEFINE_SEQLOCK(xtime_lock);
 /* flag for if timekeeping is suspended */
 int __read_mostly timekeeping_suspended;
@@ -180,6 +175,54 @@ static inline s64 timekeeping_get_ns_raw(struct timekeeper *tk)
        return nsec + arch_gettimeoffset();
 }
+static RAW_NOTIFIER_HEAD(pvclock_gtod_chain);
+static void update_pvclock_gtod(struct timekeeper *tk)
+{
+        raw_notifier_call_chain(&pvclock_gtod_chain, 0, tk);
+}
+/**
+ * pvclock_gtod_register_notifier - register a pvclock timedata update listener
+ *
+ * Must hold write on timekeeper.lock
+ */
+int pvclock_gtod_register_notifier(struct notifier_block *nb)
+{
+        struct timekeeper *tk = &timekeeper;
+        unsigned long flags;
+        int ret;
+        write_seqlock_irqsave(&tk->lock, flags);
+        ret = raw_notifier_chain_register(&pvclock_gtod_chain, nb);
+        /* update timekeeping data */
+        update_pvclock_gtod(tk);
+        write_sequnlock_irqrestore(&tk->lock, flags);
+        return ret;
+}
+EXPORT_SYMBOL_GPL(pvclock_gtod_register_notifier);
+/**
+ * pvclock_gtod_unregister_notifier - unregister a pvclock
+ * timedata update listener
+ *
+ * Must hold write on timekeeper.lock
+ */
+int pvclock_gtod_unregister_notifier(struct notifier_block *nb)
+{
+        struct timekeeper *tk = &timekeeper;
+        unsigned long flags;
+        int ret;
+        write_seqlock_irqsave(&tk->lock, flags);
+        ret = raw_notifier_chain_unregister(&pvclock_gtod_chain, nb);
+        write_sequnlock_irqrestore(&tk->lock, flags);
+        return ret;
+}
+EXPORT_SYMBOL_GPL(pvclock_gtod_unregister_notifier);
 /* must hold write on timekeeper.lock */
 static void timekeeping_update(struct timekeeper *tk, bool clearntp)
 {
@@ -188,6 +231,7 @@ static void timekeeping_update(struct timekeeper *tk, bool clearntp)
                ntp_clear();
        }
        update_vsyscall(tk);
+        update_pvclock_gtod(tk);
 }
 /**
@@ -1299,9 +1343,7 @@ struct timespec get_monotonic_coarse(void)
 }
 /*
- * The 64-bit jiffies value is not atomic - you MUST NOT read it
+ * Must hold jiffies_lock
- * without sampling the sequence number in xtime_lock.
- * jiffies is defined in the linker script...
 */
 void do_timer(unsigned long ticks)
 {
@@ -1389,7 +1431,7 @@ EXPORT_SYMBOL_GPL(ktime_get_monotonic_offset);
 */
 void xtime_update(unsigned long ticks)
 {
-        write_seqlock(&xtime_lock);
+        write_seqlock(&jiffies_lock);
        do_timer(ticks);
-        write_sequnlock(&xtime_lock);
+        write_sequnlock(&jiffies_lock);
 }
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index 4cea4f41c1d9..5d89335a485f 100644
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -119,6 +119,7 @@ config TRACING
        select BINARY_PRINTF
        select EVENT_TRACING
        select TRACE_CLOCK
+        select IRQ_WORK
 config GENERIC_TRACER
        bool
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 9dcf15d38380..3ffe4c5ad3f3 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -10,7 +10,7 @@
 * Based on code in the latency_tracer, that is:
 *
 *  Copyright (C) 2004-2006 Ingo Molnar
- *  Copyright (C) 2004 William Lee Irwin III
+ *  Copyright (C) 2004 Nadia Yvette Chambers
 */
 #include <linux/stop_machine.h>
@@ -2437,7 +2437,7 @@ static void reset_iter_read(struct ftrace_iterator *iter)
 {
        iter->pos = 0;
        iter->func_pos = 0;
-        iter->flags &= ~(FTRACE_ITER_PRINTALL & FTRACE_ITER_HASH);
+        iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_HASH);
 }
 static void *t_start(struct seq_file *m, loff_t *pos)
@@ -2675,12 +2675,12 @@ ftrace_notrace_open(struct inode *inode, struct file *file)
 }
 loff_t
-ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
+ftrace_regex_lseek(struct file *file, loff_t offset, int whence)
 {
        loff_t ret;
        if (file->f_mode & FMODE_READ)
-                ret = seq_lseek(file, offset, origin);
+                ret = seq_lseek(file, offset, whence);
        else
                file->f_pos = ret = 1;
@@ -2868,7 +2868,7 @@ static int __init ftrace_mod_cmd_init(void)
 {
        return register_ftrace_command(&ftrace_mod_cmd);
 }
-device_initcall(ftrace_mod_cmd_init);
+core_initcall(ftrace_mod_cmd_init);
 static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip,
                                      struct ftrace_ops *op, struct pt_regs *pt_regs)
@@ -4055,7 +4055,7 @@ static int __init ftrace_nodyn_init(void)
        ftrace_enabled = 1;
        return 0;
 }
-device_initcall(ftrace_nodyn_init);
+core_initcall(ftrace_nodyn_init);
 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
 static inline void ftrace_startup_enable(int command) { }
@@ -4381,7 +4381,7 @@ ftrace_pid_write(struct file *filp, const char __user *ubuf,
        if (strlen(tmp) == 0)
                return 1;
-        ret = strict_strtol(tmp, 10, &val);
+        ret = kstrtol(tmp, 10, &val);
        if (ret < 0)
                return ret;
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index b979426d16c6..ce8514feedcd 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -460,9 +460,10 @@ struct ring_buffer_per_cpu {
        unsigned long                   lost_events;
        unsigned long                   last_overrun;
        local_t                         entries_bytes;
-        local_t                         commit_overrun;
-        local_t                         overrun;
        local_t                         entries;
+        local_t                         overrun;
+        local_t                         commit_overrun;
+        local_t                         dropped_events;
        local_t                         committing;
        local_t                         commits;
        unsigned long                   read;
@@ -1396,6 +1397,8 @@ rb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer)
                struct list_head *head_page_with_bit;
                head_page = &rb_set_head_page(cpu_buffer)->list;
+                if (!head_page)
+                        break;
                prev_page = head_page->prev;
                first_page = pages->next;
@@ -1820,7 +1823,7 @@ rb_add_time_stamp(struct ring_buffer_event *event, u64 delta)
 }
 /**
- * ring_buffer_update_event - update event type and data
+ * rb_update_event - update event type and data
 * @event: the even to update
 * @type: the type of event
 * @length: the size of the event field in the ring buffer
@@ -2155,8 +2158,10 @@ rb_move_tail(struct ring_buffer_per_cpu *cpu_buffer,
                         * If we are not in overwrite mode,
                         * this is easy, just stop here.
                         */
-                        if (!(buffer->flags & RB_FL_OVERWRITE))
+                        if (!(buffer->flags & RB_FL_OVERWRITE)) {
+                                local_inc(&cpu_buffer->dropped_events);
                                goto out_reset;
+                        }
                        ret = rb_handle_head_page(cpu_buffer,
                                                  tail_page,
@@ -2720,8 +2725,8 @@ EXPORT_SYMBOL_GPL(ring_buffer_discard_commit);
 * and not the length of the event which would hold the header.
 */
 int ring_buffer_write(struct ring_buffer *buffer,
-                        unsigned long length,
+                      unsigned long length,
-                        void *data)
+                      void *data)
 {
        struct ring_buffer_per_cpu *cpu_buffer;
        struct ring_buffer_event *event;
@@ -2929,12 +2934,12 @@ rb_num_of_entries(struct ring_buffer_per_cpu *cpu_buffer)
 * @buffer: The ring buffer
 * @cpu: The per CPU buffer to read from.
 */
-unsigned long ring_buffer_oldest_event_ts(struct ring_buffer *buffer, int cpu)
+u64 ring_buffer_oldest_event_ts(struct ring_buffer *buffer, int cpu)
 {
        unsigned long flags;
        struct ring_buffer_per_cpu *cpu_buffer;
        struct buffer_page *bpage;
-        unsigned long ret;
+        u64 ret = 0;
        if (!cpumask_test_cpu(cpu, buffer->cpumask))
                return 0;
@@ -2949,7 +2954,8 @@ unsigned long ring_buffer_oldest_event_ts(struct ring_buffer *buffer, int cpu)
                bpage = cpu_buffer->reader_page;
        else
                bpage = rb_set_head_page(cpu_buffer);
-        ret = bpage->page->time_stamp;
+        if (bpage)
+                ret = bpage->page->time_stamp;
        raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
        return ret;
@@ -2995,7 +3001,8 @@ unsigned long ring_buffer_entries_cpu(struct ring_buffer *buffer, int cpu)
 EXPORT_SYMBOL_GPL(ring_buffer_entries_cpu);
 /**
- * ring_buffer_overrun_cpu - get the number of overruns in a cpu_buffer
+ * ring_buffer_overrun_cpu - get the number of overruns caused by the ring
+ * buffer wrapping around (only if RB_FL_OVERWRITE is on).
 * @buffer: The ring buffer
 * @cpu: The per CPU buffer to get the number of overruns from
 */
@@ -3015,7 +3022,9 @@ unsigned long ring_buffer_overrun_cpu(struct ring_buffer *buffer, int cpu)
 EXPORT_SYMBOL_GPL(ring_buffer_overrun_cpu);
 /**
- * ring_buffer_commit_overrun_cpu - get the number of overruns caused by commits
+ * ring_buffer_commit_overrun_cpu - get the number of overruns caused by
+ * commits failing due to the buffer wrapping around while there are uncommitted
+ * events, such as during an interrupt storm.
 * @buffer: The ring buffer
 * @cpu: The per CPU buffer to get the number of overruns from
 */
@@ -3036,6 +3045,28 @@ ring_buffer_commit_overrun_cpu(struct ring_buffer *buffer, int cpu)
 EXPORT_SYMBOL_GPL(ring_buffer_commit_overrun_cpu);
 /**
+ * ring_buffer_dropped_events_cpu - get the number of dropped events caused by
+ * the ring buffer filling up (only if RB_FL_OVERWRITE is off).
+ * @buffer: The ring buffer
+ * @cpu: The per CPU buffer to get the number of overruns from
+ */
+unsigned long
+ring_buffer_dropped_events_cpu(struct ring_buffer *buffer, int cpu)
+{
+        struct ring_buffer_per_cpu *cpu_buffer;
+        unsigned long ret;
+        if (!cpumask_test_cpu(cpu, buffer->cpumask))
+                return 0;
+        cpu_buffer = buffer->buffers[cpu];
+        ret = local_read(&cpu_buffer->dropped_events);
+        return ret;
+}
+EXPORT_SYMBOL_GPL(ring_buffer_dropped_events_cpu);
+/**
 * ring_buffer_entries - get the number of entries in a buffer
 * @buffer: The ring buffer
 *
@@ -3260,6 +3291,8 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
         * Splice the empty reader page into the list around the head.
         */
        reader = rb_set_head_page(cpu_buffer);
+        if (!reader)
+                goto out;
        cpu_buffer->reader_page->list.next = rb_list_head(reader->list.next);
        cpu_buffer->reader_page->list.prev = reader->list.prev;
@@ -3778,12 +3811,17 @@ void
 ring_buffer_read_finish(struct ring_buffer_iter *iter)
 {
        struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
+        unsigned long flags;
        /*
         * Ring buffer is disabled from recording, here's a good place
-         * to check the integrity of the ring buffer. 
+         * to check the integrity of the ring buffer.
+         * Must prevent readers from trying to read, as the check
+         * clears the HEAD page and readers require it.
         */
+        raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
        rb_check_pages(cpu_buffer);
+        raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
        atomic_dec(&cpu_buffer->record_disabled);
        atomic_dec(&cpu_buffer->buffer->resize_disabled);
@@ -3864,9 +3902,10 @@ rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer)
        local_set(&cpu_buffer->reader_page->page->commit, 0);
        cpu_buffer->reader_page->read = 0;
-        local_set(&cpu_buffer->commit_overrun, 0);
        local_set(&cpu_buffer->entries_bytes, 0);
        local_set(&cpu_buffer->overrun, 0);
+        local_set(&cpu_buffer->commit_overrun, 0);
+        local_set(&cpu_buffer->dropped_events, 0);
        local_set(&cpu_buffer->entries, 0);
        local_set(&cpu_buffer->committing, 0);
        local_set(&cpu_buffer->commits, 0);
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 31e4f55773f1..61e081b4ba11 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -9,7 +9,7 @@
 *
 * Based on code from the latency_tracer, that is:
 *  Copyright (C) 2004-2006 Ingo Molnar
- *  Copyright (C) 2004 William Lee Irwin III
+ *  Copyright (C) 2004 Nadia Yvette Chambers
 */
 #include <linux/ring_buffer.h>
 #include <generated/utsrelease.h>
@@ -19,6 +19,7 @@
 #include <linux/seq_file.h>
 #include <linux/notifier.h>
 #include <linux/irqflags.h>
+#include <linux/irq_work.h>
 #include <linux/debugfs.h>
 #include <linux/pagemap.h>
 #include <linux/hardirq.h>
@@ -78,6 +79,21 @@ static int dummy_set_flag(u32 old_flags, u32 bit, int set)
 }
 /*
+ * To prevent the comm cache from being overwritten when no
+ * tracing is active, only save the comm when a trace event
+ * occurred.
+ */
+static DEFINE_PER_CPU(bool, trace_cmdline_save);
+/*
+ * When a reader is waiting for data, then this variable is
+ * set to true.
+ */
+static bool trace_wakeup_needed;
+static struct irq_work trace_work_wakeup;
+/*
 * Kill all tracing for good (never come back).
 * It is initialized to 1 but will turn to zero if the initialization
 * of the tracer is successful. But that is the only place that sets
@@ -139,6 +155,18 @@ static int __init set_ftrace_dump_on_oops(char *str)
 }
 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
+static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
+static char *trace_boot_options __initdata;
+static int __init set_trace_boot_options(char *str)
+{
+        strncpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
+        trace_boot_options = trace_boot_options_buf;
+        return 0;
+}
+__setup("trace_options=", set_trace_boot_options);
 unsigned long long ns2usecs(cycle_t nsec)
 {
        nsec += 500;
@@ -198,20 +226,9 @@ static struct trace_array	max_tr;
 static DEFINE_PER_CPU(struct trace_array_cpu, max_tr_data);
-/* tracer_enabled is used to toggle activation of a tracer */
-static int                      tracer_enabled = 1;
-/**
- * tracing_is_enabled - return tracer_enabled status
- *
- * This function is used by other tracers to know the status
- * of the tracer_enabled flag.  Tracers may use this function
- * to know if it should enable their features when starting
- * up. See irqsoff tracer for an example (start_irqsoff_tracer).
- */
 int tracing_is_enabled(void)
 {
-        return tracer_enabled;
+        return tracing_is_on();
 }
 /*
@@ -333,12 +350,18 @@ unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
 static int trace_stop_count;
 static DEFINE_RAW_SPINLOCK(tracing_start_lock);
-static void wakeup_work_handler(struct work_struct *work)
+/**
+ * trace_wake_up - wake up tasks waiting for trace input
+ *
+ * Schedules a delayed work to wake up any task that is blocked on the
+ * trace_wait queue. These is used with trace_poll for tasks polling the
+ * trace.
+ */
+static void trace_wake_up(struct irq_work *work)
 {
-        wake_up(&trace_wait);
+        wake_up_all(&trace_wait);
-}
-static DECLARE_DELAYED_WORK(wakeup_work, wakeup_work_handler);
+}
 /**
 * tracing_on - enable tracing buffers
@@ -393,22 +416,6 @@ int tracing_is_on(void)
 }
 EXPORT_SYMBOL_GPL(tracing_is_on);
-/**
- * trace_wake_up - wake up tasks waiting for trace input
- *
- * Schedules a delayed work to wake up any task that is blocked on the
- * trace_wait queue. These is used with trace_poll for tasks polling the
- * trace.
- */
-void trace_wake_up(void)
-{
-        const unsigned long delay = msecs_to_jiffies(2);
-        if (trace_flags & TRACE_ITER_BLOCK)
-                return;
-        schedule_delayed_work(&wakeup_work, delay);
-}
 static int __init set_buf_size(char *str)
 {
        unsigned long buf_size;
@@ -431,7 +438,7 @@ static int __init set_tracing_thresh(char *str)
        if (!str)
                return 0;
-        ret = strict_strtoul(str, 0, &threshold);
+        ret = kstrtoul(str, 0, &threshold);
        if (ret < 0)
                return 0;
        tracing_thresh = threshold * 1000;
@@ -477,10 +484,12 @@ static const char *trace_options[] = {
 static struct {
        u64 (*func)(void);
        const char *name;
+        int in_ns;              /* is this clock in nanoseconds? */
 } trace_clocks[] = {
-        { trace_clock_local,    "local" },
+        { trace_clock_local,    "local",        1 },
-        { trace_clock_global,   "global" },
+        { trace_clock_global,   "global",       1 },
-        { trace_clock_counter,  "counter" },
+        { trace_clock_counter,  "counter",      0 },
+        ARCH_TRACE_CLOCKS
 };
 int trace_clock_id;
@@ -757,6 +766,40 @@ update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
 }
 #endif /* CONFIG_TRACER_MAX_TRACE */
+static void default_wait_pipe(struct trace_iterator *iter)
+{
+        DEFINE_WAIT(wait);
+        prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
+        /*
+         * The events can happen in critical sections where
+         * checking a work queue can cause deadlocks.
+         * After adding a task to the queue, this flag is set
+         * only to notify events to try to wake up the queue
+         * using irq_work.
+         *
+         * We don't clear it even if the buffer is no longer
+         * empty. The flag only causes the next event to run
+         * irq_work to do the work queue wake up. The worse
+         * that can happen if we race with !trace_empty() is that
+         * an event will cause an irq_work to try to wake up
+         * an empty queue.
+         *
+         * There's no reason to protect this flag either, as
+         * the work queue and irq_work logic will do the necessary
+         * synchronization for the wake ups. The only thing
+         * that is necessary is that the wake up happens after
+         * a task has been queued. It's OK for spurious wake ups.
+         */
+        trace_wakeup_needed = true;
+        if (trace_empty(iter))
+                schedule();
+        finish_wait(&trace_wait, &wait);
+}
 /**
 * register_tracer - register a tracer with the ftrace system.
 * @type - the plugin for the tracer
@@ -875,32 +918,6 @@ int register_tracer(struct tracer *type)
        return ret;
 }
-void unregister_tracer(struct tracer *type)
-{
-        struct tracer **t;
-        mutex_lock(&trace_types_lock);
-        for (t = &trace_types; *t; t = &(*t)->next) {
-                if (*t == type)
-                        goto found;
-        }
-        pr_info("Tracer %s not registered\n", type->name);
-        goto out;
- found:
-        *t = (*t)->next;
-        if (type == current_trace && tracer_enabled) {
-                tracer_enabled = 0;
-                tracing_stop();
-                if (current_trace->stop)
-                        current_trace->stop(&global_trace);
-                current_trace = &nop_trace;
-        }
-out:
-        mutex_unlock(&trace_types_lock);
-}
 void tracing_reset(struct trace_array *tr, int cpu)
 {
        struct ring_buffer *buffer = tr->buffer;
@@ -1131,10 +1148,14 @@ void trace_find_cmdline(int pid, char comm[])
 void tracing_record_cmdline(struct task_struct *tsk)
 {
-        if (atomic_read(&trace_record_cmdline_disabled) || !tracer_enabled ||
+        if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
-            !tracing_is_on())
                return;
+        if (!__this_cpu_read(trace_cmdline_save))
+                return;
+        __this_cpu_write(trace_cmdline_save, false);
        trace_save_cmdline(tsk);
 }
@@ -1178,27 +1199,36 @@ trace_buffer_lock_reserve(struct ring_buffer *buffer,
        return event;
 }
+void
+__buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
+{
+        __this_cpu_write(trace_cmdline_save, true);
+        if (trace_wakeup_needed) {
+                trace_wakeup_needed = false;
+                /* irq_work_queue() supplies it's own memory barriers */
+                irq_work_queue(&trace_work_wakeup);
+        }
+        ring_buffer_unlock_commit(buffer, event);
+}
 static inline void
 __trace_buffer_unlock_commit(struct ring_buffer *buffer,
                             struct ring_buffer_event *event,
-                             unsigned long flags, int pc,
+                             unsigned long flags, int pc)
-                             int wake)
 {
-        ring_buffer_unlock_commit(buffer, event);
+        __buffer_unlock_commit(buffer, event);
        ftrace_trace_stack(buffer, flags, 6, pc);
        ftrace_trace_userstack(buffer, flags, pc);
-        if (wake)
-                trace_wake_up();
 }
 void trace_buffer_unlock_commit(struct ring_buffer *buffer,
                                struct ring_buffer_event *event,
                                unsigned long flags, int pc)
 {
-        __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
+        __trace_buffer_unlock_commit(buffer, event, flags, pc);
 }
+EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
 struct ring_buffer_event *
 trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
@@ -1215,29 +1245,21 @@ void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
                                        struct ring_buffer_event *event,
                                        unsigned long flags, int pc)
 {
-        __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
+        __trace_buffer_unlock_commit(buffer, event, flags, pc);
 }
 EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
-void trace_nowake_buffer_unlock_commit(struct ring_buffer *buffer,
+void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
-                                       struct ring_buffer_event *event,
+                                     struct ring_buffer_event *event,
-                                       unsigned long flags, int pc)
+                                     unsigned long flags, int pc,
+                                     struct pt_regs *regs)
 {
-        __trace_buffer_unlock_commit(buffer, event, flags, pc, 0);
+        __buffer_unlock_commit(buffer, event);
-}
-EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit);
-void trace_nowake_buffer_unlock_commit_regs(struct ring_buffer *buffer,
-                                            struct ring_buffer_event *event,
-                                            unsigned long flags, int pc,
-                                            struct pt_regs *regs)
-{
-        ring_buffer_unlock_commit(buffer, event);
        ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
        ftrace_trace_userstack(buffer, flags, pc);
 }
-EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit_regs);
+EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
 void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
                                         struct ring_buffer_event *event)
@@ -1269,7 +1291,7 @@ trace_function(struct trace_array *tr,
        entry->parent_ip                = parent_ip;
        if (!filter_check_discard(call, entry, buffer, event))
-                ring_buffer_unlock_commit(buffer, event);
+                __buffer_unlock_commit(buffer, event);
 }
 void
@@ -1362,7 +1384,7 @@ static void __ftrace_trace_stack(struct ring_buffer *buffer,
        entry->size = trace.nr_entries;
        if (!filter_check_discard(call, entry, buffer, event))
-                ring_buffer_unlock_commit(buffer, event);
+                __buffer_unlock_commit(buffer, event);
 out:
        /* Again, don't let gcc optimize things here */
@@ -1458,7 +1480,7 @@ ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
        save_stack_trace_user(&trace);
        if (!filter_check_discard(call, entry, buffer, event))
-                ring_buffer_unlock_commit(buffer, event);
+                __buffer_unlock_commit(buffer, event);
 out_drop_count:
        __this_cpu_dec(user_stack_count);
@@ -1559,10 +1581,10 @@ static int alloc_percpu_trace_buffer(void)
        return -ENOMEM;
 }
+static int buffers_allocated;
 void trace_printk_init_buffers(void)
 {
-        static int buffers_allocated;
        if (buffers_allocated)
                return;
@@ -1571,7 +1593,38 @@ void trace_printk_init_buffers(void)
        pr_info("ftrace: Allocated trace_printk buffers\n");
+        /* Expand the buffers to set size */
+        tracing_update_buffers();
        buffers_allocated = 1;
+        /*
+         * trace_printk_init_buffers() can be called by modules.
+         * If that happens, then we need to start cmdline recording
+         * directly here. If the global_trace.buffer is already
+         * allocated here, then this was called by module code.
+         */
+        if (global_trace.buffer)
+                tracing_start_cmdline_record();
+}
+void trace_printk_start_comm(void)
+{
+        /* Start tracing comms if trace printk is set */
+        if (!buffers_allocated)
+                return;
+        tracing_start_cmdline_record();
+}
+static void trace_printk_start_stop_comm(int enabled)
+{
+        if (!buffers_allocated)
+                return;
+        if (enabled)
+                tracing_start_cmdline_record();
+        else
+                tracing_stop_cmdline_record();
 }
 /**
@@ -1622,7 +1675,7 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
        memcpy(entry->buf, tbuffer, sizeof(u32) * len);
        if (!filter_check_discard(call, entry, buffer, event)) {
-                ring_buffer_unlock_commit(buffer, event);
+                __buffer_unlock_commit(buffer, event);
                ftrace_trace_stack(buffer, flags, 6, pc);
        }
@@ -1693,7 +1746,7 @@ int trace_array_vprintk(struct trace_array *tr,
        memcpy(&entry->buf, tbuffer, len);
        entry->buf[len] = '\0';
        if (!filter_check_discard(call, entry, buffer, event)) {
-                ring_buffer_unlock_commit(buffer, event);
+                __buffer_unlock_commit(buffer, event);
                ftrace_trace_stack(buffer, flags, 6, pc);
        }
 out:
@@ -2426,6 +2479,10 @@ __tracing_open(struct inode *inode, struct file *file)
        if (ring_buffer_overruns(iter->tr->buffer))
                iter->iter_flags |= TRACE_FILE_ANNOTATE;
+        /* Output in nanoseconds only if we are using a clock in nanoseconds. */
+        if (trace_clocks[trace_clock_id].in_ns)
+                iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
        /* stop the trace while dumping */
        tracing_stop();
@@ -2794,26 +2851,19 @@ static void set_tracer_flags(unsigned int mask, int enabled)
        if (mask == TRACE_ITER_OVERWRITE)
                ring_buffer_change_overwrite(global_trace.buffer, enabled);
+        if (mask == TRACE_ITER_PRINTK)
+                trace_printk_start_stop_comm(enabled);
 }
-static ssize_t
+static int trace_set_options(char *option)
-tracing_trace_options_write(struct file *filp, const char __user *ubuf,
-                        size_t cnt, loff_t *ppos)
 {
-        char buf[64];
        char *cmp;
        int neg = 0;
-        int ret;
+        int ret = 0;
        int i;
-        if (cnt >= sizeof(buf))
+        cmp = strstrip(option);
-                return -EINVAL;
-        if (copy_from_user(&buf, ubuf, cnt))
-                return -EFAULT;
-        buf[cnt] = 0;
-        cmp = strstrip(buf);
        if (strncmp(cmp, "no", 2) == 0) {
                neg = 1;
@@ -2832,10 +2882,25 @@ tracing_trace_options_write(struct file *filp, const char __user *ubuf,
                mutex_lock(&trace_types_lock);
                ret = set_tracer_option(current_trace, cmp, neg);
                mutex_unlock(&trace_types_lock);
-                if (ret)
-                        return ret;
        }
+        return ret;
+}
+static ssize_t
+tracing_trace_options_write(struct file *filp, const char __user *ubuf,
+                        size_t cnt, loff_t *ppos)
+{
+        char buf[64];
+        if (cnt >= sizeof(buf))
+                return -EINVAL;
+        if (copy_from_user(&buf, ubuf, cnt))
+                return -EFAULT;
+        trace_set_options(buf);
        *ppos += cnt;
        return cnt;
@@ -2940,56 +3005,6 @@ static const struct file_operations tracing_saved_cmdlines_fops = {
 };
 static ssize_t
-tracing_ctrl_read(struct file *filp, char __user *ubuf,
-                  size_t cnt, loff_t *ppos)
-{
-        char buf[64];
-        int r;
-        r = sprintf(buf, "%u\n", tracer_enabled);
-        return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
-}
-static ssize_t
-tracing_ctrl_write(struct file *filp, const char __user *ubuf,
-                   size_t cnt, loff_t *ppos)
-{
-        struct trace_array *tr = filp->private_data;
-        unsigned long val;
-        int ret;
-        ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
-        if (ret)
-                return ret;
-        val = !!val;
-        mutex_lock(&trace_types_lock);
-        if (tracer_enabled ^ val) {
-                /* Only need to warn if this is used to change the state */
-                WARN_ONCE(1, "tracing_enabled is deprecated. Use tracing_on");
-                if (val) {
-                        tracer_enabled = 1;
-                        if (current_trace->start)
-                                current_trace->start(tr);
-                        tracing_start();
-                } else {
-                        tracer_enabled = 0;
-                        tracing_stop();
-                        if (current_trace->stop)
-                                current_trace->stop(tr);
-                }
-        }
-        mutex_unlock(&trace_types_lock);
-        *ppos += cnt;
-        return cnt;
-}
-static ssize_t
 tracing_set_trace_read(struct file *filp, char __user *ubuf,
                       size_t cnt, loff_t *ppos)
 {
@@ -3030,6 +3045,10 @@ static int __tracing_resize_ring_buffer(unsigned long size, int cpu)
         */
        ring_buffer_expanded = 1;
+        /* May be called before buffers are initialized */
+        if (!global_trace.buffer)
+                return 0;
        ret = ring_buffer_resize(global_trace.buffer, size, cpu);
        if (ret < 0)
                return ret;
@@ -3325,6 +3344,10 @@ static int tracing_open_pipe(struct inode *inode, struct file *filp)
        if (trace_flags & TRACE_ITER_LATENCY_FMT)
                iter->iter_flags |= TRACE_FILE_LAT_FMT;
+        /* Output in nanoseconds only if we are using a clock in nanoseconds. */
+        if (trace_clocks[trace_clock_id].in_ns)
+                iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
        iter->cpu_file = cpu_file;
        iter->tr = &global_trace;
        mutex_init(&iter->mutex);
@@ -3385,19 +3408,6 @@ tracing_poll_pipe(struct file *filp, poll_table *poll_table)
        }
 }
-void default_wait_pipe(struct trace_iterator *iter)
-{
-        DEFINE_WAIT(wait);
-        prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
-        if (trace_empty(iter))
-                schedule();
-        finish_wait(&trace_wait, &wait);
-}
 /*
 * This is a make-shift waitqueue.
 * A tracer might use this callback on some rare cases:
@@ -3438,7 +3448,7 @@ static int tracing_wait_pipe(struct file *filp)
                        return -EINTR;
                /*
-                 * We block until we read something and tracing is disabled.
+                 * We block until we read something and tracing is enabled.
                 * We still block if tracing is disabled, but we have never
                 * read anything. This allows a user to cat this file, and
                 * then enable tracing. But after we have read something,
@@ -3446,7 +3456,7 @@ static int tracing_wait_pipe(struct file *filp)
                 *
                 * iter->pos will be 0 if we haven't read anything.
                 */
-                if (!tracer_enabled && iter->pos)
+                if (tracing_is_enabled() && iter->pos)
                        break;
        }
@@ -3955,7 +3965,7 @@ tracing_mark_write(struct file *filp, const char __user *ubuf,
        } else
                entry->buf[cnt] = '\0';
-        ring_buffer_unlock_commit(buffer, event);
+        __buffer_unlock_commit(buffer, event);
        written = cnt;
@@ -4016,6 +4026,14 @@ static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
        if (max_tr.buffer)
                ring_buffer_set_clock(max_tr.buffer, trace_clocks[i].func);
+        /*
+         * New clock may not be consistent with the previous clock.
+         * Reset the buffer so that it doesn't have incomparable timestamps.
+         */
+        tracing_reset_online_cpus(&global_trace);
+        if (max_tr.buffer)
+                tracing_reset_online_cpus(&max_tr);
        mutex_unlock(&trace_types_lock);
        *fpos += cnt;
@@ -4037,13 +4055,6 @@ static const struct file_operations tracing_max_lat_fops = {
        .llseek         = generic_file_llseek,
 };
-static const struct file_operations tracing_ctrl_fops = {
-        .open           = tracing_open_generic,
-        .read           = tracing_ctrl_read,
-        .write          = tracing_ctrl_write,
-        .llseek         = generic_file_llseek,
-};
 static const struct file_operations set_tracer_fops = {
        .open           = tracing_open_generic,
        .read           = tracing_set_trace_read,
@@ -4377,13 +4388,27 @@ tracing_stats_read(struct file *filp, char __user *ubuf,
        cnt = ring_buffer_bytes_cpu(tr->buffer, cpu);
        trace_seq_printf(s, "bytes: %ld\n", cnt);
-        t = ns2usecs(ring_buffer_oldest_event_ts(tr->buffer, cpu));
+        if (trace_clocks[trace_clock_id].in_ns) {
-        usec_rem = do_div(t, USEC_PER_SEC);
+                /* local or global for trace_clock */
-        trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n", t, usec_rem);
+                t = ns2usecs(ring_buffer_oldest_event_ts(tr->buffer, cpu));
+                usec_rem = do_div(t, USEC_PER_SEC);
+                trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
+                                                                t, usec_rem);
+                t = ns2usecs(ring_buffer_time_stamp(tr->buffer, cpu));
+                usec_rem = do_div(t, USEC_PER_SEC);
+                trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
+        } else {
+                /* counter or tsc mode for trace_clock */
+                trace_seq_printf(s, "oldest event ts: %llu\n",
+                                ring_buffer_oldest_event_ts(tr->buffer, cpu));
+                trace_seq_printf(s, "now ts: %llu\n",
+                                ring_buffer_time_stamp(tr->buffer, cpu));
+        }
-        t = ns2usecs(ring_buffer_time_stamp(tr->buffer, cpu));
+        cnt = ring_buffer_dropped_events_cpu(tr->buffer, cpu);
-        usec_rem = do_div(t, USEC_PER_SEC);
+        trace_seq_printf(s, "dropped events: %ld\n", cnt);
-        trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
        count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
@@ -4815,9 +4840,6 @@ static __init int tracer_init_debugfs(void)
        d_tracer = tracing_init_dentry();
-        trace_create_file("tracing_enabled", 0644, d_tracer,
-                        &global_trace, &tracing_ctrl_fops);
        trace_create_file("trace_options", 0644, d_tracer,
                        NULL, &tracing_iter_fops);
@@ -5089,6 +5111,7 @@ __init static int tracer_alloc_buffers(void)
        /* Only allocate trace_printk buffers if a trace_printk exists */
        if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
+                /* Must be called before global_trace.buffer is allocated */
                trace_printk_init_buffers();
        /* To save memory, keep the ring buffer size to its minimum */
@@ -5136,6 +5159,7 @@ __init static int tracer_alloc_buffers(void)
 #endif
        trace_init_cmdlines();
+        init_irq_work(&trace_work_wakeup, trace_wake_up);
        register_tracer(&nop_trace);
        current_trace = &nop_trace;
@@ -5147,6 +5171,13 @@ __init static int tracer_alloc_buffers(void)
        register_die_notifier(&trace_die_notifier);
+        while (trace_boot_options) {
+                char *option;
+                option = strsep(&trace_boot_options, ",");
+                trace_set_options(option);
+        }
        return 0;
 out_free_cpumask:
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index c15f528c1af4..c75d7988902c 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -285,8 +285,8 @@ struct tracer {
        int                     (*set_flag)(u32 old_flags, u32 bit, int set);
        struct tracer           *next;
        struct tracer_flags     *flags;
-        int                     print_max;
+        bool                    print_max;
-        int                     use_max_tr;
+        bool                    use_max_tr;
 };
@@ -327,7 +327,6 @@ trace_buffer_iter(struct trace_iterator *iter, int cpu)
 int tracer_init(struct tracer *t, struct trace_array *tr);
 int tracing_is_enabled(void);
-void trace_wake_up(void);
 void tracing_reset(struct trace_array *tr, int cpu);
 void tracing_reset_online_cpus(struct trace_array *tr);
 void tracing_reset_current(int cpu);
@@ -349,9 +348,6 @@ trace_buffer_lock_reserve(struct ring_buffer *buffer,
                          unsigned long len,
                          unsigned long flags,
                          int pc);
-void trace_buffer_unlock_commit(struct ring_buffer *buffer,
-                                struct ring_buffer_event *event,
-                                unsigned long flags, int pc);
 struct trace_entry *tracing_get_trace_entry(struct trace_array *tr,
                                                struct trace_array_cpu *data);
@@ -359,6 +355,9 @@ struct trace_entry *tracing_get_trace_entry(struct trace_array *tr,
 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
                                          int *ent_cpu, u64 *ent_ts);
+void __buffer_unlock_commit(struct ring_buffer *buffer,
+                            struct ring_buffer_event *event);
 int trace_empty(struct trace_iterator *iter);
 void *trace_find_next_entry_inc(struct trace_iterator *iter);
@@ -367,7 +366,6 @@ void trace_init_global_iter(struct trace_iterator *iter);
 void tracing_iter_reset(struct trace_iterator *iter, int cpu);
-void default_wait_pipe(struct trace_iterator *iter);
 void poll_wait_pipe(struct trace_iterator *iter);
 void ftrace(struct trace_array *tr,
@@ -407,12 +405,7 @@ void tracing_sched_switch_assign_trace(struct trace_array *tr);
 void tracing_stop_sched_switch_record(void);
 void tracing_start_sched_switch_record(void);
 int register_tracer(struct tracer *type);
-void unregister_tracer(struct tracer *type);
 int is_tracing_stopped(void);
-enum trace_file_type {
-        TRACE_FILE_LAT_FMT      = 1,
-        TRACE_FILE_ANNOTATE     = 2,
-};
 extern cpumask_var_t __read_mostly tracing_buffer_mask;
@@ -841,6 +834,7 @@ extern const char *__start___trace_bprintk_fmt[];
 extern const char *__stop___trace_bprintk_fmt[];
 void trace_printk_init_buffers(void);
+void trace_printk_start_comm(void);
 #undef FTRACE_ENTRY
 #define FTRACE_ENTRY(call, struct_name, id, tstruct, print, filter)     \
diff --git a/kernel/trace/trace_branch.c b/kernel/trace/trace_branch.c
index 8d3538b4ea5f..95e96842ed29 100644
--- a/kernel/trace/trace_branch.c
+++ b/kernel/trace/trace_branch.c
@@ -77,7 +77,7 @@ probe_likely_condition(struct ftrace_branch_data *f, int val, int expect)
        entry->correct = val == expect;
        if (!filter_check_discard(call, entry, buffer, event))
-                ring_buffer_unlock_commit(buffer, event);
+                __buffer_unlock_commit(buffer, event);
 out:
        atomic_dec(&tr->data[cpu]->disabled);
@@ -199,7 +199,7 @@ __init static int init_branch_tracer(void)
        }
        return register_tracer(&branch_trace);
 }
-device_initcall(init_branch_tracer);
+core_initcall(init_branch_tracer);
 #else
 static inline
diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c
index d608d09d08c0..880073d0b946 100644
--- a/kernel/trace/trace_events.c
+++ b/kernel/trace/trace_events.c
@@ -491,19 +491,6 @@ static void t_stop(struct seq_file *m, void *p)
        mutex_unlock(&event_mutex);
 }
-static int
-ftrace_event_seq_open(struct inode *inode, struct file *file)
-{
-        const struct seq_operations *seq_ops;
-        if ((file->f_mode & FMODE_WRITE) &&
-            (file->f_flags & O_TRUNC))
-                ftrace_clear_events();
-        seq_ops = inode->i_private;
-        return seq_open(file, seq_ops);
-}
 static ssize_t
 event_enable_read(struct file *filp, char __user *ubuf, size_t cnt,
                  loff_t *ppos)
@@ -980,6 +967,9 @@ show_header(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
        return r;
 }
+static int ftrace_event_avail_open(struct inode *inode, struct file *file);
+static int ftrace_event_set_open(struct inode *inode, struct file *file);
 static const struct seq_operations show_event_seq_ops = {
        .start = t_start,
        .next = t_next,
@@ -995,14 +985,14 @@ static const struct seq_operations show_set_event_seq_ops = {
 };
 static const struct file_operations ftrace_avail_fops = {
-        .open = ftrace_event_seq_open,
+        .open = ftrace_event_avail_open,
        .read = seq_read,
        .llseek = seq_lseek,
        .release = seq_release,
 };
 static const struct file_operations ftrace_set_event_fops = {
-        .open = ftrace_event_seq_open,
+        .open = ftrace_event_set_open,
        .read = seq_read,
        .write = ftrace_event_write,
        .llseek = seq_lseek,
@@ -1078,6 +1068,26 @@ static struct dentry *event_trace_events_dir(void)
        return d_events;
 }
+static int
+ftrace_event_avail_open(struct inode *inode, struct file *file)
+{
+        const struct seq_operations *seq_ops = &show_event_seq_ops;
+        return seq_open(file, seq_ops);
+}
+static int
+ftrace_event_set_open(struct inode *inode, struct file *file)
+{
+        const struct seq_operations *seq_ops = &show_set_event_seq_ops;
+        if ((file->f_mode & FMODE_WRITE) &&
+            (file->f_flags & O_TRUNC))
+                ftrace_clear_events();
+        return seq_open(file, seq_ops);
+}
 static struct dentry *
 event_subsystem_dir(const char *name, struct dentry *d_events)
 {
@@ -1489,6 +1499,9 @@ static __init int event_trace_enable(void)
                if (ret)
                        pr_warn("Failed to enable trace event: %s\n", token);
        }
+        trace_printk_start_comm();
        return 0;
 }
@@ -1505,15 +1518,13 @@ static __init int event_trace_init(void)
                return 0;
        entry = debugfs_create_file("available_events", 0444, d_tracer,
-                                    (void *)&show_event_seq_ops,
+                                    NULL, &ftrace_avail_fops);
-                                    &ftrace_avail_fops);
        if (!entry)
                pr_warning("Could not create debugfs "
                           "'available_events' entry\n");
        entry = debugfs_create_file("set_event", 0644, d_tracer,
-                                    (void *)&show_set_event_seq_ops,
+                                    NULL, &ftrace_set_event_fops);
-                                    &ftrace_set_event_fops);
        if (!entry)
                pr_warning("Could not create debugfs "
                           "'set_event' entry\n");
@@ -1749,7 +1760,7 @@ function_test_events_call(unsigned long ip, unsigned long parent_ip,
        entry->ip                       = ip;
        entry->parent_ip                = parent_ip;
-        trace_nowake_buffer_unlock_commit(buffer, event, flags, pc);
+        trace_buffer_unlock_commit(buffer, event, flags, pc);
 out:
        atomic_dec(&per_cpu(ftrace_test_event_disable, cpu));
diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c
index c154797a7ff7..e5b0ca8b8d4d 100644
--- a/kernel/trace/trace_events_filter.c
+++ b/kernel/trace/trace_events_filter.c
@@ -1000,9 +1000,9 @@ static int init_pred(struct filter_parse_state *ps,
                }
        } else {
                if (field->is_signed)
-                        ret = strict_strtoll(pred->regex.pattern, 0, &val);
+                        ret = kstrtoll(pred->regex.pattern, 0, &val);
                else
-                        ret = strict_strtoull(pred->regex.pattern, 0, &val);
+                        ret = kstrtoull(pred->regex.pattern, 0, &val);
                if (ret) {
                        parse_error(ps, FILT_ERR_ILLEGAL_INTVAL, 0);
                        return -EINVAL;
diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c
index 507a7a9630bf..8e3ad8082ab7 100644
--- a/kernel/trace/trace_functions.c
+++ b/kernel/trace/trace_functions.c
@@ -7,7 +7,7 @@
 * Based on code from the latency_tracer, that is:
 *
 *  Copyright (C) 2004-2006 Ingo Molnar
- *  Copyright (C) 2004 William Lee Irwin III
+ *  Copyright (C) 2004 Nadia Yvette Chambers
 */
 #include <linux/ring_buffer.h>
 #include <linux/debugfs.h>
@@ -366,7 +366,7 @@ ftrace_trace_onoff_callback(struct ftrace_hash *hash,
         * We use the callback data field (which is a pointer)
         * as our counter.
         */
-        ret = strict_strtoul(number, 0, (unsigned long *)&count);
+        ret = kstrtoul(number, 0, (unsigned long *)&count);
        if (ret)
                return ret;
@@ -411,5 +411,4 @@ static __init int init_function_trace(void)
        init_func_cmd_traceon();
        return register_tracer(&function_trace);
 }
-device_initcall(init_function_trace);
+core_initcall(init_function_trace);
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c
index 99b4378393d5..4edb4b74eb7e 100644
--- a/kernel/trace/trace_functions_graph.c
+++ b/kernel/trace/trace_functions_graph.c
@@ -223,7 +223,7 @@ int __trace_graph_entry(struct trace_array *tr,
        entry   = ring_buffer_event_data(event);
        entry->graph_ent                        = *trace;
        if (!filter_current_check_discard(buffer, call, entry, event))
-                ring_buffer_unlock_commit(buffer, event);
+                __buffer_unlock_commit(buffer, event);
        return 1;
 }
@@ -327,7 +327,7 @@ void __trace_graph_return(struct trace_array *tr,
        entry   = ring_buffer_event_data(event);
        entry->ret                              = *trace;
        if (!filter_current_check_discard(buffer, call, entry, event))
-                ring_buffer_unlock_commit(buffer, event);
+                __buffer_unlock_commit(buffer, event);
 }
 void trace_graph_return(struct ftrace_graph_ret *trace)
@@ -1474,4 +1474,4 @@ static __init int init_graph_trace(void)
        return register_tracer(&graph_trace);
 }
-device_initcall(init_graph_trace);
+core_initcall(init_graph_trace);
diff --git a/kernel/trace/trace_irqsoff.c b/kernel/trace/trace_irqsoff.c
index d98ee8283b29..713a2cac4881 100644
--- a/kernel/trace/trace_irqsoff.c
+++ b/kernel/trace/trace_irqsoff.c
@@ -7,7 +7,7 @@
 * From code in the latency_tracer, that is:
 *
 *  Copyright (C) 2004-2006 Ingo Molnar
- *  Copyright (C) 2004 William Lee Irwin III
+ *  Copyright (C) 2004 Nadia Yvette Chambers
 */
 #include <linux/kallsyms.h>
 #include <linux/debugfs.h>
@@ -604,7 +604,7 @@ static struct tracer irqsoff_tracer __read_mostly =
        .reset          = irqsoff_tracer_reset,
        .start          = irqsoff_tracer_start,
        .stop           = irqsoff_tracer_stop,
-        .print_max      = 1,
+        .print_max      = true,
        .print_header   = irqsoff_print_header,
        .print_line     = irqsoff_print_line,
        .flags          = &tracer_flags,
@@ -614,7 +614,7 @@ static struct tracer irqsoff_tracer __read_mostly =
 #endif
        .open           = irqsoff_trace_open,
        .close          = irqsoff_trace_close,
-        .use_max_tr     = 1,
+        .use_max_tr     = true,
 };
 # define register_irqsoff(trace) register_tracer(&trace)
 #else
@@ -637,7 +637,7 @@ static struct tracer preemptoff_tracer __read_mostly =
        .reset          = irqsoff_tracer_reset,
        .start          = irqsoff_tracer_start,
        .stop           = irqsoff_tracer_stop,
-        .print_max      = 1,
+        .print_max      = true,
        .print_header   = irqsoff_print_header,
        .print_line     = irqsoff_print_line,
        .flags          = &tracer_flags,
@@ -647,7 +647,7 @@ static struct tracer preemptoff_tracer __read_mostly =
 #endif
        .open           = irqsoff_trace_open,
        .close          = irqsoff_trace_close,
-        .use_max_tr     = 1,
+        .use_max_tr     = true,
 };
 # define register_preemptoff(trace) register_tracer(&trace)
 #else
@@ -672,7 +672,7 @@ static struct tracer preemptirqsoff_tracer __read_mostly =
        .reset          = irqsoff_tracer_reset,
        .start          = irqsoff_tracer_start,
        .stop           = irqsoff_tracer_stop,
-        .print_max      = 1,
+        .print_max      = true,
        .print_header   = irqsoff_print_header,
        .print_line     = irqsoff_print_line,
        .flags          = &tracer_flags,
@@ -682,7 +682,7 @@ static struct tracer preemptirqsoff_tracer __read_mostly =
 #endif
        .open           = irqsoff_trace_open,
        .close          = irqsoff_trace_close,
-        .use_max_tr     = 1,
+        .use_max_tr     = true,
 };
 # define register_preemptirqsoff(trace) register_tracer(&trace)
@@ -698,4 +698,4 @@ __init static int init_irqsoff_tracer(void)
        return 0;
 }
-device_initcall(init_irqsoff_tracer);
+core_initcall(init_irqsoff_tracer);
diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c
index 1a2117043bb1..1865d5f76538 100644
--- a/kernel/trace/trace_kprobe.c
+++ b/kernel/trace/trace_kprobe.c
@@ -444,7 +444,7 @@ static int create_trace_probe(int argc, char **argv)
                        return -EINVAL;
                }
                /* an address specified */
-                ret = strict_strtoul(&argv[1][0], 0, (unsigned long *)&addr);
+                ret = kstrtoul(&argv[1][0], 0, (unsigned long *)&addr);
                if (ret) {
                        pr_info("Failed to parse address.\n");
                        return ret;
@@ -751,8 +751,8 @@ static __kprobes void kprobe_trace_func(struct kprobe *kp, struct pt_regs *regs)
        store_trace_args(sizeof(*entry), tp, regs, (u8 *)&entry[1], dsize);
        if (!filter_current_check_discard(buffer, call, entry, event))
-                trace_nowake_buffer_unlock_commit_regs(buffer, event,
+                trace_buffer_unlock_commit_regs(buffer, event,
-                                                       irq_flags, pc, regs);
+                                                irq_flags, pc, regs);
 }
 /* Kretprobe handler */
@@ -784,8 +784,8 @@ static __kprobes void kretprobe_trace_func(struct kretprobe_instance *ri,
        store_trace_args(sizeof(*entry), tp, regs, (u8 *)&entry[1], dsize);
        if (!filter_current_check_discard(buffer, call, entry, event))
-                trace_nowake_buffer_unlock_commit_regs(buffer, event,
+                trace_buffer_unlock_commit_regs(buffer, event,
-                                                       irq_flags, pc, regs);
+                                                irq_flags, pc, regs);
 }
 /* Event entry printers */
diff --git a/kernel/trace/trace_output.c b/kernel/trace/trace_output.c
index 123b189c732c..194d79602dc7 100644
--- a/kernel/trace/trace_output.c
+++ b/kernel/trace/trace_output.c
@@ -610,24 +610,54 @@ lat_print_generic(struct trace_seq *s, struct trace_entry *entry, int cpu)
        return trace_print_lat_fmt(s, entry);
 }
-static unsigned long preempt_mark_thresh = 100;
+static unsigned long preempt_mark_thresh_us = 100;
 static int
-lat_print_timestamp(struct trace_seq *s, u64 abs_usecs,
+lat_print_timestamp(struct trace_iterator *iter, u64 next_ts)
-                    unsigned long rel_usecs)
 {
-        return trace_seq_printf(s, " %4lldus%c: ", abs_usecs,
+        unsigned long verbose = trace_flags & TRACE_ITER_VERBOSE;
-                                rel_usecs > preempt_mark_thresh ? '!' :
+        unsigned long in_ns = iter->iter_flags & TRACE_FILE_TIME_IN_NS;
-                                  rel_usecs > 1 ? '+' : ' ');
+        unsigned long long abs_ts = iter->ts - iter->tr->time_start;
+        unsigned long long rel_ts = next_ts - iter->ts;
+        struct trace_seq *s = &iter->seq;
+        if (in_ns) {
+                abs_ts = ns2usecs(abs_ts);
+                rel_ts = ns2usecs(rel_ts);
+        }
+        if (verbose && in_ns) {
+                unsigned long abs_usec = do_div(abs_ts, USEC_PER_MSEC);
+                unsigned long abs_msec = (unsigned long)abs_ts;
+                unsigned long rel_usec = do_div(rel_ts, USEC_PER_MSEC);
+                unsigned long rel_msec = (unsigned long)rel_ts;
+                return trace_seq_printf(
+                                s, "[%08llx] %ld.%03ldms (+%ld.%03ldms): ",
+                                ns2usecs(iter->ts),
+                                abs_msec, abs_usec,
+                                rel_msec, rel_usec);
+        } else if (verbose && !in_ns) {
+                return trace_seq_printf(
+                                s, "[%016llx] %lld (+%lld): ",
+                                iter->ts, abs_ts, rel_ts);
+        } else if (!verbose && in_ns) {
+                return trace_seq_printf(
+                                s, " %4lldus%c: ",
+                                abs_ts,
+                                rel_ts > preempt_mark_thresh_us ? '!' :
+                                  rel_ts > 1 ? '+' : ' ');
+        } else { /* !verbose && !in_ns */
+                return trace_seq_printf(s, " %4lld: ", abs_ts);
+        }
 }
 int trace_print_context(struct trace_iterator *iter)
 {
        struct trace_seq *s = &iter->seq;
        struct trace_entry *entry = iter->ent;
-        unsigned long long t = ns2usecs(iter->ts);
+        unsigned long long t;
-        unsigned long usec_rem = do_div(t, USEC_PER_SEC);
+        unsigned long secs, usec_rem;
-        unsigned long secs = (unsigned long)t;
        char comm[TASK_COMM_LEN];
        int ret;
@@ -644,8 +674,13 @@ int trace_print_context(struct trace_iterator *iter)
                        return 0;
        }
-        return trace_seq_printf(s, " %5lu.%06lu: ",
+        if (iter->iter_flags & TRACE_FILE_TIME_IN_NS) {
-                                secs, usec_rem);
+                t = ns2usecs(iter->ts);
+                usec_rem = do_div(t, USEC_PER_SEC);
+                secs = (unsigned long)t;
+                return trace_seq_printf(s, " %5lu.%06lu: ", secs, usec_rem);
+        } else
+                return trace_seq_printf(s, " %12llu: ", iter->ts);
 }
 int trace_print_lat_context(struct trace_iterator *iter)
@@ -659,36 +694,29 @@ int trace_print_lat_context(struct trace_iterator *iter)
                           *next_entry = trace_find_next_entry(iter, NULL,
                                                               &next_ts);
        unsigned long verbose = (trace_flags & TRACE_ITER_VERBOSE);
-        unsigned long abs_usecs = ns2usecs(iter->ts - iter->tr->time_start);
-        unsigned long rel_usecs;
        /* Restore the original ent_size */
        iter->ent_size = ent_size;
        if (!next_entry)
                next_ts = iter->ts;
-        rel_usecs = ns2usecs(next_ts - iter->ts);
        if (verbose) {
                char comm[TASK_COMM_LEN];
                trace_find_cmdline(entry->pid, comm);
-                ret = trace_seq_printf(s, "%16s %5d %3d %d %08x %08lx [%08llx]"
+                ret = trace_seq_printf(
-                                       " %ld.%03ldms (+%ld.%03ldms): ", comm,
+                                s, "%16s %5d %3d %d %08x %08lx ",
-                                       entry->pid, iter->cpu, entry->flags,
+                                comm, entry->pid, iter->cpu, entry->flags,
-                                       entry->preempt_count, iter->idx,
+                                entry->preempt_count, iter->idx);
-                                       ns2usecs(iter->ts),
-                                       abs_usecs / USEC_PER_MSEC,
-                                       abs_usecs % USEC_PER_MSEC,
-                                       rel_usecs / USEC_PER_MSEC,
-                                       rel_usecs % USEC_PER_MSEC);
        } else {
                ret = lat_print_generic(s, entry, iter->cpu);
-                if (ret)
-                        ret = lat_print_timestamp(s, abs_usecs, rel_usecs);
        }
+        if (ret)
+                ret = lat_print_timestamp(iter, next_ts);
        return ret;
 }
diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c
index daa9980153af..412e959709b4 100644
--- a/kernel/trace/trace_probe.c
+++ b/kernel/trace/trace_probe.c
@@ -441,7 +441,7 @@ static const struct fetch_type *find_fetch_type(const char *type)
                        goto fail;
                type++;
-                if (strict_strtoul(type, 0, &bs))
+                if (kstrtoul(type, 0, &bs))
                        goto fail;
                switch (bs) {
@@ -501,8 +501,8 @@ int traceprobe_split_symbol_offset(char *symbol, unsigned long *offset)
        tmp = strchr(symbol, '+');
        if (tmp) {
-                /* skip sign because strict_strtol doesn't accept '+' */
+                /* skip sign because kstrtoul doesn't accept '+' */
-                ret = strict_strtoul(tmp + 1, 0, offset);
+                ret = kstrtoul(tmp + 1, 0, offset);
                if (ret)
                        return ret;
@@ -533,7 +533,7 @@ static int parse_probe_vars(char *arg, const struct fetch_type *t,
                        else
                                ret = -EINVAL;
                } else if (isdigit(arg[5])) {
-                        ret = strict_strtoul(arg + 5, 10, &param);
+                        ret = kstrtoul(arg + 5, 10, &param);
                        if (ret || param > PARAM_MAX_STACK)
                                ret = -EINVAL;
                        else {
@@ -579,7 +579,7 @@ static int parse_probe_arg(char *arg, const struct fetch_type *t,
        case '@':       /* memory or symbol */
                if (isdigit(arg[1])) {
-                        ret = strict_strtoul(arg + 1, 0, &param);
+                        ret = kstrtoul(arg + 1, 0, &param);
                        if (ret)
                                break;
@@ -597,14 +597,14 @@ static int parse_probe_arg(char *arg, const struct fetch_type *t,
                break;
        case '+':       /* deref memory */
-                arg++;  /* Skip '+', because strict_strtol() rejects it. */
+                arg++;  /* Skip '+', because kstrtol() rejects it. */
        case '-':
                tmp = strchr(arg, '(');
                if (!tmp)
                        break;
                *tmp = '\0';
-                ret = strict_strtol(arg, 0, &offset);
+                ret = kstrtol(arg, 0, &offset);
                if (ret)
                        break;
diff --git a/kernel/trace/trace_sched_switch.c b/kernel/trace/trace_sched_switch.c
index 7e62c0a18456..3374c792ccd8 100644
--- a/kernel/trace/trace_sched_switch.c
+++ b/kernel/trace/trace_sched_switch.c
@@ -102,9 +102,7 @@ tracing_sched_wakeup_trace(struct trace_array *tr,
        entry->next_cpu                 = task_cpu(wakee);
        if (!filter_check_discard(call, entry, buffer, event))
-                ring_buffer_unlock_commit(buffer, event);
+                trace_buffer_unlock_commit(buffer, event, flags, pc);
-        ftrace_trace_stack(tr->buffer, flags, 6, pc);
-        ftrace_trace_userstack(tr->buffer, flags, pc);
 }
 static void
diff --git a/kernel/trace/trace_sched_wakeup.c b/kernel/trace/trace_sched_wakeup.c
index 02170c00c413..9fe45fcefca0 100644
--- a/kernel/trace/trace_sched_wakeup.c
+++ b/kernel/trace/trace_sched_wakeup.c
@@ -7,7 +7,7 @@
 * Based on code from the latency_tracer, that is:
 *
 *  Copyright (C) 2004-2006 Ingo Molnar
- *  Copyright (C) 2004 William Lee Irwin III
+ *  Copyright (C) 2004 Nadia Yvette Chambers
 */
 #include <linux/module.h>
 #include <linux/fs.h>
@@ -589,7 +589,7 @@ static struct tracer wakeup_tracer __read_mostly =
        .reset          = wakeup_tracer_reset,
        .start          = wakeup_tracer_start,
        .stop           = wakeup_tracer_stop,
-        .print_max      = 1,
+        .print_max      = true,
        .print_header   = wakeup_print_header,
        .print_line     = wakeup_print_line,
        .flags          = &tracer_flags,
@@ -599,7 +599,7 @@ static struct tracer wakeup_tracer __read_mostly =
 #endif
        .open           = wakeup_trace_open,
        .close          = wakeup_trace_close,
-        .use_max_tr     = 1,
+        .use_max_tr     = true,
 };
 static struct tracer wakeup_rt_tracer __read_mostly =
@@ -610,7 +610,7 @@ static struct tracer wakeup_rt_tracer __read_mostly =
        .start          = wakeup_tracer_start,
        .stop           = wakeup_tracer_stop,
        .wait_pipe      = poll_wait_pipe,
-        .print_max      = 1,
+        .print_max      = true,
        .print_header   = wakeup_print_header,
        .print_line     = wakeup_print_line,
        .flags          = &tracer_flags,
@@ -620,7 +620,7 @@ static struct tracer wakeup_rt_tracer __read_mostly =
 #endif
        .open           = wakeup_trace_open,
        .close          = wakeup_trace_close,
-        .use_max_tr     = 1,
+        .use_max_tr     = true,
 };
 __init static int init_wakeup_tracer(void)
@@ -637,4 +637,4 @@ __init static int init_wakeup_tracer(void)
        return 0;
 }
-device_initcall(init_wakeup_tracer);
+core_initcall(init_wakeup_tracer);
diff --git a/kernel/trace/trace_selftest.c b/kernel/trace/trace_selftest.c
index 2c00a691a540..47623169a815 100644
--- a/kernel/trace/trace_selftest.c
+++ b/kernel/trace/trace_selftest.c
@@ -320,7 +320,6 @@ int trace_selftest_startup_dynamic_tracing(struct tracer *trace,
                                           int (*func)(void))
 {
        int save_ftrace_enabled = ftrace_enabled;
-        int save_tracer_enabled = tracer_enabled;
        unsigned long count;
        char *func_name;
        int ret;
@@ -331,7 +330,6 @@ int trace_selftest_startup_dynamic_tracing(struct tracer *trace,
        /* enable tracing, and record the filter function */
        ftrace_enabled = 1;
-        tracer_enabled = 1;
        /* passed in by parameter to fool gcc from optimizing */
        func();
@@ -395,7 +393,6 @@ int trace_selftest_startup_dynamic_tracing(struct tracer *trace,
 out:
        ftrace_enabled = save_ftrace_enabled;
-        tracer_enabled = save_tracer_enabled;
        /* Enable tracing on all functions again */
        ftrace_set_global_filter(NULL, 0, 1);
@@ -452,7 +449,6 @@ static int
 trace_selftest_function_recursion(void)
 {
        int save_ftrace_enabled = ftrace_enabled;
-        int save_tracer_enabled = tracer_enabled;
        char *func_name;
        int len;
        int ret;
@@ -465,7 +461,6 @@ trace_selftest_function_recursion(void)
        /* enable tracing, and record the filter function */
        ftrace_enabled = 1;
-        tracer_enabled = 1;
        /* Handle PPC64 '.' name */
        func_name = "*" __stringify(DYN_FTRACE_TEST_NAME);
@@ -534,7 +529,6 @@ trace_selftest_function_recursion(void)
        ret = 0;
 out:
        ftrace_enabled = save_ftrace_enabled;
-        tracer_enabled = save_tracer_enabled;
        return ret;
 }
@@ -569,7 +563,6 @@ static int
 trace_selftest_function_regs(void)
 {
        int save_ftrace_enabled = ftrace_enabled;
-        int save_tracer_enabled = tracer_enabled;
        char *func_name;
        int len;
        int ret;
@@ -586,7 +579,6 @@ trace_selftest_function_regs(void)
        /* enable tracing, and record the filter function */
        ftrace_enabled = 1;
-        tracer_enabled = 1;
        /* Handle PPC64 '.' name */
        func_name = "*" __stringify(DYN_FTRACE_TEST_NAME);
@@ -648,7 +640,6 @@ trace_selftest_function_regs(void)
        ret = 0;
 out:
        ftrace_enabled = save_ftrace_enabled;
-        tracer_enabled = save_tracer_enabled;
        return ret;
 }
@@ -662,7 +653,6 @@ int
 trace_selftest_startup_function(struct tracer *trace, struct trace_array *tr)
 {
        int save_ftrace_enabled = ftrace_enabled;
-        int save_tracer_enabled = tracer_enabled;
        unsigned long count;
        int ret;
@@ -671,7 +661,6 @@ trace_selftest_startup_function(struct tracer *trace, struct trace_array *tr)
        /* start the tracing */
        ftrace_enabled = 1;
-        tracer_enabled = 1;
        ret = tracer_init(trace, tr);
        if (ret) {
@@ -708,7 +697,6 @@ trace_selftest_startup_function(struct tracer *trace, struct trace_array *tr)
        ret = trace_selftest_function_regs();
 out:
        ftrace_enabled = save_ftrace_enabled;
-        tracer_enabled = save_tracer_enabled;
        /* kill ftrace totally if we failed */
        if (ret)
@@ -1106,6 +1094,7 @@ trace_selftest_startup_wakeup(struct tracer *trace, struct trace_array *tr)
        tracing_stop();
        /* check both trace buffers */
        ret = trace_test_buffer(tr, NULL);
+        printk("ret = %d\n", ret);
        if (!ret)
                ret = trace_test_buffer(&max_tr, &count);
diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c
index 2485a7d09b11..7609dd6714c2 100644
--- a/kernel/trace/trace_syscalls.c
+++ b/kernel/trace/trace_syscalls.c
@@ -21,9 +21,6 @@ static int syscall_enter_register(struct ftrace_event_call *event,
 static int syscall_exit_register(struct ftrace_event_call *event,
                                 enum trace_reg type, void *data);
-static int syscall_enter_define_fields(struct ftrace_event_call *call);
-static int syscall_exit_define_fields(struct ftrace_event_call *call);
 static struct list_head *
 syscall_get_enter_fields(struct ftrace_event_call *call)
 {
@@ -32,30 +29,6 @@ syscall_get_enter_fields(struct ftrace_event_call *call)
        return &entry->enter_fields;
 }
-struct trace_event_functions enter_syscall_print_funcs = {
-        .trace          = print_syscall_enter,
-};
-struct trace_event_functions exit_syscall_print_funcs = {
-        .trace          = print_syscall_exit,
-};
-struct ftrace_event_class event_class_syscall_enter = {
-        .system         = "syscalls",
-        .reg            = syscall_enter_register,
-        .define_fields  = syscall_enter_define_fields,
-        .get_fields     = syscall_get_enter_fields,
-        .raw_init       = init_syscall_trace,
-};
-struct ftrace_event_class event_class_syscall_exit = {
-        .system         = "syscalls",
-        .reg            = syscall_exit_register,
-        .define_fields  = syscall_exit_define_fields,
-        .fields         = LIST_HEAD_INIT(event_class_syscall_exit.fields),
-        .raw_init       = init_syscall_trace,
-};
 extern struct syscall_metadata *__start_syscalls_metadata[];
 extern struct syscall_metadata *__stop_syscalls_metadata[];
@@ -432,7 +405,7 @@ void unreg_event_syscall_exit(struct ftrace_event_call *call)
        mutex_unlock(&syscall_trace_lock);
 }
-int init_syscall_trace(struct ftrace_event_call *call)
+static int init_syscall_trace(struct ftrace_event_call *call)
 {
        int id;
        int num;
@@ -457,6 +430,30 @@ int init_syscall_trace(struct ftrace_event_call *call)
        return id;
 }
+struct trace_event_functions enter_syscall_print_funcs = {
+        .trace          = print_syscall_enter,
+};
+struct trace_event_functions exit_syscall_print_funcs = {
+        .trace          = print_syscall_exit,
+};
+struct ftrace_event_class event_class_syscall_enter = {
+        .system         = "syscalls",
+        .reg            = syscall_enter_register,
+        .define_fields  = syscall_enter_define_fields,
+        .get_fields     = syscall_get_enter_fields,
+        .raw_init       = init_syscall_trace,
+};
+struct ftrace_event_class event_class_syscall_exit = {
+        .system         = "syscalls",
+        .reg            = syscall_exit_register,
+        .define_fields  = syscall_exit_define_fields,
+        .fields         = LIST_HEAD_INIT(event_class_syscall_exit.fields),
+        .raw_init       = init_syscall_trace,
+};
 unsigned long __init __weak arch_syscall_addr(int nr)
 {
        return (unsigned long)sys_call_table[nr];
@@ -537,7 +534,7 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
        perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head, NULL);
 }
-int perf_sysenter_enable(struct ftrace_event_call *call)
+static int perf_sysenter_enable(struct ftrace_event_call *call)
 {
        int ret = 0;
        int num;
@@ -558,7 +555,7 @@ int perf_sysenter_enable(struct ftrace_event_call *call)
        return ret;
 }
-void perf_sysenter_disable(struct ftrace_event_call *call)
+static void perf_sysenter_disable(struct ftrace_event_call *call)
 {
        int num;
@@ -615,7 +612,7 @@ static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
        perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head, NULL);
 }
-int perf_sysexit_enable(struct ftrace_event_call *call)
+static int perf_sysexit_enable(struct ftrace_event_call *call)
 {
        int ret = 0;
        int num;
@@ -636,7 +633,7 @@ int perf_sysexit_enable(struct ftrace_event_call *call)
        return ret;
 }
-void perf_sysexit_disable(struct ftrace_event_call *call)
+static void perf_sysexit_disable(struct ftrace_event_call *call)
 {
        int num;
diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c
index 03003cd7dd96..c86e6d4f67fb 100644
--- a/kernel/trace/trace_uprobe.c
+++ b/kernel/trace/trace_uprobe.c
@@ -22,6 +22,7 @@
 #include <linux/uaccess.h>
 #include <linux/uprobes.h>
 #include <linux/namei.h>
+#include <linux/string.h>
 #include "trace_probe.h"
@@ -189,7 +190,7 @@ static int create_trace_uprobe(int argc, char **argv)
        if (argv[0][0] == '-')
                is_delete = true;
        else if (argv[0][0] != 'p') {
-                pr_info("Probe definition must be started with 'p', 'r' or" " '-'.\n");
+                pr_info("Probe definition must be started with 'p' or '-'.\n");
                return -EINVAL;
        }
@@ -252,7 +253,7 @@ static int create_trace_uprobe(int argc, char **argv)
        if (ret)
                goto fail_address_parse;
-        ret = strict_strtoul(arg, 0, &offset);
+        ret = kstrtoul(arg, 0, &offset);
        if (ret)
                goto fail_address_parse;
@@ -263,16 +264,15 @@ static int create_trace_uprobe(int argc, char **argv)
        /* setup a probe */
        if (!event) {
-                char *tail = strrchr(filename, '/');
+                char *tail;
                char *ptr;
-                ptr = kstrdup((tail ? tail + 1 : filename), GFP_KERNEL);
+                tail = kstrdup(kbasename(filename), GFP_KERNEL);
-                if (!ptr) {
+                if (!tail) {
                        ret = -ENOMEM;
                        goto fail_address_parse;
                }
-                tail = ptr;
                ptr = strpbrk(tail, ".-_");
                if (ptr)
                        *ptr = '\0';
diff --git a/kernel/wait.c b/kernel/wait.c
index 7fdd9eaca2c3..6698e0c04ead 100644
--- a/kernel/wait.c
+++ b/kernel/wait.c
@@ -1,7 +1,7 @@
 /*
 * Generic waiting primitives.
 *
- * (C) 2004 William Irwin, Oracle
+ * (C) 2004 Nadia Yvette Chambers, Oracle
 */
 #include <linux/init.h>
 #include <linux/export.h>
diff --git a/kernel/watchdog.c b/kernel/watchdog.c
index 9d4c8d5a1f53..997c6a16ec22 100644
--- a/kernel/watchdog.c
+++ b/kernel/watchdog.c
@@ -31,6 +31,7 @@
 int watchdog_enabled = 1;
 int __read_mostly watchdog_thresh = 10;
 static int __read_mostly watchdog_disabled;
+static u64 __read_mostly sample_period;
 static DEFINE_PER_CPU(unsigned long, watchdog_touch_ts);
 static DEFINE_PER_CPU(struct task_struct *, softlockup_watchdog);
@@ -116,7 +117,7 @@ static unsigned long get_timestamp(int this_cpu)
        return cpu_clock(this_cpu) >> 30LL;  /* 2^30 ~= 10^9 */
 }
-static unsigned long get_sample_period(void)
+static void set_sample_period(void)
 {
        /*
         * convert watchdog_thresh from seconds to ns
@@ -125,7 +126,7 @@ static unsigned long get_sample_period(void)
         * and hard thresholds) to increment before the
         * hardlockup detector generates a warning
         */
-        return get_softlockup_thresh() * (NSEC_PER_SEC / 5);
+        sample_period = get_softlockup_thresh() * ((u64)NSEC_PER_SEC / 5);
 }
 /* Commands for resetting the watchdog */
@@ -275,7 +276,7 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
        wake_up_process(__this_cpu_read(softlockup_watchdog));
        /* .. and repeat */
-        hrtimer_forward_now(hrtimer, ns_to_ktime(get_sample_period()));
+        hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period));
        if (touch_ts == 0) {
                if (unlikely(__this_cpu_read(softlockup_touch_sync))) {
@@ -356,7 +357,7 @@ static void watchdog_enable(unsigned int cpu)
        hrtimer->function = watchdog_timer_fn;
        /* done here because hrtimer_start can only pin to smp_processor_id() */
-        hrtimer_start(hrtimer, ns_to_ktime(get_sample_period()),
+        hrtimer_start(hrtimer, ns_to_ktime(sample_period),
                      HRTIMER_MODE_REL_PINNED);
        /* initialize timestamp */
@@ -368,6 +369,9 @@ static void watchdog_disable(unsigned int cpu)
 {
        struct hrtimer *hrtimer = &__raw_get_cpu_var(watchdog_hrtimer);
+        if (!watchdog_enabled)
+                return;
        watchdog_set_prio(SCHED_NORMAL, 0);
        hrtimer_cancel(hrtimer);
        /* disable the perf event */
@@ -383,7 +387,7 @@ static int watchdog_should_run(unsigned int cpu)
 /*
 * The watchdog thread function - touches the timestamp.
 *
- * It only runs once every get_sample_period() seconds (4 seconds by
+ * It only runs once every sample_period seconds (4 seconds by
 * default) to reset the softlockup timestamp. If this gets delayed
 * for more than 2*watchdog_thresh seconds then the debug-printout
 * triggers in watchdog_timer_fn().
@@ -516,6 +520,7 @@ int proc_dowatchdog(struct ctl_table *table, int write,
        if (ret || !write)
                return ret;
+        set_sample_period();
        if (watchdog_enabled && watchdog_thresh)
                watchdog_enable_all_cpus();
        else
@@ -537,6 +542,7 @@ static struct smp_hotplug_thread watchdog_threads = {
 void __init lockup_detector_init(void)
 {
+        set_sample_period();
        if (smpboot_register_percpu_thread(&watchdog_threads)) {
                pr_err("Failed to create watchdog threads, disabled\n");
                watchdog_disabled = -ENODEV;
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 042d221d33cc..fbc6576a83c3 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -739,8 +739,10 @@ void wq_worker_waking_up(struct task_struct *task, unsigned int cpu)
 {
        struct worker *worker = kthread_data(task);
-        if (!(worker->flags & WORKER_NOT_RUNNING))
+        if (!(worker->flags & WORKER_NOT_RUNNING)) {
+                WARN_ON_ONCE(worker->pool->gcwq->cpu != cpu);
                atomic_inc(get_pool_nr_running(worker->pool));
+        }
 }
 /**
@@ -1361,8 +1363,19 @@ static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
        WARN_ON_ONCE(timer->function != delayed_work_timer_fn ||
                     timer->data != (unsigned long)dwork);
-        BUG_ON(timer_pending(timer));
+        WARN_ON_ONCE(timer_pending(timer));
-        BUG_ON(!list_empty(&work->entry));
+        WARN_ON_ONCE(!list_empty(&work->entry));
+        /*
+         * If @delay is 0, queue @dwork->work immediately.  This is for
+         * both optimization and correctness.  The earliest @timer can
+         * expire is on the closest next tick and delayed_work users depend
+         * on that there's no such delay when @delay is 0.
+         */
+        if (!delay) {
+                __queue_work(cpu, wq, &dwork->work);
+                return;
+        }
        timer_stats_timer_set_start_info(&dwork->timer);
@@ -1417,9 +1430,6 @@ bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
        bool ret = false;
        unsigned long flags;
-        if (!delay)
-                return queue_work_on(cpu, wq, &dwork->work);
        /* read the comment in __queue_work() */
        local_irq_save(flags);
@@ -2407,8 +2417,10 @@ static int rescuer_thread(void *__wq)
 repeat:
        set_current_state(TASK_INTERRUPTIBLE);
-        if (kthread_should_stop())
+        if (kthread_should_stop()) {
+                __set_current_state(TASK_RUNNING);
                return 0;
+        }
        /*
         * See whether any cpu is asking for help.  Unbounded
@@ -3475,7 +3487,7 @@ unsigned int work_busy(struct work_struct *work)
        unsigned int ret = 0;
        if (!gcwq)
-                return false;
+                return 0;
        spin_lock_irqsave(&gcwq->lock, flags);