132 files changed, 12969 insertions, 9142 deletions

diff --git a/kernel/Makefile b/kernel/Makefile
index 90b53f6dc226..3d9c7e27e3f9 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -11,6 +11,7 @@ obj-y     = sched.o fork.o exec_domain.o panic.o printk.o \
             hrtimer.o rwsem.o nsproxy.o srcu.o semaphore.o \
             notifier.o ksysfs.o pm_qos_params.o sched_clock.o cred.o \
             async.o
+obj-y += groups.o
 
 ifdef CONFIG_FUNCTION_TRACER
 # Do not trace debug files and internal ftrace files
@@ -68,8 +69,9 @@ obj-$(CONFIG_IKCONFIG) += configs.o
 obj-$(CONFIG_RESOURCE_COUNTERS) += res_counter.o
 obj-$(CONFIG_STOP_MACHINE) += stop_machine.o
 obj-$(CONFIG_KPROBES_SANITY_TEST) += test_kprobes.o
-obj-$(CONFIG_AUDIT) += audit.o auditfilter.o
+obj-$(CONFIG_AUDIT) += audit.o auditfilter.o audit_watch.o
 obj-$(CONFIG_AUDITSYSCALL) += auditsc.o
+obj-$(CONFIG_GCOV_KERNEL) += gcov/
 obj-$(CONFIG_AUDIT_TREE) += audit_tree.o
 obj-$(CONFIG_KPROBES) += kprobes.o
 obj-$(CONFIG_KGDB) += kgdb.o
@@ -78,11 +80,9 @@ obj-$(CONFIG_DETECT_HUNG_TASK) += hung_task.o
 obj-$(CONFIG_GENERIC_HARDIRQS) += irq/
 obj-$(CONFIG_SECCOMP) += seccomp.o
 obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o
-obj-$(CONFIG_CLASSIC_RCU) += rcuclassic.o
 obj-$(CONFIG_TREE_RCU) += rcutree.o
-obj-$(CONFIG_PREEMPT_RCU) += rcupreempt.o
+obj-$(CONFIG_TREE_PREEMPT_RCU) += rcutree.o
 obj-$(CONFIG_TREE_RCU_TRACE) += rcutree_trace.o
-obj-$(CONFIG_PREEMPT_RCU_TRACE) += rcupreempt_trace.o
 obj-$(CONFIG_RELAY) += relay.o
 obj-$(CONFIG_SYSCTL) += utsname_sysctl.o
 obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o
@@ -90,10 +90,10 @@ obj-$(CONFIG_TASKSTATS) += taskstats.o tsacct.o
 obj-$(CONFIG_MARKERS) += marker.o
 obj-$(CONFIG_TRACEPOINTS) += tracepoint.o
 obj-$(CONFIG_LATENCYTOP) += latencytop.o
-obj-$(CONFIG_HAVE_GENERIC_DMA_COHERENT) += dma-coherent.o
 obj-$(CONFIG_FUNCTION_TRACER) += trace/
 obj-$(CONFIG_TRACING) += trace/
 obj-$(CONFIG_X86_DS) += trace/
+obj-$(CONFIG_RING_BUFFER) += trace/
 obj-$(CONFIG_SMP) += sched_cpupri.o
 obj-$(CONFIG_SLOW_WORK) += slow-work.o
 obj-$(CONFIG_PERF_COUNTERS) += perf_counter.o
@@ -116,7 +116,7 @@ $(obj)/config_data.gz: .config FORCE
         $(call if_changed,gzip)
 
 quiet_cmd_ikconfiggz = IKCFG   $@
-      cmd_ikconfiggz = (echo "static const char kernel_config_data[] = MAGIC_START"; cat $< | scripts/bin2c; echo "MAGIC_END;") > $@
+      cmd_ikconfiggz = (echo "static const char kernel_config_data[] __used = MAGIC_START"; cat $< | scripts/bin2c; echo "MAGIC_END;") > $@
 targets += config_data.h
 $(obj)/config_data.h: $(obj)/config_data.gz FORCE
         $(call if_changed,ikconfiggz)
diff --git a/kernel/acct.c b/kernel/acct.c
index 7afa31564162..9a4715a2f6bf 100644
--- a/kernel/acct.c
+++ b/kernel/acct.c
@@ -215,6 +215,7 @@ static void acct_file_reopen(struct bsd_acct_struct *acct, struct file *file,
 static int acct_on(char *name)
 {
         struct file *file;
+        struct vfsmount *mnt;
         int error;
         struct pid_namespace *ns;
         struct bsd_acct_struct *acct = NULL;
@@ -256,11 +257,12 @@ static int acct_on(char *name)
                 acct = NULL;
         }
 
-        mnt_pin(file->f_path.mnt);
+        mnt = file->f_path.mnt;
+        mnt_pin(mnt);
         acct_file_reopen(ns->bacct, file, ns);
         spin_unlock(&acct_lock);
 
-        mntput(file->f_path.mnt); /* it's pinned, now give up active reference */
+        mntput(mnt); /* it's pinned, now give up active reference */
         kfree(acct);
 
         return 0;
@@ -489,13 +491,17 @@ static void do_acct_process(struct bsd_acct_struct *acct,
         u64 run_time;
         struct timespec uptime;
         struct tty_struct *tty;
+        const struct cred *orig_cred;
+
+        /* Perform file operations on behalf of whoever enabled accounting */
+        orig_cred = override_creds(file->f_cred);
 
         /*
          * First check to see if there is enough free_space to continue
          * the process accounting system.
          */
         if (!check_free_space(acct, file))
-                return;
+                goto out;
 
         /*
          * Fill the accounting struct with the needed info as recorded
@@ -576,6 +582,8 @@ static void do_acct_process(struct bsd_acct_struct *acct,
                                sizeof(acct_t), &file->f_pos);
         current->signal->rlim[RLIMIT_FSIZE].rlim_cur = flim;
         set_fs(fs);
+out:
+        revert_creds(orig_cred);
 }
 
 /**
diff --git a/kernel/audit.c b/kernel/audit.c
index 9442c3533ba9..defc2e6f1e3b 100644
--- a/kernel/audit.c
+++ b/kernel/audit.c
@@ -115,9 +115,6 @@ static atomic_t    audit_lost = ATOMIC_INIT(0);
 /* The netlink socket. */
 static struct sock *audit_sock;
 
-/* Inotify handle. */
-struct inotify_handle *audit_ih;
-
 /* Hash for inode-based rules */
 struct list_head audit_inode_hash[AUDIT_INODE_BUCKETS];
 
@@ -136,7 +133,7 @@ static DECLARE_WAIT_QUEUE_HEAD(kauditd_wait);
 static DECLARE_WAIT_QUEUE_HEAD(audit_backlog_wait);
 
 /* Serialize requests from userspace. */
-static DEFINE_MUTEX(audit_cmd_mutex);
+DEFINE_MUTEX(audit_cmd_mutex);
 
 /* AUDIT_BUFSIZ is the size of the temporary buffer used for formatting
  * audit records.  Since printk uses a 1024 byte buffer, this buffer
@@ -375,6 +372,25 @@ static void audit_hold_skb(struct sk_buff *skb)
                 kfree_skb(skb);
 }
 
+/*
+ * For one reason or another this nlh isn't getting delivered to the userspace
+ * audit daemon, just send it to printk.
+ */
+static void audit_printk_skb(struct sk_buff *skb)
+{
+        struct nlmsghdr *nlh = nlmsg_hdr(skb);
+        char *data = NLMSG_DATA(nlh);
+
+        if (nlh->nlmsg_type != AUDIT_EOE) {
+                if (printk_ratelimit())
+                        printk(KERN_NOTICE "type=%d %s\n", nlh->nlmsg_type, data);
+                else
+                        audit_log_lost("printk limit exceeded\n");
+        }
+
+        audit_hold_skb(skb);
+}
+
 static void kauditd_send_skb(struct sk_buff *skb)
 {
         int err;
@@ -427,14 +443,8 @@ static int kauditd_thread(void *dummy)
                 if (skb) {
                         if (audit_pid)
                                 kauditd_send_skb(skb);
-                        else {
-                                if (printk_ratelimit())
-                                        printk(KERN_NOTICE "%s\n", skb->data + NLMSG_SPACE(0));
-                                else
-                                        audit_log_lost("printk limit exceeded\n");
-
-                                audit_hold_skb(skb);
-                        }
+                        else
+                                audit_printk_skb(skb);
                 } else {
                         DECLARE_WAITQUEUE(wait, current);
                         set_current_state(TASK_INTERRUPTIBLE);
@@ -495,42 +505,25 @@ int audit_send_list(void *_dest)
         return 0;
 }
 
-#ifdef CONFIG_AUDIT_TREE
-static int prune_tree_thread(void *unused)
-{
-        mutex_lock(&audit_cmd_mutex);
-        audit_prune_trees();
-        mutex_unlock(&audit_cmd_mutex);
-        return 0;
-}
-
-void audit_schedule_prune(void)
-{
-        kthread_run(prune_tree_thread, NULL, "audit_prune_tree");
-}
-#endif
-
 struct sk_buff *audit_make_reply(int pid, int seq, int type, int done,
                                  int multi, void *payload, int size)
 {
         struct sk_buff  *skb;
         struct nlmsghdr *nlh;
-        int             len = NLMSG_SPACE(size);
         void            *data;
         int             flags = multi ? NLM_F_MULTI : 0;
         int             t     = done  ? NLMSG_DONE  : type;
 
-        skb = alloc_skb(len, GFP_KERNEL);
+        skb = nlmsg_new(size, GFP_KERNEL);
         if (!skb)
                 return NULL;
 
-        nlh              = NLMSG_PUT(skb, pid, seq, t, size);
-        nlh->nlmsg_flags = flags;
-        data             = NLMSG_DATA(nlh);
+        nlh     = NLMSG_NEW(skb, pid, seq, t, size, flags);
+        data    = NLMSG_DATA(nlh);
         memcpy(data, payload, size);
         return skb;
 
-nlmsg_failure:                  /* Used by NLMSG_PUT */
+nlmsg_failure:                  /* Used by NLMSG_NEW */
         if (skb)
                 kfree_skb(skb);
         return NULL;
@@ -926,28 +919,29 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh)
 }
 
 /*
- * Get message from skb (based on rtnetlink_rcv_skb).  Each message is
- * processed by audit_receive_msg.  Malformed skbs with wrong length are
- * discarded silently.
+ * Get message from skb.  Each message is processed by audit_receive_msg.
+ * Malformed skbs with wrong length are discarded silently.
  */
 static void audit_receive_skb(struct sk_buff *skb)
 {
-        int             err;
-        struct nlmsghdr *nlh;
-        u32             rlen;
+        struct nlmsghdr *nlh;
+        /*
+         * len MUST be signed for NLMSG_NEXT to be able to dec it below 0
+         * if the nlmsg_len was not aligned
+         */
+        int len;
+        int err;
 
-        while (skb->len >= NLMSG_SPACE(0)) {
-                nlh = nlmsg_hdr(skb);
-                if (nlh->nlmsg_len < sizeof(*nlh) || skb->len < nlh->nlmsg_len)
-                        return;
-                rlen = NLMSG_ALIGN(nlh->nlmsg_len);
-                if (rlen > skb->len)
-                        rlen = skb->len;
-                if ((err = audit_receive_msg(skb, nlh))) {
+        nlh = nlmsg_hdr(skb);
+        len = skb->len;
+
+        while (NLMSG_OK(nlh, len)) {
+                err = audit_receive_msg(skb, nlh);
+                /* if err or if this message says it wants a response */
+                if (err || (nlh->nlmsg_flags & NLM_F_ACK))
                         netlink_ack(skb, nlh, err);
-                } else if (nlh->nlmsg_flags & NLM_F_ACK)
-                        netlink_ack(skb, nlh, 0);
-                skb_pull(skb, rlen);
+
+                nlh = NLMSG_NEXT(nlh, len);
         }
 }
 
@@ -959,13 +953,6 @@ static void audit_receive(struct sk_buff  *skb)
         mutex_unlock(&audit_cmd_mutex);
 }
 
-#ifdef CONFIG_AUDITSYSCALL
-static const struct inotify_operations audit_inotify_ops = {
-        .handle_event   = audit_handle_ievent,
-        .destroy_watch  = audit_free_parent,
-};
-#endif
-
 /* Initialize audit support at boot time. */
 static int __init audit_init(void)
 {
@@ -991,12 +978,6 @@ static int __init audit_init(void)
 
         audit_log(NULL, GFP_KERNEL, AUDIT_KERNEL, "initialized");
 
-#ifdef CONFIG_AUDITSYSCALL
-        audit_ih = inotify_init(&audit_inotify_ops);
-        if (IS_ERR(audit_ih))
-                audit_panic("cannot initialize inotify handle");
-#endif
-
         for (i = 0; i < AUDIT_INODE_BUCKETS; i++)
                 INIT_LIST_HEAD(&audit_inode_hash[i]);
 
@@ -1070,18 +1051,20 @@ static struct audit_buffer * audit_buffer_alloc(struct audit_context *ctx,
                         goto err;
         }
 
-        ab->skb = alloc_skb(AUDIT_BUFSIZ, gfp_mask);
-        if (!ab->skb)
-                goto err;
-
         ab->ctx = ctx;
         ab->gfp_mask = gfp_mask;
-        nlh = (struct nlmsghdr *)skb_put(ab->skb, NLMSG_SPACE(0));
-        nlh->nlmsg_type = type;
-        nlh->nlmsg_flags = 0;
-        nlh->nlmsg_pid = 0;
-        nlh->nlmsg_seq = 0;
+
+        ab->skb = nlmsg_new(AUDIT_BUFSIZ, gfp_mask);
+        if (!ab->skb)
+                goto nlmsg_failure;
+
+        nlh = NLMSG_NEW(ab->skb, 0, 0, type, 0, 0);
+
         return ab;
+
+nlmsg_failure:                  /* Used by NLMSG_NEW */
+        kfree_skb(ab->skb);
+        ab->skb = NULL;
 err:
         audit_buffer_free(ab);
         return NULL;
@@ -1452,6 +1435,15 @@ void audit_log_d_path(struct audit_buffer *ab, const char *prefix,
         kfree(pathname);
 }
 
+void audit_log_key(struct audit_buffer *ab, char *key)
+{
+        audit_log_format(ab, " key=");
+        if (key)
+                audit_log_untrustedstring(ab, key);
+        else
+                audit_log_format(ab, "(null)");
+}
+
 /**
  * audit_log_end - end one audit record
  * @ab: the audit_buffer
@@ -1475,15 +1467,7 @@ void audit_log_end(struct audit_buffer *ab)
                         skb_queue_tail(&audit_skb_queue, ab->skb);
                         wake_up_interruptible(&kauditd_wait);
                 } else {
-                        if (nlh->nlmsg_type != AUDIT_EOE) {
-                                if (printk_ratelimit()) {
-                                        printk(KERN_NOTICE "type=%d %s\n",
-                                                nlh->nlmsg_type,
-                                                ab->skb->data + NLMSG_SPACE(0));
-                                } else
-                                        audit_log_lost("printk limit exceeded\n");
-                        }
-                        audit_hold_skb(ab->skb);
+                        audit_printk_skb(ab->skb);
                 }
                 ab->skb = NULL;
         }
diff --git a/kernel/audit.h b/kernel/audit.h
index 16f18cac661b..208687be4f30 100644
--- a/kernel/audit.h
+++ b/kernel/audit.h
@@ -53,18 +53,7 @@ enum audit_state {
 };
 
 /* Rule lists */
-struct audit_parent;
-
-struct audit_watch {
-        atomic_t                count;  /* reference count */
-        char                    *path;  /* insertion path */
-        dev_t                   dev;    /* associated superblock device */
-        unsigned long           ino;    /* associated inode number */
-        struct audit_parent     *parent; /* associated parent */
-        struct list_head        wlist;  /* entry in parent->watches list */
-        struct list_head        rules;  /* associated rules */
-};
-
+struct audit_watch;
 struct audit_tree;
 struct audit_chunk;
 
@@ -108,19 +97,28 @@ struct audit_netlink_list {
 
 int audit_send_list(void *);
 
-struct inotify_watch;
-/* Inotify handle */
-extern struct inotify_handle *audit_ih;
-
-extern void audit_free_parent(struct inotify_watch *);
-extern void audit_handle_ievent(struct inotify_watch *, u32, u32, u32,
-                                const char *, struct inode *);
 extern int selinux_audit_rule_update(void);
 
 extern struct mutex audit_filter_mutex;
 extern void audit_free_rule_rcu(struct rcu_head *);
 extern struct list_head audit_filter_list[];
 
+/* audit watch functions */
+extern unsigned long audit_watch_inode(struct audit_watch *watch);
+extern dev_t audit_watch_dev(struct audit_watch *watch);
+extern void audit_put_watch(struct audit_watch *watch);
+extern void audit_get_watch(struct audit_watch *watch);
+extern int audit_to_watch(struct audit_krule *krule, char *path, int len, u32 op);
+extern int audit_add_watch(struct audit_krule *krule);
+extern void audit_remove_watch(struct audit_watch *watch);
+extern void audit_remove_watch_rule(struct audit_krule *krule, struct list_head *list);
+extern void audit_inotify_unregister(struct list_head *in_list);
+extern char *audit_watch_path(struct audit_watch *watch);
+extern struct list_head *audit_watch_rules(struct audit_watch *watch);
+
+extern struct audit_entry *audit_dupe_rule(struct audit_krule *old,
+                                           struct audit_watch *watch);
+
 #ifdef CONFIG_AUDIT_TREE
 extern struct audit_chunk *audit_tree_lookup(const struct inode *);
 extern void audit_put_chunk(struct audit_chunk *);
@@ -130,10 +128,9 @@ extern int audit_add_tree_rule(struct audit_krule *);
 extern int audit_remove_tree_rule(struct audit_krule *);
 extern void audit_trim_trees(void);
 extern int audit_tag_tree(char *old, char *new);
-extern void audit_schedule_prune(void);
-extern void audit_prune_trees(void);
 extern const char *audit_tree_path(struct audit_tree *);
 extern void audit_put_tree(struct audit_tree *);
+extern void audit_kill_trees(struct list_head *);
 #else
 #define audit_remove_tree_rule(rule) BUG()
 #define audit_add_tree_rule(rule) -EINVAL
@@ -142,6 +139,7 @@ extern void audit_put_tree(struct audit_tree *);
 #define audit_put_tree(tree) (void)0
 #define audit_tag_tree(old, new) -EINVAL
 #define audit_tree_path(rule) ""        /* never called */
+#define audit_kill_trees(list) BUG()
 #endif
 
 extern char *audit_unpack_string(void **, size_t *, size_t);
@@ -160,7 +158,10 @@ static inline int audit_signal_info(int sig, struct task_struct *t)
         return 0;
 }
 extern void audit_filter_inodes(struct task_struct *, struct audit_context *);
+extern struct list_head *audit_killed_trees(void);
 #else
 #define audit_signal_info(s,t) AUDIT_DISABLED
 #define audit_filter_inodes(t,c) AUDIT_DISABLED
 #endif
+
+extern struct mutex audit_cmd_mutex;
diff --git a/kernel/audit_tree.c b/kernel/audit_tree.c
index 1f6396d76687..2451dc6f3282 100644
--- a/kernel/audit_tree.c
+++ b/kernel/audit_tree.c
@@ -2,6 +2,7 @@
 #include <linux/inotify.h>
 #include <linux/namei.h>
 #include <linux/mount.h>
+#include <linux/kthread.h>
 
 struct audit_tree;
 struct audit_chunk;
@@ -441,13 +442,11 @@ static void kill_rules(struct audit_tree *tree)
                 if (rule->tree) {
                         /* not a half-baked one */
                         ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE);
-                        audit_log_format(ab, "op=remove rule dir=");
+                        audit_log_format(ab, "op=");
+                        audit_log_string(ab, "remove rule");
+                        audit_log_format(ab, " dir=");
                         audit_log_untrustedstring(ab, rule->tree->pathname);
-                        if (rule->filterkey) {
-                                audit_log_format(ab, " key=");
-                                audit_log_untrustedstring(ab, rule->filterkey);
-                        } else
-                                audit_log_format(ab, " key=(null)");
+                        audit_log_key(ab, rule->filterkey);
                         audit_log_format(ab, " list=%d res=1", rule->listnr);
                         audit_log_end(ab);
                         rule->tree = NULL;
@@ -519,6 +518,8 @@ static void trim_marked(struct audit_tree *tree)
         }
 }
 
+static void audit_schedule_prune(void);
+
 /* called with audit_filter_mutex */
 int audit_remove_tree_rule(struct audit_krule *rule)
 {
@@ -824,10 +825,11 @@ int audit_tag_tree(char *old, char *new)
 
 /*
  * That gets run when evict_chunk() ends up needing to kill audit_tree.
- * Runs from a separate thread, with audit_cmd_mutex held.
+ * Runs from a separate thread.
  */
-void audit_prune_trees(void)
+static int prune_tree_thread(void *unused)
 {
+        mutex_lock(&audit_cmd_mutex);
         mutex_lock(&audit_filter_mutex);
 
         while (!list_empty(&prune_list)) {
@@ -844,6 +846,40 @@ void audit_prune_trees(void)
         }
 
         mutex_unlock(&audit_filter_mutex);
+        mutex_unlock(&audit_cmd_mutex);
+        return 0;
+}
+
+static void audit_schedule_prune(void)
+{
+        kthread_run(prune_tree_thread, NULL, "audit_prune_tree");
+}
+
+/*
+ * ... and that one is done if evict_chunk() decides to delay until the end
+ * of syscall.  Runs synchronously.
+ */
+void audit_kill_trees(struct list_head *list)
+{
+        mutex_lock(&audit_cmd_mutex);
+        mutex_lock(&audit_filter_mutex);
+
+        while (!list_empty(list)) {
+                struct audit_tree *victim;
+
+                victim = list_entry(list->next, struct audit_tree, list);
+                kill_rules(victim);
+                list_del_init(&victim->list);
+
+                mutex_unlock(&audit_filter_mutex);
+
+                prune_one(victim);
+
+                mutex_lock(&audit_filter_mutex);
+        }
+
+        mutex_unlock(&audit_filter_mutex);
+        mutex_unlock(&audit_cmd_mutex);
 }
 
 /*
@@ -854,6 +890,8 @@ void audit_prune_trees(void)
 static void evict_chunk(struct audit_chunk *chunk)
 {
         struct audit_tree *owner;
+        struct list_head *postponed = audit_killed_trees();
+        int need_prune = 0;
         int n;
 
         if (chunk->dead)
@@ -869,15 +907,21 @@ static void evict_chunk(struct audit_chunk *chunk)
                 owner->root = NULL;
                 list_del_init(&owner->same_root);
                 spin_unlock(&hash_lock);
-                kill_rules(owner);
-                list_move(&owner->list, &prune_list);
-                audit_schedule_prune();
+                if (!postponed) {
+                        kill_rules(owner);
+                        list_move(&owner->list, &prune_list);
+                        need_prune = 1;
+                } else {
+                        list_move(&owner->list, postponed);
+                }
                 spin_lock(&hash_lock);
         }
         list_del_rcu(&chunk->hash);
         for (n = 0; n < chunk->count; n++)
                 list_del_init(&chunk->owners[n].list);
         spin_unlock(&hash_lock);
+        if (need_prune)
+                audit_schedule_prune();
         mutex_unlock(&audit_filter_mutex);
 }
 
diff --git a/kernel/audit_watch.c b/kernel/audit_watch.c
new file mode 100644
index 000000000000..0e96dbc60ea9
--- /dev/null
+++ b/kernel/audit_watch.c
@@ -0,0 +1,543 @@
+/* audit_watch.c -- watching inodes
+ *
+ * Copyright 2003-2009 Red Hat, Inc.
+ * Copyright 2005 Hewlett-Packard Development Company, L.P.
+ * Copyright 2005 IBM Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+#include <linux/kernel.h>
+#include <linux/audit.h>
+#include <linux/kthread.h>
+#include <linux/mutex.h>
+#include <linux/fs.h>
+#include <linux/namei.h>
+#include <linux/netlink.h>
+#include <linux/sched.h>
+#include <linux/inotify.h>
+#include <linux/security.h>
+#include "audit.h"
+
+/*
+ * Reference counting:
+ *
+ * audit_parent: lifetime is from audit_init_parent() to receipt of an IN_IGNORED
+ *      event.  Each audit_watch holds a reference to its associated parent.
+ *
+ * audit_watch: if added to lists, lifetime is from audit_init_watch() to
+ *      audit_remove_watch().  Additionally, an audit_watch may exist
+ *      temporarily to assist in searching existing filter data.  Each
+ *      audit_krule holds a reference to its associated watch.
+ */
+
+struct audit_watch {
+        atomic_t                count;  /* reference count */
+        char                    *path;  /* insertion path */
+        dev_t                   dev;    /* associated superblock device */
+        unsigned long           ino;    /* associated inode number */
+        struct audit_parent     *parent; /* associated parent */
+        struct list_head        wlist;  /* entry in parent->watches list */
+        struct list_head        rules;  /* associated rules */
+};
+
+struct audit_parent {
+        struct list_head        ilist;  /* entry in inotify registration list */
+        struct list_head        watches; /* associated watches */
+        struct inotify_watch    wdata;  /* inotify watch data */
+        unsigned                flags;  /* status flags */
+};
+
+/* Inotify handle. */
+struct inotify_handle *audit_ih;
+
+/*
+ * audit_parent status flags:
+ *
+ * AUDIT_PARENT_INVALID - set anytime rules/watches are auto-removed due to
+ * a filesystem event to ensure we're adding audit watches to a valid parent.
+ * Technically not needed for IN_DELETE_SELF or IN_UNMOUNT events, as we cannot
+ * receive them while we have nameidata, but must be used for IN_MOVE_SELF which
+ * we can receive while holding nameidata.
+ */
+#define AUDIT_PARENT_INVALID    0x001
+
+/* Inotify events we care about. */
+#define AUDIT_IN_WATCH IN_MOVE|IN_CREATE|IN_DELETE|IN_DELETE_SELF|IN_MOVE_SELF
+
+static void audit_free_parent(struct inotify_watch *i_watch)
+{
+        struct audit_parent *parent;
+
+        parent = container_of(i_watch, struct audit_parent, wdata);
+        WARN_ON(!list_empty(&parent->watches));
+        kfree(parent);
+}
+
+void audit_get_watch(struct audit_watch *watch)
+{
+        atomic_inc(&watch->count);
+}
+
+void audit_put_watch(struct audit_watch *watch)
+{
+        if (atomic_dec_and_test(&watch->count)) {
+                WARN_ON(watch->parent);
+                WARN_ON(!list_empty(&watch->rules));
+                kfree(watch->path);
+                kfree(watch);
+        }
+}
+
+void audit_remove_watch(struct audit_watch *watch)
+{
+        list_del(&watch->wlist);
+        put_inotify_watch(&watch->parent->wdata);
+        watch->parent = NULL;
+        audit_put_watch(watch); /* match initial get */
+}
+
+char *audit_watch_path(struct audit_watch *watch)
+{
+        return watch->path;
+}
+
+struct list_head *audit_watch_rules(struct audit_watch *watch)
+{
+        return &watch->rules;
+}
+
+unsigned long audit_watch_inode(struct audit_watch *watch)
+{
+        return watch->ino;
+}
+
+dev_t audit_watch_dev(struct audit_watch *watch)
+{
+        return watch->dev;
+}
+
+/* Initialize a parent watch entry. */
+static struct audit_parent *audit_init_parent(struct nameidata *ndp)
+{
+        struct audit_parent *parent;
+        s32 wd;
+
+        parent = kzalloc(sizeof(*parent), GFP_KERNEL);
+        if (unlikely(!parent))
+                return ERR_PTR(-ENOMEM);
+
+        INIT_LIST_HEAD(&parent->watches);
+        parent->flags = 0;
+
+        inotify_init_watch(&parent->wdata);
+        /* grab a ref so inotify watch hangs around until we take audit_filter_mutex */
+        get_inotify_watch(&parent->wdata);
+        wd = inotify_add_watch(audit_ih, &parent->wdata,
+                               ndp->path.dentry->d_inode, AUDIT_IN_WATCH);
+        if (wd < 0) {
+                audit_free_parent(&parent->wdata);
+                return ERR_PTR(wd);
+        }
+
+        return parent;
+}
+
+/* Initialize a watch entry. */
+static struct audit_watch *audit_init_watch(char *path)
+{
+        struct audit_watch *watch;
+
+        watch = kzalloc(sizeof(*watch), GFP_KERNEL);
+        if (unlikely(!watch))
+                return ERR_PTR(-ENOMEM);
+
+        INIT_LIST_HEAD(&watch->rules);
+        atomic_set(&watch->count, 1);
+        watch->path = path;
+        watch->dev = (dev_t)-1;
+        watch->ino = (unsigned long)-1;
+
+        return watch;
+}
+
+/* Translate a watch string to kernel respresentation. */
+int audit_to_watch(struct audit_krule *krule, char *path, int len, u32 op)
+{
+        struct audit_watch *watch;
+
+        if (!audit_ih)
+                return -EOPNOTSUPP;
+
+        if (path[0] != '/' || path[len-1] == '/' ||
+            krule->listnr != AUDIT_FILTER_EXIT ||
+            op != Audit_equal ||
+            krule->inode_f || krule->watch || krule->tree)
+                return -EINVAL;
+
+        watch = audit_init_watch(path);
+        if (IS_ERR(watch))
+                return PTR_ERR(watch);
+
+        audit_get_watch(watch);
+        krule->watch = watch;
+
+        return 0;
+}
+
+/* Duplicate the given audit watch.  The new watch's rules list is initialized
+ * to an empty list and wlist is undefined. */
+static struct audit_watch *audit_dupe_watch(struct audit_watch *old)
+{
+        char *path;
+        struct audit_watch *new;
+
+        path = kstrdup(old->path, GFP_KERNEL);
+        if (unlikely(!path))
+                return ERR_PTR(-ENOMEM);
+
+        new = audit_init_watch(path);
+        if (IS_ERR(new)) {
+                kfree(path);
+                goto out;
+        }
+
+        new->dev = old->dev;
+        new->ino = old->ino;
+        get_inotify_watch(&old->parent->wdata);
+        new->parent = old->parent;
+
+out:
+        return new;
+}
+
+static void audit_watch_log_rule_change(struct audit_krule *r, struct audit_watch *w, char *op)
+{
+        if (audit_enabled) {
+                struct audit_buffer *ab;
+                ab = audit_log_start(NULL, GFP_NOFS, AUDIT_CONFIG_CHANGE);
+                audit_log_format(ab, "auid=%u ses=%u op=",
+                                 audit_get_loginuid(current),
+                                 audit_get_sessionid(current));
+                audit_log_string(ab, op);
+                audit_log_format(ab, " path=");
+                audit_log_untrustedstring(ab, w->path);
+                audit_log_key(ab, r->filterkey);
+                audit_log_format(ab, " list=%d res=1", r->listnr);
+                audit_log_end(ab);
+        }
+}
+
+/* Update inode info in audit rules based on filesystem event. */
+static void audit_update_watch(struct audit_parent *parent,
+                               const char *dname, dev_t dev,
+                               unsigned long ino, unsigned invalidating)
+{
+        struct audit_watch *owatch, *nwatch, *nextw;
+        struct audit_krule *r, *nextr;
+        struct audit_entry *oentry, *nentry;
+
+        mutex_lock(&audit_filter_mutex);
+        list_for_each_entry_safe(owatch, nextw, &parent->watches, wlist) {
+                if (audit_compare_dname_path(dname, owatch->path, NULL))
+                        continue;
+
+                /* If the update involves invalidating rules, do the inode-based
+                 * filtering now, so we don't omit records. */
+                if (invalidating && current->audit_context)
+                        audit_filter_inodes(current, current->audit_context);
+
+                nwatch = audit_dupe_watch(owatch);
+                if (IS_ERR(nwatch)) {
+                        mutex_unlock(&audit_filter_mutex);
+                        audit_panic("error updating watch, skipping");
+                        return;
+                }
+                nwatch->dev = dev;
+                nwatch->ino = ino;
+
+                list_for_each_entry_safe(r, nextr, &owatch->rules, rlist) {
+
+                        oentry = container_of(r, struct audit_entry, rule);
+                        list_del(&oentry->rule.rlist);
+                        list_del_rcu(&oentry->list);
+
+                        nentry = audit_dupe_rule(&oentry->rule, nwatch);
+                        if (IS_ERR(nentry)) {
+                                list_del(&oentry->rule.list);
+                                audit_panic("error updating watch, removing");
+                        } else {
+                                int h = audit_hash_ino((u32)ino);
+                                list_add(&nentry->rule.rlist, &nwatch->rules);
+                                list_add_rcu(&nentry->list, &audit_inode_hash[h]);
+                                list_replace(&oentry->rule.list,
+                                             &nentry->rule.list);
+                        }
+
+                        audit_watch_log_rule_change(r, owatch, "updated rules");
+
+                        call_rcu(&oentry->rcu, audit_free_rule_rcu);
+                }
+
+                audit_remove_watch(owatch);
+                goto add_watch_to_parent; /* event applies to a single watch */
+        }
+        mutex_unlock(&audit_filter_mutex);
+        return;
+
+add_watch_to_parent:
+        list_add(&nwatch->wlist, &parent->watches);
+        mutex_unlock(&audit_filter_mutex);
+        return;
+}
+
+/* Remove all watches & rules associated with a parent that is going away. */
+static void audit_remove_parent_watches(struct audit_parent *parent)
+{
+        struct audit_watch *w, *nextw;
+        struct audit_krule *r, *nextr;
+        struct audit_entry *e;
+
+        mutex_lock(&audit_filter_mutex);
+        parent->flags |= AUDIT_PARENT_INVALID;
+        list_for_each_entry_safe(w, nextw, &parent->watches, wlist) {
+                list_for_each_entry_safe(r, nextr, &w->rules, rlist) {
+                        e = container_of(r, struct audit_entry, rule);
+                        audit_watch_log_rule_change(r, w, "remove rule");
+                        list_del(&r->rlist);
+                        list_del(&r->list);
+                        list_del_rcu(&e->list);
+                        call_rcu(&e->rcu, audit_free_rule_rcu);
+                }
+                audit_remove_watch(w);
+        }
+        mutex_unlock(&audit_filter_mutex);
+}
+
+/* Unregister inotify watches for parents on in_list.
+ * Generates an IN_IGNORED event. */
+void audit_inotify_unregister(struct list_head *in_list)
+{
+        struct audit_parent *p, *n;
+
+        list_for_each_entry_safe(p, n, in_list, ilist) {
+                list_del(&p->ilist);
+                inotify_rm_watch(audit_ih, &p->wdata);
+                /* the unpin matching the pin in audit_do_del_rule() */
+                unpin_inotify_watch(&p->wdata);
+        }
+}
+
+/* Get path information necessary for adding watches. */
+static int audit_get_nd(char *path, struct nameidata **ndp, struct nameidata **ndw)
+{
+        struct nameidata *ndparent, *ndwatch;
+        int err;
+
+        ndparent = kmalloc(sizeof(*ndparent), GFP_KERNEL);
+        if (unlikely(!ndparent))
+                return -ENOMEM;
+
+        ndwatch = kmalloc(sizeof(*ndwatch), GFP_KERNEL);
+        if (unlikely(!ndwatch)) {
+                kfree(ndparent);
+                return -ENOMEM;
+        }
+
+        err = path_lookup(path, LOOKUP_PARENT, ndparent);
+        if (err) {
+                kfree(ndparent);
+                kfree(ndwatch);
+                return err;
+        }
+
+        err = path_lookup(path, 0, ndwatch);
+        if (err) {
+                kfree(ndwatch);
+                ndwatch = NULL;
+        }
+
+        *ndp = ndparent;
+        *ndw = ndwatch;
+
+        return 0;
+}
+
+/* Release resources used for watch path information. */
+static void audit_put_nd(struct nameidata *ndp, struct nameidata *ndw)
+{
+        if (ndp) {
+                path_put(&ndp->path);
+                kfree(ndp);
+        }
+        if (ndw) {
+                path_put(&ndw->path);
+                kfree(ndw);
+        }
+}
+
+/* Associate the given rule with an existing parent inotify_watch.
+ * Caller must hold audit_filter_mutex. */
+static void audit_add_to_parent(struct audit_krule *krule,
+                                struct audit_parent *parent)
+{
+        struct audit_watch *w, *watch = krule->watch;
+        int watch_found = 0;
+
+        list_for_each_entry(w, &parent->watches, wlist) {
+                if (strcmp(watch->path, w->path))
+                        continue;
+
+                watch_found = 1;
+
+                /* put krule's and initial refs to temporary watch */
+                audit_put_watch(watch);
+                audit_put_watch(watch);
+
+                audit_get_watch(w);
+                krule->watch = watch = w;
+                break;
+        }
+
+        if (!watch_found) {
+                get_inotify_watch(&parent->wdata);
+                watch->parent = parent;
+
+                list_add(&watch->wlist, &parent->watches);
+        }
+        list_add(&krule->rlist, &watch->rules);
+}
+
+/* Find a matching watch entry, or add this one.
+ * Caller must hold audit_filter_mutex. */
+int audit_add_watch(struct audit_krule *krule)
+{
+        struct audit_watch *watch = krule->watch;
+        struct inotify_watch *i_watch;
+        struct audit_parent *parent;
+        struct nameidata *ndp = NULL, *ndw = NULL;
+        int ret = 0;
+
+        mutex_unlock(&audit_filter_mutex);
+
+        /* Avoid calling path_lookup under audit_filter_mutex. */
+        ret = audit_get_nd(watch->path, &ndp, &ndw);
+        if (ret) {
+                /* caller expects mutex locked */
+                mutex_lock(&audit_filter_mutex);
+                goto error;
+        }
+
+        /* update watch filter fields */
+        if (ndw) {
+                watch->dev = ndw->path.dentry->d_inode->i_sb->s_dev;
+                watch->ino = ndw->path.dentry->d_inode->i_ino;
+        }
+
+        /* The audit_filter_mutex must not be held during inotify calls because
+         * we hold it during inotify event callback processing.  If an existing
+         * inotify watch is found, inotify_find_watch() grabs a reference before
+         * returning.
+         */
+        if (inotify_find_watch(audit_ih, ndp->path.dentry->d_inode,
+                               &i_watch) < 0) {
+                parent = audit_init_parent(ndp);
+                if (IS_ERR(parent)) {
+                        /* caller expects mutex locked */
+                        mutex_lock(&audit_filter_mutex);
+                        ret = PTR_ERR(parent);
+                        goto error;
+                }
+        } else
+                parent = container_of(i_watch, struct audit_parent, wdata);
+
+        mutex_lock(&audit_filter_mutex);
+
+        /* parent was moved before we took audit_filter_mutex */
+        if (parent->flags & AUDIT_PARENT_INVALID)
+                ret = -ENOENT;
+        else
+                audit_add_to_parent(krule, parent);
+
+        /* match get in audit_init_parent or inotify_find_watch */
+        put_inotify_watch(&parent->wdata);
+
+error:
+        audit_put_nd(ndp, ndw);         /* NULL args OK */
+        return ret;
+
+}
+
+void audit_remove_watch_rule(struct audit_krule *krule, struct list_head *list)
+{
+        struct audit_watch *watch = krule->watch;
+        struct audit_parent *parent = watch->parent;
+
+        list_del(&krule->rlist);
+
+        if (list_empty(&watch->rules)) {
+                audit_remove_watch(watch);
+
+                if (list_empty(&parent->watches)) {
+                        /* Put parent on the inotify un-registration
+                         * list.  Grab a reference before releasing
+                         * audit_filter_mutex, to be released in
+                         * audit_inotify_unregister().
+                         * If filesystem is going away, just leave
+                         * the sucker alone, eviction will take
+                         * care of it. */
+                        if (pin_inotify_watch(&parent->wdata))
+                                list_add(&parent->ilist, list);
+                }
+        }
+}
+
+/* Update watch data in audit rules based on inotify events. */
+static void audit_handle_ievent(struct inotify_watch *i_watch, u32 wd, u32 mask,
+                         u32 cookie, const char *dname, struct inode *inode)
+{
+        struct audit_parent *parent;
+
+        parent = container_of(i_watch, struct audit_parent, wdata);
+
+        if (mask & (IN_CREATE|IN_MOVED_TO) && inode)
+                audit_update_watch(parent, dname, inode->i_sb->s_dev,
+                                   inode->i_ino, 0);
+        else if (mask & (IN_DELETE|IN_MOVED_FROM))
+                audit_update_watch(parent, dname, (dev_t)-1, (unsigned long)-1, 1);
+        /* inotify automatically removes the watch and sends IN_IGNORED */
+        else if (mask & (IN_DELETE_SELF|IN_UNMOUNT))
+                audit_remove_parent_watches(parent);
+        /* inotify does not remove the watch, so remove it manually */
+        else if(mask & IN_MOVE_SELF) {
+                audit_remove_parent_watches(parent);
+                inotify_remove_watch_locked(audit_ih, i_watch);
+        } else if (mask & IN_IGNORED)
+                put_inotify_watch(i_watch);
+}
+
+static const struct inotify_operations audit_inotify_ops = {
+        .handle_event   = audit_handle_ievent,
+        .destroy_watch  = audit_free_parent,
+};
+
+static int __init audit_watch_init(void)
+{
+        audit_ih = inotify_init(&audit_inotify_ops);
+        if (IS_ERR(audit_ih))
+                audit_panic("cannot initialize inotify handle");
+        return 0;
+}
+subsys_initcall(audit_watch_init);
diff --git a/kernel/auditfilter.c b/kernel/auditfilter.c
index 713098ee5a02..a70604047f3c 100644
--- a/kernel/auditfilter.c
+++ b/kernel/auditfilter.c
@@ -27,7 +27,6 @@
 #include <linux/namei.h>
 #include <linux/netlink.h>
 #include <linux/sched.h>
-#include <linux/inotify.h>
 #include <linux/security.h>
 #include "audit.h"
 
@@ -44,36 +43,6 @@
  *              be written directly provided audit_filter_mutex is held.
  */
 
-/*
- * Reference counting:
- *
- * audit_parent: lifetime is from audit_init_parent() to receipt of an IN_IGNORED
- *      event.  Each audit_watch holds a reference to its associated parent.
- *
- * audit_watch: if added to lists, lifetime is from audit_init_watch() to
- *      audit_remove_watch().  Additionally, an audit_watch may exist
- *      temporarily to assist in searching existing filter data.  Each
- *      audit_krule holds a reference to its associated watch.
- */
-
-struct audit_parent {
-        struct list_head        ilist;  /* entry in inotify registration list */
-        struct list_head        watches; /* associated watches */
-        struct inotify_watch    wdata;  /* inotify watch data */
-        unsigned                flags;  /* status flags */
-};
-
-/*
- * audit_parent status flags:
- *
- * AUDIT_PARENT_INVALID - set anytime rules/watches are auto-removed due to
- * a filesystem event to ensure we're adding audit watches to a valid parent.
- * Technically not needed for IN_DELETE_SELF or IN_UNMOUNT events, as we cannot
- * receive them while we have nameidata, but must be used for IN_MOVE_SELF which
- * we can receive while holding nameidata.
- */
-#define AUDIT_PARENT_INVALID    0x001
-
 /* Audit filter lists, defined in <linux/audit.h> */
 struct list_head audit_filter_list[AUDIT_NR_FILTERS] = {
         LIST_HEAD_INIT(audit_filter_list[0]),
@@ -97,41 +66,6 @@ static struct list_head audit_rules_list[AUDIT_NR_FILTERS] = {
 
 DEFINE_MUTEX(audit_filter_mutex);
 
-/* Inotify events we care about. */
-#define AUDIT_IN_WATCH IN_MOVE|IN_CREATE|IN_DELETE|IN_DELETE_SELF|IN_MOVE_SELF
-
-void audit_free_parent(struct inotify_watch *i_watch)
-{
-        struct audit_parent *parent;
-
-        parent = container_of(i_watch, struct audit_parent, wdata);
-        WARN_ON(!list_empty(&parent->watches));
-        kfree(parent);
-}
-
-static inline void audit_get_watch(struct audit_watch *watch)
-{
-        atomic_inc(&watch->count);
-}
-
-static void audit_put_watch(struct audit_watch *watch)
-{
-        if (atomic_dec_and_test(&watch->count)) {
-                WARN_ON(watch->parent);
-                WARN_ON(!list_empty(&watch->rules));
-                kfree(watch->path);
-                kfree(watch);
-        }
-}
-
-static void audit_remove_watch(struct audit_watch *watch)
-{
-        list_del(&watch->wlist);
-        put_inotify_watch(&watch->parent->wdata);
-        watch->parent = NULL;
-        audit_put_watch(watch); /* match initial get */
-}
-
 static inline void audit_free_rule(struct audit_entry *e)
 {
         int i;
@@ -156,50 +90,6 @@ void audit_free_rule_rcu(struct rcu_head *head)
         audit_free_rule(e);
 }
 
-/* Initialize a parent watch entry. */
-static struct audit_parent *audit_init_parent(struct nameidata *ndp)
-{
-        struct audit_parent *parent;
-        s32 wd;
-
-        parent = kzalloc(sizeof(*parent), GFP_KERNEL);
-        if (unlikely(!parent))
-                return ERR_PTR(-ENOMEM);
-
-        INIT_LIST_HEAD(&parent->watches);
-        parent->flags = 0;
-
-        inotify_init_watch(&parent->wdata);
-        /* grab a ref so inotify watch hangs around until we take audit_filter_mutex */
-        get_inotify_watch(&parent->wdata);
-        wd = inotify_add_watch(audit_ih, &parent->wdata,
-                               ndp->path.dentry->d_inode, AUDIT_IN_WATCH);
-        if (wd < 0) {
-                audit_free_parent(&parent->wdata);
-                return ERR_PTR(wd);
-        }
-
-        return parent;
-}
-
-/* Initialize a watch entry. */
-static struct audit_watch *audit_init_watch(char *path)
-{
-        struct audit_watch *watch;
-
-        watch = kzalloc(sizeof(*watch), GFP_KERNEL);
-        if (unlikely(!watch))
-                return ERR_PTR(-ENOMEM);
-
-        INIT_LIST_HEAD(&watch->rules);
-        atomic_set(&watch->count, 1);
-        watch->path = path;
-        watch->dev = (dev_t)-1;
-        watch->ino = (unsigned long)-1;
-
-        return watch;
-}
-
 /* Initialize an audit filterlist entry. */
 static inline struct audit_entry *audit_init_entry(u32 field_count)
 {
@@ -260,31 +150,6 @@ static inline int audit_to_inode(struct audit_krule *krule,
         return 0;
 }
 
-/* Translate a watch string to kernel respresentation. */
-static int audit_to_watch(struct audit_krule *krule, char *path, int len,
-                          u32 op)
-{
-        struct audit_watch *watch;
-
-        if (!audit_ih)
-                return -EOPNOTSUPP;
-
-        if (path[0] != '/' || path[len-1] == '/' ||
-            krule->listnr != AUDIT_FILTER_EXIT ||
-            op != Audit_equal ||
-            krule->inode_f || krule->watch || krule->tree)
-                return -EINVAL;
-
-        watch = audit_init_watch(path);
-        if (IS_ERR(watch))
-                return PTR_ERR(watch);
-
-        audit_get_watch(watch);
-        krule->watch = watch;
-
-        return 0;
-}
-
 static __u32 *classes[AUDIT_SYSCALL_CLASSES];
 
 int __init audit_register_class(int class, unsigned *list)
@@ -766,7 +631,8 @@ static struct audit_rule_data *audit_krule_to_data(struct audit_krule *krule)
                         break;
                 case AUDIT_WATCH:
                         data->buflen += data->values[i] =
-                                audit_pack_string(&bufp, krule->watch->path);
+                                audit_pack_string(&bufp,
+                                                  audit_watch_path(krule->watch));
                         break;
                 case AUDIT_DIR:
                         data->buflen += data->values[i] =
@@ -818,7 +684,8 @@ static int audit_compare_rule(struct audit_krule *a, struct audit_krule *b)
                                 return 1;
                         break;
                 case AUDIT_WATCH:
-                        if (strcmp(a->watch->path, b->watch->path))
+                        if (strcmp(audit_watch_path(a->watch),
+                                   audit_watch_path(b->watch)))
                                 return 1;
                         break;
                 case AUDIT_DIR:
@@ -844,32 +711,6 @@ static int audit_compare_rule(struct audit_krule *a, struct audit_krule *b)
         return 0;
 }
 
-/* Duplicate the given audit watch.  The new watch's rules list is initialized
- * to an empty list and wlist is undefined. */
-static struct audit_watch *audit_dupe_watch(struct audit_watch *old)
-{
-        char *path;
-        struct audit_watch *new;
-
-        path = kstrdup(old->path, GFP_KERNEL);
-        if (unlikely(!path))
-                return ERR_PTR(-ENOMEM);
-
-        new = audit_init_watch(path);
-        if (IS_ERR(new)) {
-                kfree(path);
-                goto out;
-        }
-
-        new->dev = old->dev;
-        new->ino = old->ino;
-        get_inotify_watch(&old->parent->wdata);
-        new->parent = old->parent;
-
-out:
-        return new;
-}
-
 /* Duplicate LSM field information.  The lsm_rule is opaque, so must be
  * re-initialized. */
 static inline int audit_dupe_lsm_field(struct audit_field *df,
@@ -904,8 +745,8 @@ static inline int audit_dupe_lsm_field(struct audit_field *df,
  * rule with the new rule in the filterlist, then free the old rule.
  * The rlist element is undefined; list manipulations are handled apart from
  * the initial copy. */
-static struct audit_entry *audit_dupe_rule(struct audit_krule *old,
-                                           struct audit_watch *watch)
+struct audit_entry *audit_dupe_rule(struct audit_krule *old,
+                                    struct audit_watch *watch)
 {
         u32 fcount = old->field_count;
         struct audit_entry *entry;
@@ -977,137 +818,6 @@ static struct audit_entry *audit_dupe_rule(struct audit_krule *old,
         return entry;
 }
 
-/* Update inode info in audit rules based on filesystem event. */
-static void audit_update_watch(struct audit_parent *parent,
-                               const char *dname, dev_t dev,
-                               unsigned long ino, unsigned invalidating)
-{
-        struct audit_watch *owatch, *nwatch, *nextw;
-        struct audit_krule *r, *nextr;
-        struct audit_entry *oentry, *nentry;
-
-        mutex_lock(&audit_filter_mutex);
-        list_for_each_entry_safe(owatch, nextw, &parent->watches, wlist) {
-                if (audit_compare_dname_path(dname, owatch->path, NULL))
-                        continue;
-
-                /* If the update involves invalidating rules, do the inode-based
-                 * filtering now, so we don't omit records. */
-                if (invalidating && current->audit_context)
-                        audit_filter_inodes(current, current->audit_context);
-
-                nwatch = audit_dupe_watch(owatch);
-                if (IS_ERR(nwatch)) {
-                        mutex_unlock(&audit_filter_mutex);
-                        audit_panic("error updating watch, skipping");
-                        return;
-                }
-                nwatch->dev = dev;
-                nwatch->ino = ino;
-
-                list_for_each_entry_safe(r, nextr, &owatch->rules, rlist) {
-
-                        oentry = container_of(r, struct audit_entry, rule);
-                        list_del(&oentry->rule.rlist);
-                        list_del_rcu(&oentry->list);
-
-                        nentry = audit_dupe_rule(&oentry->rule, nwatch);
-                        if (IS_ERR(nentry)) {
-                                list_del(&oentry->rule.list);
-                                audit_panic("error updating watch, removing");
-                        } else {
-                                int h = audit_hash_ino((u32)ino);
-                                list_add(&nentry->rule.rlist, &nwatch->rules);
-                                list_add_rcu(&nentry->list, &audit_inode_hash[h]);
-                                list_replace(&oentry->rule.list,
-                                             &nentry->rule.list);
-                        }
-
-                        call_rcu(&oentry->rcu, audit_free_rule_rcu);
-                }
-
-                if (audit_enabled) {
-                        struct audit_buffer *ab;
-                        ab = audit_log_start(NULL, GFP_NOFS,
-                                AUDIT_CONFIG_CHANGE);
-                        audit_log_format(ab, "auid=%u ses=%u",
-                                audit_get_loginuid(current),
-                                audit_get_sessionid(current));
-                        audit_log_format(ab,
-                                " op=updated rules specifying path=");
-                        audit_log_untrustedstring(ab, owatch->path);
-                        audit_log_format(ab, " with dev=%u ino=%lu\n",
-                                 dev, ino);
-                        audit_log_format(ab, " list=%d res=1", r->listnr);
-                        audit_log_end(ab);
-                }
-                audit_remove_watch(owatch);
-                goto add_watch_to_parent; /* event applies to a single watch */
-        }
-        mutex_unlock(&audit_filter_mutex);
-        return;
-
-add_watch_to_parent:
-        list_add(&nwatch->wlist, &parent->watches);
-        mutex_unlock(&audit_filter_mutex);
-        return;
-}
-
-/* Remove all watches & rules associated with a parent that is going away. */
-static void audit_remove_parent_watches(struct audit_parent *parent)
-{
-        struct audit_watch *w, *nextw;
-        struct audit_krule *r, *nextr;
-        struct audit_entry *e;
-
-        mutex_lock(&audit_filter_mutex);
-        parent->flags |= AUDIT_PARENT_INVALID;
-        list_for_each_entry_safe(w, nextw, &parent->watches, wlist) {
-                list_for_each_entry_safe(r, nextr, &w->rules, rlist) {
-                        e = container_of(r, struct audit_entry, rule);
-                        if (audit_enabled) {
-                                struct audit_buffer *ab;
-                                ab = audit_log_start(NULL, GFP_NOFS,
-                                        AUDIT_CONFIG_CHANGE);
-                                audit_log_format(ab, "auid=%u ses=%u",
-                                        audit_get_loginuid(current),
-                                        audit_get_sessionid(current));
-                                audit_log_format(ab, " op=remove rule path=");
-                                audit_log_untrustedstring(ab, w->path);
-                                if (r->filterkey) {
-                                        audit_log_format(ab, " key=");
-                                        audit_log_untrustedstring(ab,
-                                                        r->filterkey);
-                                } else
-                                        audit_log_format(ab, " key=(null)");
-                                audit_log_format(ab, " list=%d res=1",
-                                        r->listnr);
-                                audit_log_end(ab);
-                        }
-                        list_del(&r->rlist);
-                        list_del(&r->list);
-                        list_del_rcu(&e->list);
-                        call_rcu(&e->rcu, audit_free_rule_rcu);
-                }
-                audit_remove_watch(w);
-        }
-        mutex_unlock(&audit_filter_mutex);
-}
-
-/* Unregister inotify watches for parents on in_list.
- * Generates an IN_IGNORED event. */
-static void audit_inotify_unregister(struct list_head *in_list)
-{
-        struct audit_parent *p, *n;
-
-        list_for_each_entry_safe(p, n, in_list, ilist) {
-                list_del(&p->ilist);
-                inotify_rm_watch(audit_ih, &p->wdata);
-                /* the unpin matching the pin in audit_do_del_rule() */
-                unpin_inotify_watch(&p->wdata);
-        }
-}
-
 /* Find an existing audit rule.
  * Caller must hold audit_filter_mutex to prevent stale rule data. */
 static struct audit_entry *audit_find_rule(struct audit_entry *entry,
@@ -1145,134 +855,6 @@ out:
         return found;
 }
 
-/* Get path information necessary for adding watches. */
-static int audit_get_nd(char *path, struct nameidata **ndp,
-                        struct nameidata **ndw)
-{
-        struct nameidata *ndparent, *ndwatch;
-        int err;
-
-        ndparent = kmalloc(sizeof(*ndparent), GFP_KERNEL);
-        if (unlikely(!ndparent))
-                return -ENOMEM;
-
-        ndwatch = kmalloc(sizeof(*ndwatch), GFP_KERNEL);
-        if (unlikely(!ndwatch)) {
-                kfree(ndparent);
-                return -ENOMEM;
-        }
-
-        err = path_lookup(path, LOOKUP_PARENT, ndparent);
-        if (err) {
-                kfree(ndparent);
-                kfree(ndwatch);
-                return err;
-        }
-
-        err = path_lookup(path, 0, ndwatch);
-        if (err) {
-                kfree(ndwatch);
-                ndwatch = NULL;
-        }
-
-        *ndp = ndparent;
-        *ndw = ndwatch;
-
-        return 0;
-}
-
-/* Release resources used for watch path information. */
-static void audit_put_nd(struct nameidata *ndp, struct nameidata *ndw)
-{
-        if (ndp) {
-                path_put(&ndp->path);
-                kfree(ndp);
-        }
-        if (ndw) {
-                path_put(&ndw->path);
-                kfree(ndw);
-        }
-}
-
-/* Associate the given rule with an existing parent inotify_watch.
- * Caller must hold audit_filter_mutex. */
-static void audit_add_to_parent(struct audit_krule *krule,
-                                struct audit_parent *parent)
-{
-        struct audit_watch *w, *watch = krule->watch;
-        int watch_found = 0;
-
-        list_for_each_entry(w, &parent->watches, wlist) {
-                if (strcmp(watch->path, w->path))
-                        continue;
-
-                watch_found = 1;
-
-                /* put krule's and initial refs to temporary watch */
-                audit_put_watch(watch);
-                audit_put_watch(watch);
-
-                audit_get_watch(w);
-                krule->watch = watch = w;
-                break;
-        }
-
-        if (!watch_found) {
-                get_inotify_watch(&parent->wdata);
-                watch->parent = parent;
-
-                list_add(&watch->wlist, &parent->watches);
-        }
-        list_add(&krule->rlist, &watch->rules);
-}
-
-/* Find a matching watch entry, or add this one.
- * Caller must hold audit_filter_mutex. */
-static int audit_add_watch(struct audit_krule *krule, struct nameidata *ndp,
-                           struct nameidata *ndw)
-{
-        struct audit_watch *watch = krule->watch;
-        struct inotify_watch *i_watch;
-        struct audit_parent *parent;
-        int ret = 0;
-
-        /* update watch filter fields */
-        if (ndw) {
-                watch->dev = ndw->path.dentry->d_inode->i_sb->s_dev;
-                watch->ino = ndw->path.dentry->d_inode->i_ino;
-        }
-
-        /* The audit_filter_mutex must not be held during inotify calls because
-         * we hold it during inotify event callback processing.  If an existing
-         * inotify watch is found, inotify_find_watch() grabs a reference before
-         * returning.
-         */
-        mutex_unlock(&audit_filter_mutex);
-
-        if (inotify_find_watch(audit_ih, ndp->path.dentry->d_inode,
-                               &i_watch) < 0) {
-                parent = audit_init_parent(ndp);
-                if (IS_ERR(parent)) {
-                        /* caller expects mutex locked */
-                        mutex_lock(&audit_filter_mutex);
-                        return PTR_ERR(parent);
-                }
-        } else
-                parent = container_of(i_watch, struct audit_parent, wdata);
-
-        mutex_lock(&audit_filter_mutex);
-
-        /* parent was moved before we took audit_filter_mutex */
-        if (parent->flags & AUDIT_PARENT_INVALID)
-                ret = -ENOENT;
-        else
-                audit_add_to_parent(krule, parent);
-
-        /* match get in audit_init_parent or inotify_find_watch */
-        put_inotify_watch(&parent->wdata);
-        return ret;
-}
-
 static u64 prio_low = ~0ULL/2;
 static u64 prio_high = ~0ULL/2 - 1;
 
@@ -1282,7 +864,6 @@ static inline int audit_add_rule(struct audit_entry *entry)
         struct audit_entry *e;
         struct audit_watch *watch = entry->rule.watch;
         struct audit_tree *tree = entry->rule.tree;
-        struct nameidata *ndp = NULL, *ndw = NULL;
         struct list_head *list;
         int h, err;
 #ifdef CONFIG_AUDITSYSCALL
@@ -1296,8 +877,8 @@ static inline int audit_add_rule(struct audit_entry *entry)
 
         mutex_lock(&audit_filter_mutex);
         e = audit_find_rule(entry, &list);
-        mutex_unlock(&audit_filter_mutex);
         if (e) {
+                mutex_unlock(&audit_filter_mutex);
                 err = -EEXIST;
                 /* normally audit_add_tree_rule() will free it on failure */
                 if (tree)
@@ -1305,22 +886,16 @@ static inline int audit_add_rule(struct audit_entry *entry)
                 goto error;
         }
 
-        /* Avoid calling path_lookup under audit_filter_mutex. */
-        if (watch) {
-                err = audit_get_nd(watch->path, &ndp, &ndw);
-                if (err)
-                        goto error;
-        }
-
-        mutex_lock(&audit_filter_mutex);
         if (watch) {
                 /* audit_filter_mutex is dropped and re-taken during this call */
-                err = audit_add_watch(&entry->rule, ndp, ndw);
+                err = audit_add_watch(&entry->rule);
                 if (err) {
                         mutex_unlock(&audit_filter_mutex);
                         goto error;
                 }
-                h = audit_hash_ino((u32)watch->ino);
+                /* entry->rule.watch may have changed during audit_add_watch() */
+                watch = entry->rule.watch;
+                h = audit_hash_ino((u32)audit_watch_inode(watch));
                 list = &audit_inode_hash[h];
         }
         if (tree) {
@@ -1358,11 +933,9 @@ static inline int audit_add_rule(struct audit_entry *entry)
 #endif
         mutex_unlock(&audit_filter_mutex);
 
-        audit_put_nd(ndp, ndw);         /* NULL args OK */
         return 0;
 
 error:
-        audit_put_nd(ndp, ndw);         /* NULL args OK */
         if (watch)
                 audit_put_watch(watch); /* tmp watch, matches initial get */
         return err;
@@ -1372,7 +945,7 @@ error:
 static inline int audit_del_rule(struct audit_entry *entry)
 {
         struct audit_entry  *e;
-        struct audit_watch *watch, *tmp_watch = entry->rule.watch;
+        struct audit_watch *watch = entry->rule.watch;
         struct audit_tree *tree = entry->rule.tree;
         struct list_head *list;
         LIST_HEAD(inotify_list);
@@ -1394,29 +967,8 @@ static inline int audit_del_rule(struct audit_entry *entry)
                 goto out;
         }
 
-        watch = e->rule.watch;
-        if (watch) {
-                struct audit_parent *parent = watch->parent;
-
-                list_del(&e->rule.rlist);
-
-                if (list_empty(&watch->rules)) {
-                        audit_remove_watch(watch);
-
-                        if (list_empty(&parent->watches)) {
-                                /* Put parent on the inotify un-registration
-                                 * list.  Grab a reference before releasing
-                                 * audit_filter_mutex, to be released in
-                                 * audit_inotify_unregister().
-                                 * If filesystem is going away, just leave
-                                 * the sucker alone, eviction will take
-                                 * care of it.
-                                 */
-                                if (pin_inotify_watch(&parent->wdata))
-                                        list_add(&parent->ilist, &inotify_list);
-                        }
-                }
-        }
+        if (e->rule.watch)
+                audit_remove_watch_rule(&e->rule, &inotify_list);
 
         if (e->rule.tree)
                 audit_remove_tree_rule(&e->rule);
@@ -1438,8 +990,8 @@ static inline int audit_del_rule(struct audit_entry *entry)
                 audit_inotify_unregister(&inotify_list);
 
 out:
-        if (tmp_watch)
-                audit_put_watch(tmp_watch); /* match initial get */
+        if (watch)
+                audit_put_watch(watch); /* match initial get */
         if (tree)
                 audit_put_tree(tree);   /* that's the temporary one */
 
@@ -1527,11 +1079,9 @@ static void audit_log_rule_change(uid_t loginuid, u32 sessionid, u32 sid,
                         security_release_secctx(ctx, len);
                 }
         }
-        audit_log_format(ab, " op=%s rule key=", action);
-        if (rule->filterkey)
-                audit_log_untrustedstring(ab, rule->filterkey);
-        else
-                audit_log_format(ab, "(null)");
+        audit_log_format(ab, " op=");
+        audit_log_string(ab, action);
+        audit_log_key(ab, rule->filterkey);
         audit_log_format(ab, " list=%d res=%d", rule->listnr, res);
         audit_log_end(ab);
 }
@@ -1595,7 +1145,7 @@ int audit_receive_filter(int type, int pid, int uid, int seq, void *data,
                         return PTR_ERR(entry);
 
                 err = audit_add_rule(entry);
-                audit_log_rule_change(loginuid, sessionid, sid, "add",
+                audit_log_rule_change(loginuid, sessionid, sid, "add rule",
                                       &entry->rule, !err);
 
                 if (err)
@@ -1611,7 +1161,7 @@ int audit_receive_filter(int type, int pid, int uid, int seq, void *data,
                         return PTR_ERR(entry);
 
                 err = audit_del_rule(entry);
-                audit_log_rule_change(loginuid, sessionid, sid, "remove",
+                audit_log_rule_change(loginuid, sessionid, sid, "remove rule",
                                       &entry->rule, !err);
 
                 audit_free_rule(entry);
@@ -1793,7 +1343,7 @@ static int update_lsm_rule(struct audit_krule *r)
                 list_del(&r->list);
         } else {
                 if (watch) {
-                        list_add(&nentry->rule.rlist, &watch->rules);
+                        list_add(&nentry->rule.rlist, audit_watch_rules(watch));
                         list_del(&r->rlist);
                 } else if (tree)
                         list_replace_init(&r->rlist, &nentry->rule.rlist);
@@ -1829,27 +1379,3 @@ int audit_update_lsm_rules(void)
 
         return err;
 }
-
-/* Update watch data in audit rules based on inotify events. */
-void audit_handle_ievent(struct inotify_watch *i_watch, u32 wd, u32 mask,
-                         u32 cookie, const char *dname, struct inode *inode)
-{
-        struct audit_parent *parent;
-
-        parent = container_of(i_watch, struct audit_parent, wdata);
-
-        if (mask & (IN_CREATE|IN_MOVED_TO) && inode)
-                audit_update_watch(parent, dname, inode->i_sb->s_dev,
-                                   inode->i_ino, 0);
-        else if (mask & (IN_DELETE|IN_MOVED_FROM))
-                audit_update_watch(parent, dname, (dev_t)-1, (unsigned long)-1, 1);
-        /* inotify automatically removes the watch and sends IN_IGNORED */
-        else if (mask & (IN_DELETE_SELF|IN_UNMOUNT))
-                audit_remove_parent_watches(parent);
-        /* inotify does not remove the watch, so remove it manually */
-        else if(mask & IN_MOVE_SELF) {
-                audit_remove_parent_watches(parent);
-                inotify_remove_watch_locked(audit_ih, i_watch);
-        } else if (mask & IN_IGNORED)
-                put_inotify_watch(i_watch);
-}
diff --git a/kernel/auditsc.c b/kernel/auditsc.c
index 7d6ac7c1f414..68d3c6a0ecd6 100644
--- a/kernel/auditsc.c
+++ b/kernel/auditsc.c
@@ -199,6 +199,7 @@ struct audit_context {
 
         struct audit_tree_refs *trees, *first_trees;
         int tree_count;
+        struct list_head killed_trees;
 
         int type;
         union {
@@ -548,9 +549,9 @@ static int audit_filter_rules(struct task_struct *tsk,
                         }
                         break;
                 case AUDIT_WATCH:
-                        if (name && rule->watch->ino != (unsigned long)-1)
-                                result = (name->dev == rule->watch->dev &&
-                                          name->ino == rule->watch->ino);
+                        if (name && audit_watch_inode(rule->watch) != (unsigned long)-1)
+                                result = (name->dev == audit_watch_dev(rule->watch) &&
+                                          name->ino == audit_watch_inode(rule->watch));
                         break;
                 case AUDIT_DIR:
                         if (ctx)
@@ -853,6 +854,7 @@ static inline struct audit_context *audit_alloc_context(enum audit_state state)
         if (!(context = kmalloc(sizeof(*context), GFP_KERNEL)))
                 return NULL;
         audit_zero_context(context, state);
+        INIT_LIST_HEAD(&context->killed_trees);
         return context;
 }
 
@@ -1024,8 +1026,8 @@ static int audit_log_single_execve_arg(struct audit_context *context,
 {
         char arg_num_len_buf[12];
         const char __user *tmp_p = p;
-        /* how many digits are in arg_num? 3 is the length of " a=" */
-        size_t arg_num_len = snprintf(arg_num_len_buf, 12, "%d", arg_num) + 3;
+        /* how many digits are in arg_num? 5 is the length of ' a=""' */
+        size_t arg_num_len = snprintf(arg_num_len_buf, 12, "%d", arg_num) + 5;
         size_t len, len_left, to_send;
         size_t max_execve_audit_len = MAX_EXECVE_AUDIT_LEN;
         unsigned int i, has_cntl = 0, too_long = 0;
@@ -1137,7 +1139,7 @@ static int audit_log_single_execve_arg(struct audit_context *context,
                 if (has_cntl)
                         audit_log_n_hex(*ab, buf, to_send);
                 else
-                        audit_log_format(*ab, "\"%s\"", buf);
+                        audit_log_string(*ab, buf);
 
                 p += to_send;
                 len_left -= to_send;
@@ -1372,11 +1374,7 @@ static void audit_log_exit(struct audit_context *context, struct task_struct *ts
 
 
         audit_log_task_info(ab, tsk);
-        if (context->filterkey) {
-                audit_log_format(ab, " key=");
-                audit_log_untrustedstring(ab, context->filterkey);
-        } else
-                audit_log_format(ab, " key=(null)");
+        audit_log_key(ab, context->filterkey);
         audit_log_end(ab);
 
         for (aux = context->aux; aux; aux = aux->next) {
@@ -1549,6 +1547,8 @@ void audit_free(struct task_struct *tsk)
         /* that can happen only if we are called from do_exit() */
         if (context->in_syscall && context->current_state == AUDIT_RECORD_CONTEXT)
                 audit_log_exit(context, tsk);
+        if (!list_empty(&context->killed_trees))
+                audit_kill_trees(&context->killed_trees);
 
         audit_free_context(context);
 }
@@ -1692,6 +1692,9 @@ void audit_syscall_exit(int valid, long return_code)
         context->in_syscall = 0;
         context->prio = context->state == AUDIT_RECORD_CONTEXT ? ~0ULL : 0;
 
+        if (!list_empty(&context->killed_trees))
+                audit_kill_trees(&context->killed_trees);
+
         if (context->previous) {
                 struct audit_context *new_context = context->previous;
                 context->previous  = NULL;
@@ -2525,3 +2528,11 @@ void audit_core_dumps(long signr)
         audit_log_format(ab, " sig=%ld", signr);
         audit_log_end(ab);
 }
+
+struct list_head *audit_killed_trees(void)
+{
+        struct audit_context *ctx = current->audit_context;
+        if (likely(!ctx || !ctx->in_syscall))
+                return NULL;
+        return &ctx->killed_trees;
+}
diff --git a/kernel/cgroup.c b/kernel/cgroup.c
index 3fb789f6df94..c7ece8f027f2 100644
--- a/kernel/cgroup.c
+++ b/kernel/cgroup.c
@@ -47,6 +47,7 @@
 #include <linux/hash.h>
 #include <linux/namei.h>
 #include <linux/smp_lock.h>
+#include <linux/pid_namespace.h>
 
 #include <asm/atomic.h>
 
@@ -599,6 +600,7 @@ static struct inode_operations cgroup_dir_inode_operations;
 static struct file_operations proc_cgroupstats_operations;
 
 static struct backing_dev_info cgroup_backing_dev_info = {
+        .name           = "cgroup",
         .capabilities   = BDI_CAP_NO_ACCT_AND_WRITEBACK,
 };
 
@@ -734,16 +736,28 @@ static void cgroup_d_remove_dir(struct dentry *dentry)
  * reference to css->refcnt. In general, this refcnt is expected to goes down
  * to zero, soon.
  *
- * CGRP_WAIT_ON_RMDIR flag is modified under cgroup's inode->i_mutex;
+ * CGRP_WAIT_ON_RMDIR flag is set under cgroup's inode->i_mutex;
  */
 DECLARE_WAIT_QUEUE_HEAD(cgroup_rmdir_waitq);
 
-static void cgroup_wakeup_rmdir_waiters(const struct cgroup *cgrp)
+static void cgroup_wakeup_rmdir_waiter(struct cgroup *cgrp)
 {
-        if (unlikely(test_bit(CGRP_WAIT_ON_RMDIR, &cgrp->flags)))
+        if (unlikely(test_and_clear_bit(CGRP_WAIT_ON_RMDIR, &cgrp->flags)))
                 wake_up_all(&cgroup_rmdir_waitq);
 }
 
+void cgroup_exclude_rmdir(struct cgroup_subsys_state *css)
+{
+        css_get(css);
+}
+
+void cgroup_release_and_wakeup_rmdir(struct cgroup_subsys_state *css)
+{
+        cgroup_wakeup_rmdir_waiter(css->cgroup);
+        css_put(css);
+}
+
+
 static int rebind_subsystems(struct cgroupfs_root *root,
                               unsigned long final_bits)
 {
@@ -843,6 +857,11 @@ static int parse_cgroupfs_options(char *data,
                                      struct cgroup_sb_opts *opts)
 {
         char *token, *o = data ?: "all";
+        unsigned long mask = (unsigned long)-1;
+
+#ifdef CONFIG_CPUSETS
+        mask = ~(1UL << cpuset_subsys_id);
+#endif
 
         opts->subsys_bits = 0;
         opts->flags = 0;
@@ -887,6 +906,15 @@ static int parse_cgroupfs_options(char *data,
                 }
         }
 
+        /*
+         * Option noprefix was introduced just for backward compatibility
+         * with the old cpuset, so we allow noprefix only if mounting just
+         * the cpuset subsystem.
+         */
+        if (test_bit(ROOT_NOPREFIX, &opts->flags) &&
+            (opts->subsys_bits & mask))
+                return -EINVAL;
+
         /* We can't have an empty hierarchy */
         if (!opts->subsys_bits)
                 return -EINVAL;
@@ -946,6 +974,7 @@ static void init_cgroup_housekeeping(struct cgroup *cgrp)
         INIT_LIST_HEAD(&cgrp->children);
         INIT_LIST_HEAD(&cgrp->css_sets);
         INIT_LIST_HEAD(&cgrp->release_list);
+        INIT_LIST_HEAD(&cgrp->pids_list);
         init_rwsem(&cgrp->pids_mutex);
 }
 static void init_cgroup_root(struct cgroupfs_root *root)
@@ -1343,7 +1372,7 @@ int cgroup_attach_task(struct cgroup *cgrp, struct task_struct *tsk)
          * wake up rmdir() waiter. the rmdir should fail since the cgroup
          * is no longer empty.
          */
-        cgroup_wakeup_rmdir_waiters(cgrp);
+        cgroup_wakeup_rmdir_waiter(cgrp);
         return 0;
 }
 
@@ -2187,12 +2216,30 @@ err:
         return ret;
 }
 
+/*
+ * Cache pids for all threads in the same pid namespace that are
+ * opening the same "tasks" file.
+ */
+struct cgroup_pids {
+        /* The node in cgrp->pids_list */
+        struct list_head list;
+        /* The cgroup those pids belong to */
+        struct cgroup *cgrp;
+        /* The namepsace those pids belong to */
+        struct pid_namespace *ns;
+        /* Array of process ids in the cgroup */
+        pid_t *tasks_pids;
+        /* How many files are using the this tasks_pids array */
+        int use_count;
+        /* Length of the current tasks_pids array */
+        int length;
+};
+
 static int cmppid(const void *a, const void *b)
 {
         return *(pid_t *)a - *(pid_t *)b;
 }
 
-
 /*
  * seq_file methods for the "tasks" file. The seq_file position is the
  * next pid to display; the seq_file iterator is a pointer to the pid
@@ -2207,45 +2254,47 @@ static void *cgroup_tasks_start(struct seq_file *s, loff_t *pos)
          * after a seek to the start). Use a binary-search to find the
          * next pid to display, if any
          */
-        struct cgroup *cgrp = s->private;
+        struct cgroup_pids *cp = s->private;
+        struct cgroup *cgrp = cp->cgrp;
         int index = 0, pid = *pos;
         int *iter;
 
         down_read(&cgrp->pids_mutex);
         if (pid) {
-                int end = cgrp->pids_length;
+                int end = cp->length;
 
                 while (index < end) {
                         int mid = (index + end) / 2;
-                        if (cgrp->tasks_pids[mid] == pid) {
+                        if (cp->tasks_pids[mid] == pid) {
                                 index = mid;
                                 break;
-                        } else if (cgrp->tasks_pids[mid] <= pid)
+                        } else if (cp->tasks_pids[mid] <= pid)
                                 index = mid + 1;
                         else
                                 end = mid;
                 }
         }
         /* If we're off the end of the array, we're done */
-        if (index >= cgrp->pids_length)
+        if (index >= cp->length)
                 return NULL;
         /* Update the abstract position to be the actual pid that we found */
-        iter = cgrp->tasks_pids + index;
+        iter = cp->tasks_pids + index;
         *pos = *iter;
         return iter;
 }
 
 static void cgroup_tasks_stop(struct seq_file *s, void *v)
 {
-        struct cgroup *cgrp = s->private;
+        struct cgroup_pids *cp = s->private;
+        struct cgroup *cgrp = cp->cgrp;
         up_read(&cgrp->pids_mutex);
 }
 
 static void *cgroup_tasks_next(struct seq_file *s, void *v, loff_t *pos)
 {
-        struct cgroup *cgrp = s->private;
+        struct cgroup_pids *cp = s->private;
         int *p = v;
-        int *end = cgrp->tasks_pids + cgrp->pids_length;
+        int *end = cp->tasks_pids + cp->length;
 
         /*
          * Advance to the next pid in the array. If this goes off the
@@ -2272,26 +2321,33 @@ static struct seq_operations cgroup_tasks_seq_operations = {
         .show = cgroup_tasks_show,
 };
 
-static void release_cgroup_pid_array(struct cgroup *cgrp)
+static void release_cgroup_pid_array(struct cgroup_pids *cp)
 {
+        struct cgroup *cgrp = cp->cgrp;
+
         down_write(&cgrp->pids_mutex);
-        BUG_ON(!cgrp->pids_use_count);
-        if (!--cgrp->pids_use_count) {
-                kfree(cgrp->tasks_pids);
-                cgrp->tasks_pids = NULL;
-                cgrp->pids_length = 0;
+        BUG_ON(!cp->use_count);
+        if (!--cp->use_count) {
+                list_del(&cp->list);
+                put_pid_ns(cp->ns);
+                kfree(cp->tasks_pids);
+                kfree(cp);
         }
         up_write(&cgrp->pids_mutex);
 }
 
 static int cgroup_tasks_release(struct inode *inode, struct file *file)
 {
-        struct cgroup *cgrp = __d_cgrp(file->f_dentry->d_parent);
+        struct seq_file *seq;
+        struct cgroup_pids *cp;
 
         if (!(file->f_mode & FMODE_READ))
                 return 0;
 
-        release_cgroup_pid_array(cgrp);
+        seq = file->private_data;
+        cp = seq->private;
+
+        release_cgroup_pid_array(cp);
         return seq_release(inode, file);
 }
 
@@ -2310,6 +2366,8 @@ static struct file_operations cgroup_tasks_operations = {
 static int cgroup_tasks_open(struct inode *unused, struct file *file)
 {
         struct cgroup *cgrp = __d_cgrp(file->f_dentry->d_parent);
+        struct pid_namespace *ns = current->nsproxy->pid_ns;
+        struct cgroup_pids *cp;
         pid_t *pidarray;
         int npids;
         int retval;
@@ -2336,20 +2394,37 @@ static int cgroup_tasks_open(struct inode *unused, struct file *file)
          * array if necessary
          */
         down_write(&cgrp->pids_mutex);
-        kfree(cgrp->tasks_pids);
-        cgrp->tasks_pids = pidarray;
-        cgrp->pids_length = npids;
-        cgrp->pids_use_count++;
+
+        list_for_each_entry(cp, &cgrp->pids_list, list) {
+                if (ns == cp->ns)
+                        goto found;
+        }
+
+        cp = kzalloc(sizeof(*cp), GFP_KERNEL);
+        if (!cp) {
+                up_write(&cgrp->pids_mutex);
+                kfree(pidarray);
+                return -ENOMEM;
+        }
+        cp->cgrp = cgrp;
+        cp->ns = ns;
+        get_pid_ns(ns);
+        list_add(&cp->list, &cgrp->pids_list);
+found:
+        kfree(cp->tasks_pids);
+        cp->tasks_pids = pidarray;
+        cp->length = npids;
+        cp->use_count++;
         up_write(&cgrp->pids_mutex);
 
         file->f_op = &cgroup_tasks_operations;
 
         retval = seq_open(file, &cgroup_tasks_seq_operations);
         if (retval) {
-                release_cgroup_pid_array(cgrp);
+                release_cgroup_pid_array(cp);
                 return retval;
         }
-        ((struct seq_file *)file->private_data)->private = cgrp;
+        ((struct seq_file *)file->private_data)->private = cp;
         return 0;
 }
 
@@ -2682,33 +2757,42 @@ again:
         mutex_unlock(&cgroup_mutex);
 
         /*
+         * In general, subsystem has no css->refcnt after pre_destroy(). But
+         * in racy cases, subsystem may have to get css->refcnt after
+         * pre_destroy() and it makes rmdir return with -EBUSY. This sometimes
+         * make rmdir return -EBUSY too often. To avoid that, we use waitqueue
+         * for cgroup's rmdir. CGRP_WAIT_ON_RMDIR is for synchronizing rmdir
+         * and subsystem's reference count handling. Please see css_get/put
+         * and css_tryget() and cgroup_wakeup_rmdir_waiter() implementation.
+         */
+        set_bit(CGRP_WAIT_ON_RMDIR, &cgrp->flags);
+
+        /*
          * Call pre_destroy handlers of subsys. Notify subsystems
          * that rmdir() request comes.
          */
         ret = cgroup_call_pre_destroy(cgrp);
-        if (ret)
+        if (ret) {
+                clear_bit(CGRP_WAIT_ON_RMDIR, &cgrp->flags);
                 return ret;
+        }
 
         mutex_lock(&cgroup_mutex);
         parent = cgrp->parent;
         if (atomic_read(&cgrp->count) || !list_empty(&cgrp->children)) {
+                clear_bit(CGRP_WAIT_ON_RMDIR, &cgrp->flags);
                 mutex_unlock(&cgroup_mutex);
                 return -EBUSY;
         }
-        /*
-         * css_put/get is provided for subsys to grab refcnt to css. In typical
-         * case, subsystem has no reference after pre_destroy(). But, under
-         * hierarchy management, some *temporal* refcnt can be hold.
-         * To avoid returning -EBUSY to a user, waitqueue is used. If subsys
-         * is really busy, it should return -EBUSY at pre_destroy(). wake_up
-         * is called when css_put() is called and refcnt goes down to 0.
-         */
-        set_bit(CGRP_WAIT_ON_RMDIR, &cgrp->flags);
         prepare_to_wait(&cgroup_rmdir_waitq, &wait, TASK_INTERRUPTIBLE);
-
         if (!cgroup_clear_css_refs(cgrp)) {
                 mutex_unlock(&cgroup_mutex);
-                schedule();
+                /*
+                 * Because someone may call cgroup_wakeup_rmdir_waiter() before
+                 * prepare_to_wait(), we need to check this flag.
+                 */
+                if (test_bit(CGRP_WAIT_ON_RMDIR, &cgrp->flags))
+                        schedule();
                 finish_wait(&cgroup_rmdir_waitq, &wait);
                 clear_bit(CGRP_WAIT_ON_RMDIR, &cgrp->flags);
                 if (signal_pending(current))
@@ -3280,7 +3364,7 @@ void __css_put(struct cgroup_subsys_state *css)
                         set_bit(CGRP_RELEASABLE, &cgrp->flags);
                         check_for_release(cgrp);
                 }
-                cgroup_wakeup_rmdir_waiters(cgrp);
+                cgroup_wakeup_rmdir_waiter(cgrp);
         }
         rcu_read_unlock();
 }
diff --git a/kernel/cpu.c b/kernel/cpu.c
index 395b6974dc8d..6ba0f1ecb212 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -34,14 +34,11 @@ static struct {
          * an ongoing cpu hotplug operation.
          */
         int refcount;
-} cpu_hotplug;
-
-void __init cpu_hotplug_init(void)
-{
-        cpu_hotplug.active_writer = NULL;
-        mutex_init(&cpu_hotplug.lock);
-        cpu_hotplug.refcount = 0;
-}
+} cpu_hotplug = {
+        .active_writer = NULL,
+        .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock),
+        .refcount = 0,
+};
 
 #ifdef CONFIG_HOTPLUG_CPU
 
@@ -404,6 +401,7 @@ int disable_nonboot_cpus(void)
                         break;
                 }
         }
+
         if (!error) {
                 BUG_ON(num_online_cpus() > 1);
                 /* Make sure the CPUs won't be enabled by someone else */
@@ -416,6 +414,14 @@ int disable_nonboot_cpus(void)
         return error;
 }
 
+void __weak arch_enable_nonboot_cpus_begin(void)
+{
+}
+
+void __weak arch_enable_nonboot_cpus_end(void)
+{
+}
+
 void __ref enable_nonboot_cpus(void)
 {
         int cpu, error;
@@ -427,6 +433,9 @@ void __ref enable_nonboot_cpus(void)
                 goto out;
 
         printk("Enabling non-boot CPUs ...\n");
+
+        arch_enable_nonboot_cpus_begin();
+
         for_each_cpu(cpu, frozen_cpus) {
                 error = _cpu_up(cpu, 1);
                 if (!error) {
@@ -435,6 +444,9 @@ void __ref enable_nonboot_cpus(void)
                 }
                 printk(KERN_WARNING "Error taking CPU%d up: %d\n", cpu, error);
         }
+
+        arch_enable_nonboot_cpus_end();
+
         cpumask_clear(frozen_cpus);
 out:
         cpu_maps_update_done();
diff --git a/kernel/cpuset.c b/kernel/cpuset.c
index d5a7e17474ee..7e75a41bd508 100644
--- a/kernel/cpuset.c
+++ b/kernel/cpuset.c
@@ -97,12 +97,6 @@ struct cpuset {
 
         struct cpuset *parent;          /* my parent */
 
-        /*
-         * Copy of global cpuset_mems_generation as of the most
-         * recent time this cpuset changed its mems_allowed.
-         */
-        int mems_generation;
-
         struct fmeter fmeter;           /* memory_pressure filter */
 
         /* partition number for rebuild_sched_domains() */
@@ -176,27 +170,6 @@ static inline int is_spread_slab(const struct cpuset *cs)
         return test_bit(CS_SPREAD_SLAB, &cs->flags);
 }
 
-/*
- * Increment this integer everytime any cpuset changes its
- * mems_allowed value.  Users of cpusets can track this generation
- * number, and avoid having to lock and reload mems_allowed unless
- * the cpuset they're using changes generation.
- *
- * A single, global generation is needed because cpuset_attach_task() could
- * reattach a task to a different cpuset, which must not have its
- * generation numbers aliased with those of that tasks previous cpuset.
- *
- * Generations are needed for mems_allowed because one task cannot
- * modify another's memory placement.  So we must enable every task,
- * on every visit to __alloc_pages(), to efficiently check whether
- * its current->cpuset->mems_allowed has changed, requiring an update
- * of its current->mems_allowed.
- *
- * Since writes to cpuset_mems_generation are guarded by the cgroup lock
- * there is no need to mark it atomic.
- */
-static int cpuset_mems_generation;
-
 static struct cpuset top_cpuset = {
         .flags = ((1 << CS_CPU_EXCLUSIVE) | (1 << CS_MEM_EXCLUSIVE)),
 };
@@ -228,8 +201,9 @@ static struct cpuset top_cpuset = {
  * If a task is only holding callback_mutex, then it has read-only
  * access to cpusets.
  *
- * The task_struct fields mems_allowed and mems_generation may only
- * be accessed in the context of that task, so require no locks.
+ * Now, the task_struct fields mems_allowed and mempolicy may be changed
+ * by other task, we use alloc_lock in the task_struct fields to protect
+ * them.
  *
  * The cpuset_common_file_read() handlers only hold callback_mutex across
  * small pieces of code, such as when reading out possibly multi-word
@@ -331,75 +305,22 @@ static void guarantee_online_mems(const struct cpuset *cs, nodemask_t *pmask)
         BUG_ON(!nodes_intersects(*pmask, node_states[N_HIGH_MEMORY]));
 }
 
-/**
- * cpuset_update_task_memory_state - update task memory placement
- *
- * If the current tasks cpusets mems_allowed changed behind our
- * backs, update current->mems_allowed, mems_generation and task NUMA
- * mempolicy to the new value.
- *
- * Task mempolicy is updated by rebinding it relative to the
- * current->cpuset if a task has its memory placement changed.
- * Do not call this routine if in_interrupt().
- *
- * Call without callback_mutex or task_lock() held.  May be
- * called with or without cgroup_mutex held.  Thanks in part to
- * 'the_top_cpuset_hack', the task's cpuset pointer will never
- * be NULL.  This routine also might acquire callback_mutex during
- * call.
- *
- * Reading current->cpuset->mems_generation doesn't need task_lock
- * to guard the current->cpuset derefence, because it is guarded
- * from concurrent freeing of current->cpuset using RCU.
- *
- * The rcu_dereference() is technically probably not needed,
- * as I don't actually mind if I see a new cpuset pointer but
- * an old value of mems_generation.  However this really only
- * matters on alpha systems using cpusets heavily.  If I dropped
- * that rcu_dereference(), it would save them a memory barrier.
- * For all other arch's, rcu_dereference is a no-op anyway, and for
- * alpha systems not using cpusets, another planned optimization,
- * avoiding the rcu critical section for tasks in the root cpuset
- * which is statically allocated, so can't vanish, will make this
- * irrelevant.  Better to use RCU as intended, than to engage in
- * some cute trick to save a memory barrier that is impossible to
- * test, for alpha systems using cpusets heavily, which might not
- * even exist.
- *
- * This routine is needed to update the per-task mems_allowed data,
- * within the tasks context, when it is trying to allocate memory
- * (in various mm/mempolicy.c routines) and notices that some other
- * task has been modifying its cpuset.
+/*
+ * update task's spread flag if cpuset's page/slab spread flag is set
+ *
+ * Called with callback_mutex/cgroup_mutex held
  */
-
-void cpuset_update_task_memory_state(void)
+static void cpuset_update_task_spread_flag(struct cpuset *cs,
+                                        struct task_struct *tsk)
 {
-        int my_cpusets_mem_gen;
-        struct task_struct *tsk = current;
-        struct cpuset *cs;
-
-        rcu_read_lock();
-        my_cpusets_mem_gen = task_cs(tsk)->mems_generation;
-        rcu_read_unlock();
-
-        if (my_cpusets_mem_gen != tsk->cpuset_mems_generation) {
-                mutex_lock(&callback_mutex);
-                task_lock(tsk);
-                cs = task_cs(tsk); /* Maybe changed when task not locked */
-                guarantee_online_mems(cs, &tsk->mems_allowed);
-                tsk->cpuset_mems_generation = cs->mems_generation;
-                if (is_spread_page(cs))
-                        tsk->flags |= PF_SPREAD_PAGE;
-                else
-                        tsk->flags &= ~PF_SPREAD_PAGE;
-                if (is_spread_slab(cs))
-                        tsk->flags |= PF_SPREAD_SLAB;
-                else
-                        tsk->flags &= ~PF_SPREAD_SLAB;
-                task_unlock(tsk);
-                mutex_unlock(&callback_mutex);
-                mpol_rebind_task(tsk, &tsk->mems_allowed);
-        }
+        if (is_spread_page(cs))
+                tsk->flags |= PF_SPREAD_PAGE;
+        else
+                tsk->flags &= ~PF_SPREAD_PAGE;
+        if (is_spread_slab(cs))
+                tsk->flags |= PF_SPREAD_SLAB;
+        else
+                tsk->flags &= ~PF_SPREAD_SLAB;
 }
 
 /*
@@ -1007,14 +928,6 @@ static int update_cpumask(struct cpuset *cs, struct cpuset *trialcs,
  *    other task, the task_struct mems_allowed that we are hacking
  *    is for our current task, which must allocate new pages for that
  *    migrating memory region.
- *
- *    We call cpuset_update_task_memory_state() before hacking
- *    our tasks mems_allowed, so that we are assured of being in
- *    sync with our tasks cpuset, and in particular, callbacks to
- *    cpuset_update_task_memory_state() from nested page allocations
- *    won't see any mismatch of our cpuset and task mems_generation
- *    values, so won't overwrite our hacked tasks mems_allowed
- *    nodemask.
  */
 
 static void cpuset_migrate_mm(struct mm_struct *mm, const nodemask_t *from,
@@ -1022,22 +935,37 @@ static void cpuset_migrate_mm(struct mm_struct *mm, const nodemask_t *from,
 {
         struct task_struct *tsk = current;
 
-        cpuset_update_task_memory_state();
-
-        mutex_lock(&callback_mutex);
         tsk->mems_allowed = *to;
-        mutex_unlock(&callback_mutex);
 
         do_migrate_pages(mm, from, to, MPOL_MF_MOVE_ALL);
 
-        mutex_lock(&callback_mutex);
         guarantee_online_mems(task_cs(tsk),&tsk->mems_allowed);
-        mutex_unlock(&callback_mutex);
 }
 
 /*
- * Rebind task's vmas to cpuset's new mems_allowed, and migrate pages to new
- * nodes if memory_migrate flag is set. Called with cgroup_mutex held.
+ * cpuset_change_task_nodemask - change task's mems_allowed and mempolicy
+ * @tsk: the task to change
+ * @newmems: new nodes that the task will be set
+ *
+ * In order to avoid seeing no nodes if the old and new nodes are disjoint,
+ * we structure updates as setting all new allowed nodes, then clearing newly
+ * disallowed ones.
+ *
+ * Called with task's alloc_lock held
+ */
+static void cpuset_change_task_nodemask(struct task_struct *tsk,
+                                        nodemask_t *newmems)
+{
+        nodes_or(tsk->mems_allowed, tsk->mems_allowed, *newmems);
+        mpol_rebind_task(tsk, &tsk->mems_allowed);
+        mpol_rebind_task(tsk, newmems);
+        tsk->mems_allowed = *newmems;
+}
+
+/*
+ * Update task's mems_allowed and rebind its mempolicy and vmas' mempolicy
+ * of it to cpuset's new mems_allowed, and migrate pages to new nodes if
+ * memory_migrate flag is set. Called with cgroup_mutex held.
  */
 static void cpuset_change_nodemask(struct task_struct *p,
                                    struct cgroup_scanner *scan)
@@ -1046,12 +974,19 @@ static void cpuset_change_nodemask(struct task_struct *p,
         struct cpuset *cs;
         int migrate;
         const nodemask_t *oldmem = scan->data;
+        nodemask_t newmems;
+
+        cs = cgroup_cs(scan->cg);
+        guarantee_online_mems(cs, &newmems);
+
+        task_lock(p);
+        cpuset_change_task_nodemask(p, &newmems);
+        task_unlock(p);
 
         mm = get_task_mm(p);
         if (!mm)
                 return;
 
-        cs = cgroup_cs(scan->cg);
         migrate = is_memory_migrate(cs);
 
         mpol_rebind_mm(mm, &cs->mems_allowed);
@@ -1104,10 +1039,10 @@ static void update_tasks_nodemask(struct cpuset *cs, const nodemask_t *oldmem,
 /*
  * Handle user request to change the 'mems' memory placement
  * of a cpuset.  Needs to validate the request, update the
- * cpusets mems_allowed and mems_generation, and for each
- * task in the cpuset, rebind any vma mempolicies and if
- * the cpuset is marked 'memory_migrate', migrate the tasks
- * pages to the new memory.
+ * cpusets mems_allowed, and for each task in the cpuset,
+ * update mems_allowed and rebind task's mempolicy and any vma
+ * mempolicies and if the cpuset is marked 'memory_migrate',
+ * migrate the tasks pages to the new memory.
  *
  * Call with cgroup_mutex held.  May take callback_mutex during call.
  * Will take tasklist_lock, scan tasklist for tasks in cpuset cs,
@@ -1160,7 +1095,6 @@ static int update_nodemask(struct cpuset *cs, struct cpuset *trialcs,
 
         mutex_lock(&callback_mutex);
         cs->mems_allowed = trialcs->mems_allowed;
-        cs->mems_generation = cpuset_mems_generation++;
         mutex_unlock(&callback_mutex);
 
         update_tasks_nodemask(cs, &oldmem, &heap);
@@ -1193,6 +1127,46 @@ static int update_relax_domain_level(struct cpuset *cs, s64 val)
 }
 
 /*
+ * cpuset_change_flag - make a task's spread flags the same as its cpuset's
+ * @tsk: task to be updated
+ * @scan: struct cgroup_scanner containing the cgroup of the task
+ *
+ * Called by cgroup_scan_tasks() for each task in a cgroup.
+ *
+ * We don't need to re-check for the cgroup/cpuset membership, since we're
+ * holding cgroup_lock() at this point.
+ */
+static void cpuset_change_flag(struct task_struct *tsk,
+                                struct cgroup_scanner *scan)
+{
+        cpuset_update_task_spread_flag(cgroup_cs(scan->cg), tsk);
+}
+
+/*
+ * update_tasks_flags - update the spread flags of tasks in the cpuset.
+ * @cs: the cpuset in which each task's spread flags needs to be changed
+ * @heap: if NULL, defer allocating heap memory to cgroup_scan_tasks()
+ *
+ * Called with cgroup_mutex held
+ *
+ * The cgroup_scan_tasks() function will scan all the tasks in a cgroup,
+ * calling callback functions for each.
+ *
+ * No return value. It's guaranteed that cgroup_scan_tasks() always returns 0
+ * if @heap != NULL.
+ */
+static void update_tasks_flags(struct cpuset *cs, struct ptr_heap *heap)
+{
+        struct cgroup_scanner scan;
+
+        scan.cg = cs->css.cgroup;
+        scan.test_task = NULL;
+        scan.process_task = cpuset_change_flag;
+        scan.heap = heap;
+        cgroup_scan_tasks(&scan);
+}
+
+/*
  * update_flag - read a 0 or a 1 in a file and update associated flag
  * bit:         the bit to update (see cpuset_flagbits_t)
  * cs:          the cpuset to update
@@ -1205,8 +1179,10 @@ static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs,
                        int turning_on)
 {
         struct cpuset *trialcs;
-        int err;
         int balance_flag_changed;
+        int spread_flag_changed;
+        struct ptr_heap heap;
+        int err;
 
         trialcs = alloc_trial_cpuset(cs);
         if (!trialcs)
@@ -1221,9 +1197,16 @@ static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs,
         if (err < 0)
                 goto out;
 
+        err = heap_init(&heap, PAGE_SIZE, GFP_KERNEL, NULL);
+        if (err < 0)
+                goto out;
+
         balance_flag_changed = (is_sched_load_balance(cs) !=
                                 is_sched_load_balance(trialcs));
 
+        spread_flag_changed = ((is_spread_slab(cs) != is_spread_slab(trialcs))
+                        || (is_spread_page(cs) != is_spread_page(trialcs)));
+
         mutex_lock(&callback_mutex);
         cs->flags = trialcs->flags;
         mutex_unlock(&callback_mutex);
@@ -1231,6 +1214,9 @@ static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs,
         if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed)
                 async_rebuild_sched_domains();
 
+        if (spread_flag_changed)
+                update_tasks_flags(cs, &heap);
+        heap_free(&heap);
 out:
         free_trial_cpuset(trialcs);
         return err;
@@ -1372,15 +1358,20 @@ static void cpuset_attach(struct cgroup_subsys *ss,
 
         if (cs == &top_cpuset) {
                 cpumask_copy(cpus_attach, cpu_possible_mask);
+                to = node_possible_map;
         } else {
-                mutex_lock(&callback_mutex);
                 guarantee_online_cpus(cs, cpus_attach);
-                mutex_unlock(&callback_mutex);
+                guarantee_online_mems(cs, &to);
         }
         err = set_cpus_allowed_ptr(tsk, cpus_attach);
         if (err)
                 return;
 
+        task_lock(tsk);
+        cpuset_change_task_nodemask(tsk, &to);
+        task_unlock(tsk);
+        cpuset_update_task_spread_flag(cs, tsk);
+
         from = oldcs->mems_allowed;
         to = cs->mems_allowed;
         mm = get_task_mm(tsk);
@@ -1442,11 +1433,9 @@ static int cpuset_write_u64(struct cgroup *cgrp, struct cftype *cft, u64 val)
                 break;
         case FILE_SPREAD_PAGE:
                 retval = update_flag(CS_SPREAD_PAGE, cs, val);
-                cs->mems_generation = cpuset_mems_generation++;
                 break;
         case FILE_SPREAD_SLAB:
                 retval = update_flag(CS_SPREAD_SLAB, cs, val);
-                cs->mems_generation = cpuset_mems_generation++;
                 break;
         default:
                 retval = -EINVAL;
@@ -1786,8 +1775,6 @@ static struct cgroup_subsys_state *cpuset_create(
         struct cpuset *parent;
 
         if (!cont->parent) {
-                /* This is early initialization for the top cgroup */
-                top_cpuset.mems_generation = cpuset_mems_generation++;
                 return &top_cpuset.css;
         }
         parent = cgroup_cs(cont->parent);
@@ -1799,7 +1786,6 @@ static struct cgroup_subsys_state *cpuset_create(
                 return ERR_PTR(-ENOMEM);
         }
 
-        cpuset_update_task_memory_state();
         cs->flags = 0;
         if (is_spread_page(parent))
                 set_bit(CS_SPREAD_PAGE, &cs->flags);
@@ -1808,7 +1794,6 @@ static struct cgroup_subsys_state *cpuset_create(
         set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
         cpumask_clear(cs->cpus_allowed);
         nodes_clear(cs->mems_allowed);
-        cs->mems_generation = cpuset_mems_generation++;
         fmeter_init(&cs->fmeter);
         cs->relax_domain_level = -1;
 
@@ -1827,8 +1812,6 @@ static void cpuset_destroy(struct cgroup_subsys *ss, struct cgroup *cont)
 {
         struct cpuset *cs = cgroup_cs(cont);
 
-        cpuset_update_task_memory_state();
-
         if (is_sched_load_balance(cs))
                 update_flag(CS_SCHED_LOAD_BALANCE, cs, 0);
 
@@ -1849,21 +1832,6 @@ struct cgroup_subsys cpuset_subsys = {
         .early_init = 1,
 };
 
-/*
- * cpuset_init_early - just enough so that the calls to
- * cpuset_update_task_memory_state() in early init code
- * are harmless.
- */
-
-int __init cpuset_init_early(void)
-{
-        alloc_cpumask_var(&top_cpuset.cpus_allowed, GFP_NOWAIT);
-
-        top_cpuset.mems_generation = cpuset_mems_generation++;
-        return 0;
-}
-
-
 /**
  * cpuset_init - initialize cpusets at system boot
  *
@@ -1874,11 +1842,13 @@ int __init cpuset_init(void)
 {
         int err = 0;
 
+        if (!alloc_cpumask_var(&top_cpuset.cpus_allowed, GFP_KERNEL))
+                BUG();
+
         cpumask_setall(top_cpuset.cpus_allowed);
         nodes_setall(top_cpuset.mems_allowed);
 
         fmeter_init(&top_cpuset.fmeter);
-        top_cpuset.mems_generation = cpuset_mems_generation++;
         set_bit(CS_SCHED_LOAD_BALANCE, &top_cpuset.flags);
         top_cpuset.relax_domain_level = -1;
 
diff --git a/kernel/cred.c b/kernel/cred.c
index 1bb4d7e5d616..d7f7a01082eb 100644
--- a/kernel/cred.c
+++ b/kernel/cred.c
@@ -18,6 +18,18 @@
 #include <linux/cn_proc.h>
 #include "cred-internals.h"
 
+#if 0
+#define kdebug(FMT, ...) \
+        printk("[%-5.5s%5u] "FMT"\n", current->comm, current->pid ,##__VA_ARGS__)
+#else
+static inline __attribute__((format(printf, 1, 2)))
+void no_printk(const char *fmt, ...)
+{
+}
+#define kdebug(FMT, ...) \
+        no_printk("[%-5.5s%5u] "FMT"\n", current->comm, current->pid ,##__VA_ARGS__)
+#endif
+
 static struct kmem_cache *cred_jar;
 
 /*
@@ -36,6 +48,10 @@ static struct thread_group_cred init_tgcred = {
  */
 struct cred init_cred = {
         .usage                  = ATOMIC_INIT(4),
+#ifdef CONFIG_DEBUG_CREDENTIALS
+        .subscribers            = ATOMIC_INIT(2),
+        .magic                  = CRED_MAGIC,
+#endif
         .securebits             = SECUREBITS_DEFAULT,
         .cap_inheritable        = CAP_INIT_INH_SET,
         .cap_permitted          = CAP_FULL_SET,
@@ -48,6 +64,31 @@ struct cred init_cred = {
 #endif
 };
 
+static inline void set_cred_subscribers(struct cred *cred, int n)
+{
+#ifdef CONFIG_DEBUG_CREDENTIALS
+        atomic_set(&cred->subscribers, n);
+#endif
+}
+
+static inline int read_cred_subscribers(const struct cred *cred)
+{
+#ifdef CONFIG_DEBUG_CREDENTIALS
+        return atomic_read(&cred->subscribers);
+#else
+        return 0;
+#endif
+}
+
+static inline void alter_cred_subscribers(const struct cred *_cred, int n)
+{
+#ifdef CONFIG_DEBUG_CREDENTIALS
+        struct cred *cred = (struct cred *) _cred;
+
+        atomic_add(n, &cred->subscribers);
+#endif
+}
+
 /*
  * Dispose of the shared task group credentials
  */
@@ -85,15 +126,29 @@ static void put_cred_rcu(struct rcu_head *rcu)
 {
         struct cred *cred = container_of(rcu, struct cred, rcu);
 
+        kdebug("put_cred_rcu(%p)", cred);
+
+#ifdef CONFIG_DEBUG_CREDENTIALS
+        if (cred->magic != CRED_MAGIC_DEAD ||
+            atomic_read(&cred->usage) != 0 ||
+            read_cred_subscribers(cred) != 0)
+                panic("CRED: put_cred_rcu() sees %p with"
+                      " mag %x, put %p, usage %d, subscr %d\n",
+                      cred, cred->magic, cred->put_addr,
+                      atomic_read(&cred->usage),
+                      read_cred_subscribers(cred));
+#else
         if (atomic_read(&cred->usage) != 0)
                 panic("CRED: put_cred_rcu() sees %p with usage %d\n",
                       cred, atomic_read(&cred->usage));
+#endif
 
         security_cred_free(cred);
         key_put(cred->thread_keyring);
         key_put(cred->request_key_auth);
         release_tgcred(cred);
-        put_group_info(cred->group_info);
+        if (cred->group_info)
+                put_group_info(cred->group_info);
         free_uid(cred->user);
         kmem_cache_free(cred_jar, cred);
 }
@@ -106,12 +161,90 @@ static void put_cred_rcu(struct rcu_head *rcu)
  */
 void __put_cred(struct cred *cred)
 {
+        kdebug("__put_cred(%p{%d,%d})", cred,
+               atomic_read(&cred->usage),
+               read_cred_subscribers(cred));
+
         BUG_ON(atomic_read(&cred->usage) != 0);
+#ifdef CONFIG_DEBUG_CREDENTIALS
+        BUG_ON(read_cred_subscribers(cred) != 0);
+        cred->magic = CRED_MAGIC_DEAD;
+        cred->put_addr = __builtin_return_address(0);
+#endif
+        BUG_ON(cred == current->cred);
+        BUG_ON(cred == current->real_cred);
 
         call_rcu(&cred->rcu, put_cred_rcu);
 }
 EXPORT_SYMBOL(__put_cred);
 
+/*
+ * Clean up a task's credentials when it exits
+ */
+void exit_creds(struct task_struct *tsk)
+{
+        struct cred *cred;
+
+        kdebug("exit_creds(%u,%p,%p,{%d,%d})", tsk->pid, tsk->real_cred, tsk->cred,
+               atomic_read(&tsk->cred->usage),
+               read_cred_subscribers(tsk->cred));
+
+        cred = (struct cred *) tsk->real_cred;
+        tsk->real_cred = NULL;
+        validate_creds(cred);
+        alter_cred_subscribers(cred, -1);
+        put_cred(cred);
+
+        cred = (struct cred *) tsk->cred;
+        tsk->cred = NULL;
+        validate_creds(cred);
+        alter_cred_subscribers(cred, -1);
+        put_cred(cred);
+
+        cred = (struct cred *) tsk->replacement_session_keyring;
+        if (cred) {
+                tsk->replacement_session_keyring = NULL;
+                validate_creds(cred);
+                put_cred(cred);
+        }
+}
+
+/*
+ * Allocate blank credentials, such that the credentials can be filled in at a
+ * later date without risk of ENOMEM.
+ */
+struct cred *cred_alloc_blank(void)
+{
+        struct cred *new;
+
+        new = kmem_cache_zalloc(cred_jar, GFP_KERNEL);
+        if (!new)
+                return NULL;
+
+#ifdef CONFIG_KEYS
+        new->tgcred = kzalloc(sizeof(*new->tgcred), GFP_KERNEL);
+        if (!new->tgcred) {
+                kfree(new);
+                return NULL;
+        }
+        atomic_set(&new->tgcred->usage, 1);
+#endif
+
+        atomic_set(&new->usage, 1);
+
+        if (security_cred_alloc_blank(new, GFP_KERNEL) < 0)
+                goto error;
+
+#ifdef CONFIG_DEBUG_CREDENTIALS
+        new->magic = CRED_MAGIC;
+#endif
+        return new;
+
+error:
+        abort_creds(new);
+        return NULL;
+}
+
 /**
  * prepare_creds - Prepare a new set of credentials for modification
  *
@@ -132,16 +265,19 @@ struct cred *prepare_creds(void)
         const struct cred *old;
         struct cred *new;
 
-        BUG_ON(atomic_read(&task->real_cred->usage) < 1);
+        validate_process_creds();
 
         new = kmem_cache_alloc(cred_jar, GFP_KERNEL);
         if (!new)
                 return NULL;
 
+        kdebug("prepare_creds() alloc %p", new);
+
         old = task->cred;
         memcpy(new, old, sizeof(struct cred));
 
         atomic_set(&new->usage, 1);
+        set_cred_subscribers(new, 0);
         get_group_info(new->group_info);
         get_uid(new->user);
 
@@ -157,6 +293,7 @@ struct cred *prepare_creds(void)
 
         if (security_prepare_creds(new, old, GFP_KERNEL) < 0)
                 goto error;
+        validate_creds(new);
         return new;
 
 error:
@@ -229,9 +366,12 @@ struct cred *prepare_usermodehelper_creds(void)
         if (!new)
                 return NULL;
 
+        kdebug("prepare_usermodehelper_creds() alloc %p", new);
+
         memcpy(new, &init_cred, sizeof(struct cred));
 
         atomic_set(&new->usage, 1);
+        set_cred_subscribers(new, 0);
         get_group_info(new->group_info);
         get_uid(new->user);
 
@@ -250,6 +390,7 @@ struct cred *prepare_usermodehelper_creds(void)
 #endif
         if (security_prepare_creds(new, &init_cred, GFP_ATOMIC) < 0)
                 goto error;
+        validate_creds(new);
 
         BUG_ON(atomic_read(&new->usage) != 1);
         return new;
@@ -286,6 +427,10 @@ int copy_creds(struct task_struct *p, unsigned long clone_flags)
             ) {
                 p->real_cred = get_cred(p->cred);
                 get_cred(p->cred);
+                alter_cred_subscribers(p->cred, 2);
+                kdebug("share_creds(%p{%d,%d})",
+                       p->cred, atomic_read(&p->cred->usage),
+                       read_cred_subscribers(p->cred));
                 atomic_inc(&p->cred->user->processes);
                 return 0;
         }
@@ -331,6 +476,8 @@ int copy_creds(struct task_struct *p, unsigned long clone_flags)
 
         atomic_inc(&new->user->processes);
         p->cred = p->real_cred = get_cred(new);
+        alter_cred_subscribers(new, 2);
+        validate_creds(new);
         return 0;
 
 error_put:
@@ -355,13 +502,20 @@ error_put:
 int commit_creds(struct cred *new)
 {
         struct task_struct *task = current;
-        const struct cred *old;
+        const struct cred *old = task->real_cred;
 
-        BUG_ON(task->cred != task->real_cred);
-        BUG_ON(atomic_read(&task->real_cred->usage) < 2);
+        kdebug("commit_creds(%p{%d,%d})", new,
+               atomic_read(&new->usage),
+               read_cred_subscribers(new));
+
+        BUG_ON(task->cred != old);
+#ifdef CONFIG_DEBUG_CREDENTIALS
+        BUG_ON(read_cred_subscribers(old) < 2);
+        validate_creds(old);
+        validate_creds(new);
+#endif
         BUG_ON(atomic_read(&new->usage) < 1);
 
-        old = task->real_cred;
         security_commit_creds(new, old);
 
         get_cred(new); /* we will require a ref for the subj creds too */
@@ -390,12 +544,14 @@ int commit_creds(struct cred *new)
          *   cheaply with the new uid cache, so if it matters
          *   we should be checking for it.  -DaveM
          */
+        alter_cred_subscribers(new, 2);
         if (new->user != old->user)
                 atomic_inc(&new->user->processes);
         rcu_assign_pointer(task->real_cred, new);
         rcu_assign_pointer(task->cred, new);
         if (new->user != old->user)
                 atomic_dec(&old->user->processes);
+        alter_cred_subscribers(old, -2);
 
         sched_switch_user(task);
 
@@ -428,6 +584,13 @@ EXPORT_SYMBOL(commit_creds);
  */
 void abort_creds(struct cred *new)
 {
+        kdebug("abort_creds(%p{%d,%d})", new,
+               atomic_read(&new->usage),
+               read_cred_subscribers(new));
+
+#ifdef CONFIG_DEBUG_CREDENTIALS
+        BUG_ON(read_cred_subscribers(new) != 0);
+#endif
         BUG_ON(atomic_read(&new->usage) < 1);
         put_cred(new);
 }
@@ -444,7 +607,20 @@ const struct cred *override_creds(const struct cred *new)
 {
         const struct cred *old = current->cred;
 
-        rcu_assign_pointer(current->cred, get_cred(new));
+        kdebug("override_creds(%p{%d,%d})", new,
+               atomic_read(&new->usage),
+               read_cred_subscribers(new));
+
+        validate_creds(old);
+        validate_creds(new);
+        get_cred(new);
+        alter_cred_subscribers(new, 1);
+        rcu_assign_pointer(current->cred, new);
+        alter_cred_subscribers(old, -1);
+
+        kdebug("override_creds() = %p{%d,%d}", old,
+               atomic_read(&old->usage),
+               read_cred_subscribers(old));
         return old;
 }
 EXPORT_SYMBOL(override_creds);
@@ -460,7 +636,15 @@ void revert_creds(const struct cred *old)
 {
         const struct cred *override = current->cred;
 
+        kdebug("revert_creds(%p{%d,%d})", old,
+               atomic_read(&old->usage),
+               read_cred_subscribers(old));
+
+        validate_creds(old);
+        validate_creds(override);
+        alter_cred_subscribers(old, 1);
         rcu_assign_pointer(current->cred, old);
+        alter_cred_subscribers(override, -1);
         put_cred(override);
 }
 EXPORT_SYMBOL(revert_creds);
@@ -502,11 +686,15 @@ struct cred *prepare_kernel_cred(struct task_struct *daemon)
         if (!new)
                 return NULL;
 
+        kdebug("prepare_kernel_cred() alloc %p", new);
+
         if (daemon)
                 old = get_task_cred(daemon);
         else
                 old = get_cred(&init_cred);
 
+        validate_creds(old);
+
         *new = *old;
         get_uid(new->user);
         get_group_info(new->group_info);
@@ -526,7 +714,9 @@ struct cred *prepare_kernel_cred(struct task_struct *daemon)
                 goto error;
 
         atomic_set(&new->usage, 1);
+        set_cred_subscribers(new, 0);
         put_cred(old);
+        validate_creds(new);
         return new;
 
 error:
@@ -589,3 +779,95 @@ int set_create_files_as(struct cred *new, struct inode *inode)
         return security_kernel_create_files_as(new, inode);
 }
 EXPORT_SYMBOL(set_create_files_as);
+
+#ifdef CONFIG_DEBUG_CREDENTIALS
+
+/*
+ * dump invalid credentials
+ */
+static void dump_invalid_creds(const struct cred *cred, const char *label,
+                               const struct task_struct *tsk)
+{
+        printk(KERN_ERR "CRED: %s credentials: %p %s%s%s\n",
+               label, cred,
+               cred == &init_cred ? "[init]" : "",
+               cred == tsk->real_cred ? "[real]" : "",
+               cred == tsk->cred ? "[eff]" : "");
+        printk(KERN_ERR "CRED: ->magic=%x, put_addr=%p\n",
+               cred->magic, cred->put_addr);
+        printk(KERN_ERR "CRED: ->usage=%d, subscr=%d\n",
+               atomic_read(&cred->usage),
+               read_cred_subscribers(cred));
+        printk(KERN_ERR "CRED: ->*uid = { %d,%d,%d,%d }\n",
+               cred->uid, cred->euid, cred->suid, cred->fsuid);
+        printk(KERN_ERR "CRED: ->*gid = { %d,%d,%d,%d }\n",
+               cred->gid, cred->egid, cred->sgid, cred->fsgid);
+#ifdef CONFIG_SECURITY
+        printk(KERN_ERR "CRED: ->security is %p\n", cred->security);
+        if ((unsigned long) cred->security >= PAGE_SIZE &&
+            (((unsigned long) cred->security & 0xffffff00) !=
+             (POISON_FREE << 24 | POISON_FREE << 16 | POISON_FREE << 8)))
+                printk(KERN_ERR "CRED: ->security {%x, %x}\n",
+                       ((u32*)cred->security)[0],
+                       ((u32*)cred->security)[1]);
+#endif
+}
+
+/*
+ * report use of invalid credentials
+ */
+void __invalid_creds(const struct cred *cred, const char *file, unsigned line)
+{
+        printk(KERN_ERR "CRED: Invalid credentials\n");
+        printk(KERN_ERR "CRED: At %s:%u\n", file, line);
+        dump_invalid_creds(cred, "Specified", current);
+        BUG();
+}
+EXPORT_SYMBOL(__invalid_creds);
+
+/*
+ * check the credentials on a process
+ */
+void __validate_process_creds(struct task_struct *tsk,
+                              const char *file, unsigned line)
+{
+        if (tsk->cred == tsk->real_cred) {
+                if (unlikely(read_cred_subscribers(tsk->cred) < 2 ||
+                             creds_are_invalid(tsk->cred)))
+                        goto invalid_creds;
+        } else {
+                if (unlikely(read_cred_subscribers(tsk->real_cred) < 1 ||
+                             read_cred_subscribers(tsk->cred) < 1 ||
+                             creds_are_invalid(tsk->real_cred) ||
+                             creds_are_invalid(tsk->cred)))
+                        goto invalid_creds;
+        }
+        return;
+
+invalid_creds:
+        printk(KERN_ERR "CRED: Invalid process credentials\n");
+        printk(KERN_ERR "CRED: At %s:%u\n", file, line);
+
+        dump_invalid_creds(tsk->real_cred, "Real", tsk);
+        if (tsk->cred != tsk->real_cred)
+                dump_invalid_creds(tsk->cred, "Effective", tsk);
+        else
+                printk(KERN_ERR "CRED: Effective creds == Real creds\n");
+        BUG();
+}
+EXPORT_SYMBOL(__validate_process_creds);
+
+/*
+ * check creds for do_exit()
+ */
+void validate_creds_for_do_exit(struct task_struct *tsk)
+{
+        kdebug("validate_creds_for_do_exit(%p,%p{%d,%d})",
+               tsk->real_cred, tsk->cred,
+               atomic_read(&tsk->cred->usage),
+               read_cred_subscribers(tsk->cred));
+
+        __validate_process_creds(tsk, __FILE__, __LINE__);
+}
+
+#endif /* CONFIG_DEBUG_CREDENTIALS */
diff --git a/kernel/dma-coherent.c b/kernel/dma-coherent.c
deleted file mode 100644
index 962a3b574f21..000000000000
--- a/kernel/dma-coherent.c
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Coherent per-device memory handling.
- * Borrowed from i386
- */
-#include <linux/kernel.h>
-#include <linux/dma-mapping.h>
-
-struct dma_coherent_mem {
-        void            *virt_base;
-        u32             device_base;
-        int             size;
-        int             flags;
-        unsigned long   *bitmap;
-};
-
-int dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr,
-                                dma_addr_t device_addr, size_t size, int flags)
-{
-        void __iomem *mem_base = NULL;
-        int pages = size >> PAGE_SHIFT;
-        int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long);
-
-        if ((flags & (DMA_MEMORY_MAP | DMA_MEMORY_IO)) == 0)
-                goto out;
-        if (!size)
-                goto out;
-        if (dev->dma_mem)
-                goto out;
-
-        /* FIXME: this routine just ignores DMA_MEMORY_INCLUDES_CHILDREN */
-
-        mem_base = ioremap(bus_addr, size);
-        if (!mem_base)
-                goto out;
-
-        dev->dma_mem = kzalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL);
-        if (!dev->dma_mem)
-                goto out;
-        dev->dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
-        if (!dev->dma_mem->bitmap)
-                goto free1_out;
-
-        dev->dma_mem->virt_base = mem_base;
-        dev->dma_mem->device_base = device_addr;
-        dev->dma_mem->size = pages;
-        dev->dma_mem->flags = flags;
-
-        if (flags & DMA_MEMORY_MAP)
-                return DMA_MEMORY_MAP;
-
-        return DMA_MEMORY_IO;
-
- free1_out:
-        kfree(dev->dma_mem);
- out:
-        if (mem_base)
-                iounmap(mem_base);
-        return 0;
-}
-EXPORT_SYMBOL(dma_declare_coherent_memory);
-
-void dma_release_declared_memory(struct device *dev)
-{
-        struct dma_coherent_mem *mem = dev->dma_mem;
-
-        if (!mem)
-                return;
-        dev->dma_mem = NULL;
-        iounmap(mem->virt_base);
-        kfree(mem->bitmap);
-        kfree(mem);
-}
-EXPORT_SYMBOL(dma_release_declared_memory);
-
-void *dma_mark_declared_memory_occupied(struct device *dev,
-                                        dma_addr_t device_addr, size_t size)
-{
-        struct dma_coherent_mem *mem = dev->dma_mem;
-        int pos, err;
-
-        size += device_addr & ~PAGE_MASK;
-
-        if (!mem)
-                return ERR_PTR(-EINVAL);
-
-        pos = (device_addr - mem->device_base) >> PAGE_SHIFT;
-        err = bitmap_allocate_region(mem->bitmap, pos, get_order(size));
-        if (err != 0)
-                return ERR_PTR(err);
-        return mem->virt_base + (pos << PAGE_SHIFT);
-}
-EXPORT_SYMBOL(dma_mark_declared_memory_occupied);
-
-/**
- * dma_alloc_from_coherent() - try to allocate memory from the per-device coherent area
- *
- * @dev:        device from which we allocate memory
- * @size:       size of requested memory area
- * @dma_handle: This will be filled with the correct dma handle
- * @ret:        This pointer will be filled with the virtual address
- *              to allocated area.
- *
- * This function should be only called from per-arch dma_alloc_coherent()
- * to support allocation from per-device coherent memory pools.
- *
- * Returns 0 if dma_alloc_coherent should continue with allocating from
- * generic memory areas, or !0 if dma_alloc_coherent should return @ret.
- */
-int dma_alloc_from_coherent(struct device *dev, ssize_t size,
-                                       dma_addr_t *dma_handle, void **ret)
-{
-        struct dma_coherent_mem *mem;
-        int order = get_order(size);
-        int pageno;
-
-        if (!dev)
-                return 0;
-        mem = dev->dma_mem;
-        if (!mem)
-                return 0;
-
-        *ret = NULL;
-
-        if (unlikely(size > (mem->size << PAGE_SHIFT)))
-                goto err;
-
-        pageno = bitmap_find_free_region(mem->bitmap, mem->size, order);
-        if (unlikely(pageno < 0))
-                goto err;
-
-        /*
-         * Memory was found in the per-device area.
-         */
-        *dma_handle = mem->device_base + (pageno << PAGE_SHIFT);
-        *ret = mem->virt_base + (pageno << PAGE_SHIFT);
-        memset(*ret, 0, size);
-
-        return 1;
-
-err:
-        /*
-         * In the case where the allocation can not be satisfied from the
-         * per-device area, try to fall back to generic memory if the
-         * constraints allow it.
-         */
-        return mem->flags & DMA_MEMORY_EXCLUSIVE;
-}
-EXPORT_SYMBOL(dma_alloc_from_coherent);
-
-/**
- * dma_release_from_coherent() - try to free the memory allocated from per-device coherent memory pool
- * @dev:        device from which the memory was allocated
- * @order:      the order of pages allocated
- * @vaddr:      virtual address of allocated pages
- *
- * This checks whether the memory was allocated from the per-device
- * coherent memory pool and if so, releases that memory.
- *
- * Returns 1 if we correctly released the memory, or 0 if
- * dma_release_coherent() should proceed with releasing memory from
- * generic pools.
- */
-int dma_release_from_coherent(struct device *dev, int order, void *vaddr)
-{
-        struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
-
-        if (mem && vaddr >= mem->virt_base && vaddr <
-                   (mem->virt_base + (mem->size << PAGE_SHIFT))) {
-                int page = (vaddr - mem->virt_base) >> PAGE_SHIFT;
-
-                bitmap_release_region(mem->bitmap, page, order);
-                return 1;
-        }
-        return 0;
-}
-EXPORT_SYMBOL(dma_release_from_coherent);
diff --git a/kernel/exit.c b/kernel/exit.c
index b6c90b5ef509..ae5d8660ddff 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -12,7 +12,6 @@
 #include <linux/completion.h>
 #include <linux/personality.h>
 #include <linux/tty.h>
-#include <linux/mnt_namespace.h>
 #include <linux/iocontext.h>
 #include <linux/key.h>
 #include <linux/security.h>
@@ -375,9 +374,8 @@ static void set_special_pids(struct pid *pid)
 }
 
 /*
- * Let kernel threads use this to say that they
- * allow a certain signal (since daemonize() will
- * have disabled all of them by default).
+ * Let kernel threads use this to say that they allow a certain signal.
+ * Must not be used if kthread was cloned with CLONE_SIGHAND.
  */
 int allow_signal(int sig)
 {
@@ -385,14 +383,14 @@ int allow_signal(int sig)
                 return -EINVAL;
 
         spin_lock_irq(&current->sighand->siglock);
+        /* This is only needed for daemonize()'ed kthreads */
         sigdelset(&current->blocked, sig);
-        if (!current->mm) {
-                /* Kernel threads handle their own signals.
-                   Let the signal code know it'll be handled, so
-                   that they don't get converted to SIGKILL or
-                   just silently dropped */
-                current->sighand->action[(sig)-1].sa.sa_handler = (void __user *)2;
-        }
+        /*
+         * Kernel threads handle their own signals. Let the signal code
+         * know it'll be handled, so that they don't get converted to
+         * SIGKILL or just silently dropped.
+         */
+        current->sighand->action[(sig)-1].sa.sa_handler = (void __user *)2;
         recalc_sigpending();
         spin_unlock_irq(&current->sighand->siglock);
         return 0;
@@ -591,7 +589,7 @@ retry:
         /*
          * Search in the siblings
          */
-        list_for_each_entry(c, &p->parent->children, sibling) {
+        list_for_each_entry(c, &p->real_parent->children, sibling) {
                 if (c->mm == mm)
                         goto assign_new_owner;
         }
@@ -758,7 +756,7 @@ static void reparent_thread(struct task_struct *father, struct task_struct *p,
         p->exit_signal = SIGCHLD;
 
         /* If it has exited notify the new parent about this child's death. */
-        if (!p->ptrace &&
+        if (!task_ptrace(p) &&
             p->exit_state == EXIT_ZOMBIE && thread_group_empty(p)) {
                 do_notify_parent(p, p->exit_signal);
                 if (task_detached(p)) {
@@ -783,7 +781,7 @@ static void forget_original_parent(struct task_struct *father)
         list_for_each_entry_safe(p, n, &father->children, sibling) {
                 p->real_parent = reaper;
                 if (p->parent == father) {
-                        BUG_ON(p->ptrace);
+                        BUG_ON(task_ptrace(p));
                         p->parent = p->real_parent;
                 }
                 reparent_thread(father, p, &dead_children);
@@ -903,6 +901,8 @@ NORET_TYPE void do_exit(long code)
 
         tracehook_report_exit(&code);
 
+        validate_creds_for_do_exit(tsk);
+
         /*
          * We're taking recursive faults here in do_exit. Safest is to just
          * leave this task alone and wait for reboot.
@@ -1011,7 +1011,10 @@ NORET_TYPE void do_exit(long code)
         if (tsk->splice_pipe)
                 __free_pipe_info(tsk->splice_pipe);
 
+        validate_creds_for_do_exit(tsk);
+
         preempt_disable();
+        exit_rcu();
         /* causes final put_task_struct in finish_task_switch(). */
         tsk->state = TASK_DEAD;
         schedule();
@@ -1081,6 +1084,18 @@ SYSCALL_DEFINE1(exit_group, int, error_code)
         return 0;
 }
 
+struct wait_opts {
+        enum pid_type           wo_type;
+        int                     wo_flags;
+        struct pid              *wo_pid;
+
+        struct siginfo __user   *wo_info;
+        int __user              *wo_stat;
+        struct rusage __user    *wo_rusage;
+
+        int                     notask_error;
+};
+
 static struct pid *task_pid_type(struct task_struct *task, enum pid_type type)
 {
         struct pid *pid = NULL;
@@ -1091,13 +1106,12 @@ static struct pid *task_pid_type(struct task_struct *task, enum pid_type type)
         return pid;
 }
 
-static int eligible_child(enum pid_type type, struct pid *pid, int options,
-                          struct task_struct *p)
+static int eligible_child(struct wait_opts *wo, struct task_struct *p)
 {
         int err;
 
-        if (type < PIDTYPE_MAX) {
-                if (task_pid_type(p, type) != pid)
+        if (wo->wo_type < PIDTYPE_MAX) {
+                if (task_pid_type(p, wo->wo_type) != wo->wo_pid)
                         return 0;
         }
 
@@ -1106,8 +1120,8 @@ static int eligible_child(enum pid_type type, struct pid *pid, int options,
          * set; otherwise, wait for non-clone children *only*.  (Note:
          * A "clone" child here is one that reports to its parent
          * using a signal other than SIGCHLD.) */
-        if (((p->exit_signal != SIGCHLD) ^ ((options & __WCLONE) != 0))
-            && !(options & __WALL))
+        if (((p->exit_signal != SIGCHLD) ^ !!(wo->wo_flags & __WCLONE))
+            && !(wo->wo_flags & __WALL))
                 return 0;
 
         err = security_task_wait(p);
@@ -1117,14 +1131,15 @@ static int eligible_child(enum pid_type type, struct pid *pid, int options,
         return 1;
 }
 
-static int wait_noreap_copyout(struct task_struct *p, pid_t pid, uid_t uid,
-                               int why, int status,
-                               struct siginfo __user *infop,
-                               struct rusage __user *rusagep)
+static int wait_noreap_copyout(struct wait_opts *wo, struct task_struct *p,
+                                pid_t pid, uid_t uid, int why, int status)
 {
-        int retval = rusagep ? getrusage(p, RUSAGE_BOTH, rusagep) : 0;
+        struct siginfo __user *infop;
+        int retval = wo->wo_rusage
+                ? getrusage(p, RUSAGE_BOTH, wo->wo_rusage) : 0;
 
         put_task_struct(p);
+        infop = wo->wo_info;
         if (!retval)
                 retval = put_user(SIGCHLD, &infop->si_signo);
         if (!retval)
@@ -1148,19 +1163,18 @@ static int wait_noreap_copyout(struct task_struct *p, pid_t pid, uid_t uid,
  * the lock and this task is uninteresting.  If we return nonzero, we have
  * released the lock and the system call should return.
  */
-static int wait_task_zombie(struct task_struct *p, int options,
-                            struct siginfo __user *infop,
-                            int __user *stat_addr, struct rusage __user *ru)
+static int wait_task_zombie(struct wait_opts *wo, struct task_struct *p)
 {
         unsigned long state;
         int retval, status, traced;
         pid_t pid = task_pid_vnr(p);
         uid_t uid = __task_cred(p)->uid;
+        struct siginfo __user *infop;
 
-        if (!likely(options & WEXITED))
+        if (!likely(wo->wo_flags & WEXITED))
                 return 0;
 
-        if (unlikely(options & WNOWAIT)) {
+        if (unlikely(wo->wo_flags & WNOWAIT)) {
                 int exit_code = p->exit_code;
                 int why, status;
 
@@ -1173,8 +1187,7 @@ static int wait_task_zombie(struct task_struct *p, int options,
                         why = (exit_code & 0x80) ? CLD_DUMPED : CLD_KILLED;
                         status = exit_code & 0x7f;
                 }
-                return wait_noreap_copyout(p, pid, uid, why,
-                                           status, infop, ru);
+                return wait_noreap_copyout(wo, p, pid, uid, why, status);
         }
 
         /*
@@ -1188,11 +1201,13 @@ static int wait_task_zombie(struct task_struct *p, int options,
         }
 
         traced = ptrace_reparented(p);
-
-        if (likely(!traced)) {
+        /*
+         * It can be ptraced but not reparented, check
+         * !task_detached() to filter out sub-threads.
+         */
+        if (likely(!traced) && likely(!task_detached(p))) {
                 struct signal_struct *psig;
                 struct signal_struct *sig;
-                struct task_cputime cputime;
 
                 /*
                  * The resource counters for the group leader are in its
@@ -1205,26 +1220,23 @@ static int wait_task_zombie(struct task_struct *p, int options,
                  * p->signal fields, because they are only touched by
                  * __exit_signal, which runs with tasklist_lock
                  * write-locked anyway, and so is excluded here.  We do
-                 * need to protect the access to p->parent->signal fields,
+                 * need to protect the access to parent->signal fields,
                  * as other threads in the parent group can be right
                  * here reaping other children at the same time.
-                 *
-                 * We use thread_group_cputime() to get times for the thread
-                 * group, which consolidates times for all threads in the
-                 * group including the group leader.
                  */
-                thread_group_cputime(p, &cputime);
-                spin_lock_irq(&p->parent->sighand->siglock);
-                psig = p->parent->signal;
+                spin_lock_irq(&p->real_parent->sighand->siglock);
+                psig = p->real_parent->signal;
                 sig = p->signal;
                 psig->cutime =
                         cputime_add(psig->cutime,
-                        cputime_add(cputime.utime,
-                                    sig->cutime));
+                        cputime_add(p->utime,
+                        cputime_add(sig->utime,
+                                    sig->cutime)));
                 psig->cstime =
                         cputime_add(psig->cstime,
-                        cputime_add(cputime.stime,
-                                    sig->cstime));
+                        cputime_add(p->stime,
+                        cputime_add(sig->stime,
+                                    sig->cstime)));
                 psig->cgtime =
                         cputime_add(psig->cgtime,
                         cputime_add(p->gtime,
@@ -1246,7 +1258,7 @@ static int wait_task_zombie(struct task_struct *p, int options,
                         sig->oublock + sig->coublock;
                 task_io_accounting_add(&psig->ioac, &p->ioac);
                 task_io_accounting_add(&psig->ioac, &sig->ioac);
-                spin_unlock_irq(&p->parent->sighand->siglock);
+                spin_unlock_irq(&p->real_parent->sighand->siglock);
         }
 
         /*
@@ -1255,11 +1267,14 @@ static int wait_task_zombie(struct task_struct *p, int options,
          */
         read_unlock(&tasklist_lock);
 
-        retval = ru ? getrusage(p, RUSAGE_BOTH, ru) : 0;
+        retval = wo->wo_rusage
+                ? getrusage(p, RUSAGE_BOTH, wo->wo_rusage) : 0;
         status = (p->signal->flags & SIGNAL_GROUP_EXIT)
                 ? p->signal->group_exit_code : p->exit_code;
-        if (!retval && stat_addr)
-                retval = put_user(status, stat_addr);
+        if (!retval && wo->wo_stat)
+                retval = put_user(status, wo->wo_stat);
+
+        infop = wo->wo_info;
         if (!retval && infop)
                 retval = put_user(SIGCHLD, &infop->si_signo);
         if (!retval && infop)
@@ -1327,15 +1342,18 @@ static int *task_stopped_code(struct task_struct *p, bool ptrace)
  * the lock and this task is uninteresting.  If we return nonzero, we have
  * released the lock and the system call should return.
  */
-static int wait_task_stopped(int ptrace, struct task_struct *p,
-                             int options, struct siginfo __user *infop,
-                             int __user *stat_addr, struct rusage __user *ru)
+static int wait_task_stopped(struct wait_opts *wo,
+                                int ptrace, struct task_struct *p)
 {
+        struct siginfo __user *infop;
         int retval, exit_code, *p_code, why;
         uid_t uid = 0; /* unneeded, required by compiler */
         pid_t pid;
 
-        if (!(options & WUNTRACED))
+        /*
+         * Traditionally we see ptrace'd stopped tasks regardless of options.
+         */
+        if (!ptrace && !(wo->wo_flags & WUNTRACED))
                 return 0;
 
         exit_code = 0;
@@ -1349,7 +1367,7 @@ static int wait_task_stopped(int ptrace, struct task_struct *p,
         if (!exit_code)
                 goto unlock_sig;
 
-        if (!unlikely(options & WNOWAIT))
+        if (!unlikely(wo->wo_flags & WNOWAIT))
                 *p_code = 0;
 
         /* don't need the RCU readlock here as we're holding a spinlock */
@@ -1371,14 +1389,15 @@ unlock_sig:
         why = ptrace ? CLD_TRAPPED : CLD_STOPPED;
         read_unlock(&tasklist_lock);
 
-        if (unlikely(options & WNOWAIT))
-                return wait_noreap_copyout(p, pid, uid,
-                                           why, exit_code,
-                                           infop, ru);
+        if (unlikely(wo->wo_flags & WNOWAIT))
+                return wait_noreap_copyout(wo, p, pid, uid, why, exit_code);
+
+        retval = wo->wo_rusage
+                ? getrusage(p, RUSAGE_BOTH, wo->wo_rusage) : 0;
+        if (!retval && wo->wo_stat)
+                retval = put_user((exit_code << 8) | 0x7f, wo->wo_stat);
 
-        retval = ru ? getrusage(p, RUSAGE_BOTH, ru) : 0;
-        if (!retval && stat_addr)
-                retval = put_user((exit_code << 8) | 0x7f, stat_addr);
+        infop = wo->wo_info;
         if (!retval && infop)
                 retval = put_user(SIGCHLD, &infop->si_signo);
         if (!retval && infop)
@@ -1405,15 +1424,13 @@ unlock_sig:
  * the lock and this task is uninteresting.  If we return nonzero, we have
  * released the lock and the system call should return.
  */
-static int wait_task_continued(struct task_struct *p, int options,
-                               struct siginfo __user *infop,
-                               int __user *stat_addr, struct rusage __user *ru)
+static int wait_task_continued(struct wait_opts *wo, struct task_struct *p)
 {
         int retval;
         pid_t pid;
         uid_t uid;
 
-        if (!unlikely(options & WCONTINUED))
+        if (!unlikely(wo->wo_flags & WCONTINUED))
                 return 0;
 
         if (!(p->signal->flags & SIGNAL_STOP_CONTINUED))
@@ -1425,7 +1442,7 @@ static int wait_task_continued(struct task_struct *p, int options,
                 spin_unlock_irq(&p->sighand->siglock);
                 return 0;
         }
-        if (!unlikely(options & WNOWAIT))
+        if (!unlikely(wo->wo_flags & WNOWAIT))
                 p->signal->flags &= ~SIGNAL_STOP_CONTINUED;
         uid = __task_cred(p)->uid;
         spin_unlock_irq(&p->sighand->siglock);
@@ -1434,17 +1451,17 @@ static int wait_task_continued(struct task_struct *p, int options,
         get_task_struct(p);
         read_unlock(&tasklist_lock);
 
-        if (!infop) {
-                retval = ru ? getrusage(p, RUSAGE_BOTH, ru) : 0;
+        if (!wo->wo_info) {
+                retval = wo->wo_rusage
+                        ? getrusage(p, RUSAGE_BOTH, wo->wo_rusage) : 0;
                 put_task_struct(p);
-                if (!retval && stat_addr)
-                        retval = put_user(0xffff, stat_addr);
+                if (!retval && wo->wo_stat)
+                        retval = put_user(0xffff, wo->wo_stat);
                 if (!retval)
                         retval = pid;
         } else {
-                retval = wait_noreap_copyout(p, pid, uid,
-                                             CLD_CONTINUED, SIGCONT,
-                                             infop, ru);
+                retval = wait_noreap_copyout(wo, p, pid, uid,
+                                             CLD_CONTINUED, SIGCONT);
                 BUG_ON(retval == 0);
         }
 
@@ -1454,19 +1471,16 @@ static int wait_task_continued(struct task_struct *p, int options,
 /*
  * Consider @p for a wait by @parent.
  *
- * -ECHILD should be in *@notask_error before the first call.
+ * -ECHILD should be in ->notask_error before the first call.
  * Returns nonzero for a final return, when we have unlocked tasklist_lock.
  * Returns zero if the search for a child should continue;
- * then *@notask_error is 0 if @p is an eligible child,
+ * then ->notask_error is 0 if @p is an eligible child,
  * or another error from security_task_wait(), or still -ECHILD.
  */
-static int wait_consider_task(struct task_struct *parent, int ptrace,
-                              struct task_struct *p, int *notask_error,
-                              enum pid_type type, struct pid *pid, int options,
-                              struct siginfo __user *infop,
-                              int __user *stat_addr, struct rusage __user *ru)
+static int wait_consider_task(struct wait_opts *wo, struct task_struct *parent,
+                                int ptrace, struct task_struct *p)
 {
-        int ret = eligible_child(type, pid, options, p);
+        int ret = eligible_child(wo, p);
         if (!ret)
                 return ret;
 
@@ -1478,17 +1492,17 @@ static int wait_consider_task(struct task_struct *parent, int ptrace,
                  * to look for security policy problems, rather
                  * than for mysterious wait bugs.
                  */
-                if (*notask_error)
-                        *notask_error = ret;
+                if (wo->notask_error)
+                        wo->notask_error = ret;
                 return 0;
         }
 
-        if (likely(!ptrace) && unlikely(p->ptrace)) {
+        if (likely(!ptrace) && unlikely(task_ptrace(p))) {
                 /*
                  * This child is hidden by ptrace.
                  * We aren't allowed to see it now, but eventually we will.
                  */
-                *notask_error = 0;
+                wo->notask_error = 0;
                 return 0;
         }
 
@@ -1499,34 +1513,30 @@ static int wait_consider_task(struct task_struct *parent, int ptrace,
          * We don't reap group leaders with subthreads.
          */
         if (p->exit_state == EXIT_ZOMBIE && !delay_group_leader(p))
-                return wait_task_zombie(p, options, infop, stat_addr, ru);
+                return wait_task_zombie(wo, p);
 
         /*
          * It's stopped or running now, so it might
          * later continue, exit, or stop again.
          */
-        *notask_error = 0;
+        wo->notask_error = 0;
 
         if (task_stopped_code(p, ptrace))
-                return wait_task_stopped(ptrace, p, options,
-                                         infop, stat_addr, ru);
+                return wait_task_stopped(wo, ptrace, p);
 
-        return wait_task_continued(p, options, infop, stat_addr, ru);
+        return wait_task_continued(wo, p);
 }
 
 /*
  * Do the work of do_wait() for one thread in the group, @tsk.
  *
- * -ECHILD should be in *@notask_error before the first call.
+ * -ECHILD should be in ->notask_error before the first call.
  * Returns nonzero for a final return, when we have unlocked tasklist_lock.
  * Returns zero if the search for a child should continue; then
- * *@notask_error is 0 if there were any eligible children,
+ * ->notask_error is 0 if there were any eligible children,
  * or another error from security_task_wait(), or still -ECHILD.
  */
-static int do_wait_thread(struct task_struct *tsk, int *notask_error,
-                          enum pid_type type, struct pid *pid, int options,
-                          struct siginfo __user *infop, int __user *stat_addr,
-                          struct rusage __user *ru)
+static int do_wait_thread(struct wait_opts *wo, struct task_struct *tsk)
 {
         struct task_struct *p;
 
@@ -1535,9 +1545,7 @@ static int do_wait_thread(struct task_struct *tsk, int *notask_error,
                  * Do not consider detached threads.
                  */
                 if (!task_detached(p)) {
-                        int ret = wait_consider_task(tsk, 0, p, notask_error,
-                                                     type, pid, options,
-                                                     infop, stat_addr, ru);
+                        int ret = wait_consider_task(wo, tsk, 0, p);
                         if (ret)
                                 return ret;
                 }
@@ -1546,22 +1554,12 @@ static int do_wait_thread(struct task_struct *tsk, int *notask_error,
         return 0;
 }
 
-static int ptrace_do_wait(struct task_struct *tsk, int *notask_error,
-                          enum pid_type type, struct pid *pid, int options,
-                          struct siginfo __user *infop, int __user *stat_addr,
-                          struct rusage __user *ru)
+static int ptrace_do_wait(struct wait_opts *wo, struct task_struct *tsk)
 {
         struct task_struct *p;
 
-        /*
-         * Traditionally we see ptrace'd stopped tasks regardless of options.
-         */
-        options |= WUNTRACED;
-
         list_for_each_entry(p, &tsk->ptraced, ptrace_entry) {
-                int ret = wait_consider_task(tsk, 1, p, notask_error,
-                                             type, pid, options,
-                                             infop, stat_addr, ru);
+                int ret = wait_consider_task(wo, tsk, 1, p);
                 if (ret)
                         return ret;
         }
@@ -1569,65 +1567,59 @@ static int ptrace_do_wait(struct task_struct *tsk, int *notask_error,
         return 0;
 }
 
-static long do_wait(enum pid_type type, struct pid *pid, int options,
-                    struct siginfo __user *infop, int __user *stat_addr,
-                    struct rusage __user *ru)
+static long do_wait(struct wait_opts *wo)
 {
         DECLARE_WAITQUEUE(wait, current);
         struct task_struct *tsk;
         int retval;
 
-        trace_sched_process_wait(pid);
+        trace_sched_process_wait(wo->wo_pid);
 
         add_wait_queue(&current->signal->wait_chldexit,&wait);
 repeat:
         /*
          * If there is nothing that can match our critiera just get out.
-         * We will clear @retval to zero if we see any child that might later
-         * match our criteria, even if we are not able to reap it yet.
+         * We will clear ->notask_error to zero if we see any child that
+         * might later match our criteria, even if we are not able to reap
+         * it yet.
          */
-        retval = -ECHILD;
-        if ((type < PIDTYPE_MAX) && (!pid || hlist_empty(&pid->tasks[type])))
-                goto end;
+        wo->notask_error = -ECHILD;
+        if ((wo->wo_type < PIDTYPE_MAX) &&
+           (!wo->wo_pid || hlist_empty(&wo->wo_pid->tasks[wo->wo_type])))
+                goto notask;
 
-        current->state = TASK_INTERRUPTIBLE;
+        set_current_state(TASK_INTERRUPTIBLE);
         read_lock(&tasklist_lock);
         tsk = current;
         do {
-                int tsk_result = do_wait_thread(tsk, &retval,
-                                                type, pid, options,
-                                                infop, stat_addr, ru);
-                if (!tsk_result)
-                        tsk_result = ptrace_do_wait(tsk, &retval,
-                                                    type, pid, options,
-                                                    infop, stat_addr, ru);
-                if (tsk_result) {
-                        /*
-                         * tasklist_lock is unlocked and we have a final result.
-                         */
-                        retval = tsk_result;
+                retval = do_wait_thread(wo, tsk);
+                if (retval)
+                        goto end;
+
+                retval = ptrace_do_wait(wo, tsk);
+                if (retval)
                         goto end;
-                }
 
-                if (options & __WNOTHREAD)
+                if (wo->wo_flags & __WNOTHREAD)
                         break;
-                tsk = next_thread(tsk);
-                BUG_ON(tsk->signal != current->signal);
-        } while (tsk != current);
+        } while_each_thread(current, tsk);
         read_unlock(&tasklist_lock);
 
-        if (!retval && !(options & WNOHANG)) {
+notask:
+        retval = wo->notask_error;
+        if (!retval && !(wo->wo_flags & WNOHANG)) {
                 retval = -ERESTARTSYS;
                 if (!signal_pending(current)) {
                         schedule();
                         goto repeat;
                 }
         }
-
 end:
-        current->state = TASK_RUNNING;
+        __set_current_state(TASK_RUNNING);
         remove_wait_queue(&current->signal->wait_chldexit,&wait);
-        if (infop) {
+        if (wo->wo_info) {
+                struct siginfo __user *infop = wo->wo_info;
+
                 if (retval > 0)
                         retval = 0;
                 else {
@@ -1656,6 +1648,7 @@ end:
 SYSCALL_DEFINE5(waitid, int, which, pid_t, upid, struct siginfo __user *,
                 infop, int, options, struct rusage __user *, ru)
 {
+        struct wait_opts wo;
         struct pid *pid = NULL;
         enum pid_type type;
         long ret;
@@ -1685,7 +1678,14 @@ SYSCALL_DEFINE5(waitid, int, which, pid_t, upid, struct siginfo __user *,
 
         if (type < PIDTYPE_MAX)
                 pid = find_get_pid(upid);
-        ret = do_wait(type, pid, options, infop, NULL, ru);
+
+        wo.wo_type      = type;
+        wo.wo_pid       = pid;
+        wo.wo_flags     = options;
+        wo.wo_info      = infop;
+        wo.wo_stat      = NULL;
+        wo.wo_rusage    = ru;
+        ret = do_wait(&wo);
         put_pid(pid);
 
         /* avoid REGPARM breakage on x86: */
@@ -1696,6 +1696,7 @@ SYSCALL_DEFINE5(waitid, int, which, pid_t, upid, struct siginfo __user *,
 SYSCALL_DEFINE4(wait4, pid_t, upid, int __user *, stat_addr,
                 int, options, struct rusage __user *, ru)
 {
+        struct wait_opts wo;
         struct pid *pid = NULL;
         enum pid_type type;
         long ret;
@@ -1717,7 +1718,13 @@ SYSCALL_DEFINE4(wait4, pid_t, upid, int __user *, stat_addr,
                 pid = find_get_pid(upid);
         }
 
-        ret = do_wait(type, pid, options | WEXITED, NULL, stat_addr, ru);
+        wo.wo_type      = type;
+        wo.wo_pid       = pid;
+        wo.wo_flags     = options | WEXITED;
+        wo.wo_info      = NULL;
+        wo.wo_stat      = stat_addr;
+        wo.wo_rusage    = ru;
+        ret = do_wait(&wo);
         put_pid(pid);
 
         /* avoid REGPARM breakage on x86: */
diff --git a/kernel/fork.c b/kernel/fork.c
index 4430eb1376f2..bfee931ee3fb 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -17,7 +17,6 @@
 #include <linux/module.h>
 #include <linux/vmalloc.h>
 #include <linux/completion.h>
-#include <linux/mnt_namespace.h>
 #include <linux/personality.h>
 #include <linux/mempolicy.h>
 #include <linux/sem.h>
@@ -153,8 +152,7 @@ void __put_task_struct(struct task_struct *tsk)
         WARN_ON(atomic_read(&tsk->usage));
         WARN_ON(tsk == current);
 
-        put_cred(tsk->real_cred);
-        put_cred(tsk->cred);
+        exit_creds(tsk);
         delayacct_tsk_free(tsk);
 
         if (!profile_handoff_task(tsk))
@@ -178,7 +176,7 @@ void __init fork_init(unsigned long mempages)
         /* create a slab on which task_structs can be allocated */
         task_struct_cachep =
                 kmem_cache_create("task_struct", sizeof(struct task_struct),
-                        ARCH_MIN_TASKALIGN, SLAB_PANIC, NULL);
+                        ARCH_MIN_TASKALIGN, SLAB_PANIC | SLAB_NOTRACK, NULL);
 #endif
 
         /* do the arch specific task caches init */
@@ -568,18 +566,18 @@ void mm_release(struct task_struct *tsk, struct mm_struct *mm)
          * the value intact in a core dump, and to save the unnecessary
          * trouble otherwise.  Userland only wants this done for a sys_exit.
          */
-        if (tsk->clear_child_tid
-            && !(tsk->flags & PF_SIGNALED)
-            && atomic_read(&mm->mm_users) > 1) {
-                u32 __user * tidptr = tsk->clear_child_tid;
+        if (tsk->clear_child_tid) {
+                if (!(tsk->flags & PF_SIGNALED) &&
+                    atomic_read(&mm->mm_users) > 1) {
+                        /*
+                         * We don't check the error code - if userspace has
+                         * not set up a proper pointer then tough luck.
+                         */
+                        put_user(0, tsk->clear_child_tid);
+                        sys_futex(tsk->clear_child_tid, FUTEX_WAKE,
+                                        1, NULL, NULL, 0);
+                }
                 tsk->clear_child_tid = NULL;
-
-                /*
-                 * We don't check the error code - if userspace has
-                 * not set up a proper pointer then tough luck.
-                 */
-                put_user(0, tidptr);
-                sys_futex(tidptr, FUTEX_WAKE, 1, NULL, NULL, 0);
         }
 }
 
@@ -816,11 +814,8 @@ static int copy_signal(unsigned long clone_flags, struct task_struct *tsk)
 {
         struct signal_struct *sig;
 
-        if (clone_flags & CLONE_THREAD) {
-                atomic_inc(&current->signal->count);
-                atomic_inc(&current->signal->live);
+        if (clone_flags & CLONE_THREAD)
                 return 0;
-        }
 
         sig = kmem_cache_alloc(signal_cachep, GFP_KERNEL);
         tsk->signal = sig;
@@ -878,16 +873,6 @@ void __cleanup_signal(struct signal_struct *sig)
         kmem_cache_free(signal_cachep, sig);
 }
 
-static void cleanup_signal(struct task_struct *tsk)
-{
-        struct signal_struct *sig = tsk->signal;
-
-        atomic_dec(&sig->live);
-
-        if (atomic_dec_and_test(&sig->count))
-                __cleanup_signal(sig);
-}
-
 static void copy_flags(unsigned long clone_flags, struct task_struct *p)
 {
         unsigned long new_flags = p->flags;
@@ -1022,14 +1007,10 @@ static struct task_struct *copy_process(unsigned long clone_flags,
         copy_flags(clone_flags, p);
         INIT_LIST_HEAD(&p->children);
         INIT_LIST_HEAD(&p->sibling);
-#ifdef CONFIG_PREEMPT_RCU
-        p->rcu_read_lock_nesting = 0;
-        p->rcu_flipctr_idx = 0;
-#endif /* #ifdef CONFIG_PREEMPT_RCU */
+        rcu_copy_process(p);
         p->vfork_done = NULL;
         spin_lock_init(&p->alloc_lock);
 
-        clear_tsk_thread_flag(p, TIF_SIGPENDING);
         init_sigpending(&p->pending);
 
         p->utime = cputime_zero;
@@ -1241,6 +1222,8 @@ static struct task_struct *copy_process(unsigned long clone_flags,
         }
 
         if (clone_flags & CLONE_THREAD) {
+                atomic_inc(&current->signal->count);
+                atomic_inc(&current->signal->live);
                 p->group_leader = current->group_leader;
                 list_add_tail_rcu(&p->thread_group, &p->group_leader->thread_group);
         }
@@ -1270,6 +1253,7 @@ static struct task_struct *copy_process(unsigned long clone_flags,
         write_unlock_irq(&tasklist_lock);
         proc_fork_connector(p);
         cgroup_post_fork(p);
+        perf_counter_fork(p);
         return p;
 
 bad_fork_free_pid:
@@ -1283,7 +1267,8 @@ bad_fork_cleanup_mm:
         if (p->mm)
                 mmput(p->mm);
 bad_fork_cleanup_signal:
-        cleanup_signal(p);
+        if (!(clone_flags & CLONE_THREAD))
+                __cleanup_signal(p->signal);
 bad_fork_cleanup_sighand:
         __cleanup_sighand(p->sighand);
 bad_fork_cleanup_fs:
@@ -1308,8 +1293,7 @@ bad_fork_cleanup_put_domain:
         module_put(task_thread_info(p)->exec_domain->module);
 bad_fork_cleanup_count:
         atomic_dec(&p->cred->user->processes);
-        put_cred(p->real_cred);
-        put_cred(p->cred);
+        exit_creds(p);
 bad_fork_free:
         free_task(p);
 fork_out:
@@ -1409,12 +1393,6 @@ long do_fork(unsigned long clone_flags,
                 if (clone_flags & CLONE_VFORK) {
                         p->vfork_done = &vfork;
                         init_completion(&vfork);
-                } else if (!(clone_flags & CLONE_VM)) {
-                        /*
-                         * vfork will do an exec which will call
-                         * set_task_comm()
-                         */
-                        perf_counter_fork(p);
                 }
 
                 audit_finish_fork(p);
@@ -1470,20 +1448,20 @@ void __init proc_caches_init(void)
 {
         sighand_cachep = kmem_cache_create("sighand_cache",
                         sizeof(struct sighand_struct), 0,
-                        SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_DESTROY_BY_RCU,
-                        sighand_ctor);
+                        SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_DESTROY_BY_RCU|
+                        SLAB_NOTRACK, sighand_ctor);
         signal_cachep = kmem_cache_create("signal_cache",
                         sizeof(struct signal_struct), 0,
-                        SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
+                        SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL);
         files_cachep = kmem_cache_create("files_cache",
                         sizeof(struct files_struct), 0,
-                        SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
+                        SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL);
         fs_cachep = kmem_cache_create("fs_cache",
                         sizeof(struct fs_struct), 0,
-                        SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
+                        SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL);
         mm_cachep = kmem_cache_create("mm_struct",
                         sizeof(struct mm_struct), ARCH_MIN_MMSTRUCT_ALIGN,
-                        SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
+                        SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL);
         vm_area_cachep = KMEM_CACHE(vm_area_struct, SLAB_PANIC);
         mmap_init();
 }
diff --git a/kernel/freezer.c b/kernel/freezer.c
index 2f4936cf7083..bd1d42b17cb2 100644
--- a/kernel/freezer.c
+++ b/kernel/freezer.c
@@ -44,12 +44,19 @@ void refrigerator(void)
         recalc_sigpending(); /* We sent fake signal, clean it up */
         spin_unlock_irq(&current->sighand->siglock);
 
+        /* prevent accounting of that task to load */
+        current->flags |= PF_FREEZING;
+
         for (;;) {
                 set_current_state(TASK_UNINTERRUPTIBLE);
                 if (!frozen(current))
                         break;
                 schedule();
         }
+
+        /* Remove the accounting blocker */
+        current->flags &= ~PF_FREEZING;
+
         pr_debug("%s left refrigerator\n", current->comm);
         __set_current_state(save);
 }
diff --git a/kernel/futex.c b/kernel/futex.c
index 80b5ce716596..248dd119a86e 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -115,6 +115,9 @@ struct futex_q {
         /* rt_waiter storage for requeue_pi: */
         struct rt_mutex_waiter *rt_waiter;
 
+        /* The expected requeue pi target futex key: */
+        union futex_key *requeue_pi_key;
+
         /* Bitset for the optional bitmasked wakeup */
         u32 bitset;
 };
@@ -247,6 +250,7 @@ again:
         if (err < 0)
                 return err;
 
+        page = compound_head(page);
         lock_page(page);
         if (!page->mapping) {
                 unlock_page(page);
@@ -284,6 +288,25 @@ void put_futex_key(int fshared, union futex_key *key)
         drop_futex_key_refs(key);
 }
 
+/*
+ * fault_in_user_writeable - fault in user address and verify RW access
+ * @uaddr:      pointer to faulting user space address
+ *
+ * Slow path to fixup the fault we just took in the atomic write
+ * access to @uaddr.
+ *
+ * We have no generic implementation of a non destructive write to the
+ * user address. We know that we faulted in the atomic pagefault
+ * disabled section so we can as well avoid the #PF overhead by
+ * calling get_user_pages() right away.
+ */
+static int fault_in_user_writeable(u32 __user *uaddr)
+{
+        int ret = get_user_pages(current, current->mm, (unsigned long)uaddr,
+                                 1, 1, 0, NULL, NULL);
+        return ret < 0 ? ret : 0;
+}
+
 /**
  * futex_top_waiter() - Return the highest priority waiter on a futex
  * @hb:     the hash bucket the futex_q's reside in
@@ -896,7 +919,6 @@ retry:
 retry_private:
         op_ret = futex_atomic_op_inuser(op, uaddr2);
         if (unlikely(op_ret < 0)) {
-                u32 dummy;
 
                 double_unlock_hb(hb1, hb2);
 
@@ -914,7 +936,7 @@ retry_private:
                         goto out_put_keys;
                 }
 
-                ret = get_user(dummy, uaddr2);
+                ret = fault_in_user_writeable(uaddr2);
                 if (ret)
                         goto out_put_keys;
 
@@ -991,15 +1013,19 @@ void requeue_futex(struct futex_q *q, struct futex_hash_bucket *hb1,
  * requeue_pi_wake_futex() - Wake a task that acquired the lock during requeue
  * q:   the futex_q
  * key: the key of the requeue target futex
+ * hb:  the hash_bucket of the requeue target futex
  *
  * During futex_requeue, with requeue_pi=1, it is possible to acquire the
  * target futex if it is uncontended or via a lock steal.  Set the futex_q key
  * to the requeue target futex so the waiter can detect the wakeup on the right
  * futex, but remove it from the hb and NULL the rt_waiter so it can detect
- * atomic lock acquisition.  Must be called with the q->lock_ptr held.
+ * atomic lock acquisition.  Set the q->lock_ptr to the requeue target hb->lock
+ * to protect access to the pi_state to fixup the owner later.  Must be called
+ * with both q->lock_ptr and hb->lock held.
  */
 static inline
-void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key)
+void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key,
+                           struct futex_hash_bucket *hb)
 {
         drop_futex_key_refs(&q->key);
         get_futex_key_refs(key);
@@ -1011,6 +1037,11 @@ void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key)
         WARN_ON(!q->rt_waiter);
         q->rt_waiter = NULL;
 
+        q->lock_ptr = &hb->lock;
+#ifdef CONFIG_DEBUG_PI_LIST
+        q->list.plist.lock = &hb->lock;
+#endif
+
         wake_up_state(q->task, TASK_NORMAL);
 }
 
@@ -1061,6 +1092,10 @@ static int futex_proxy_trylock_atomic(u32 __user *pifutex,
         if (!top_waiter)
                 return 0;
 
+        /* Ensure we requeue to the expected futex. */
+        if (!match_futex(top_waiter->requeue_pi_key, key2))
+                return -EINVAL;
+
         /*
          * Try to take the lock for top_waiter.  Set the FUTEX_WAITERS bit in
          * the contended case or if set_waiters is 1.  The pi_state is returned
@@ -1069,7 +1104,7 @@ static int futex_proxy_trylock_atomic(u32 __user *pifutex,
         ret = futex_lock_pi_atomic(pifutex, hb2, key2, ps, top_waiter->task,
                                    set_waiters);
         if (ret == 1)
-                requeue_pi_wake_futex(top_waiter, key2);
+                requeue_pi_wake_futex(top_waiter, key2, hb2);
 
         return ret;
 }
@@ -1204,7 +1239,7 @@ retry_private:
                         double_unlock_hb(hb1, hb2);
                         put_futex_key(fshared, &key2);
                         put_futex_key(fshared, &key1);
-                        ret = get_user(curval2, uaddr2);
+                        ret = fault_in_user_writeable(uaddr2);
                         if (!ret)
                                 goto retry;
                         goto out;
@@ -1228,8 +1263,15 @@ retry_private:
                 if (!match_futex(&this->key, &key1))
                         continue;
 
-                WARN_ON(!requeue_pi && this->rt_waiter);
-                WARN_ON(requeue_pi && !this->rt_waiter);
+                /*
+                 * FUTEX_WAIT_REQEUE_PI and FUTEX_CMP_REQUEUE_PI should always
+                 * be paired with each other and no other futex ops.
+                 */
+                if ((requeue_pi && !this->rt_waiter) ||
+                    (!requeue_pi && this->rt_waiter)) {
+                        ret = -EINVAL;
+                        break;
+                }
 
                 /*
                  * Wake nr_wake waiters.  For requeue_pi, if we acquired the
@@ -1241,6 +1283,12 @@ retry_private:
                         continue;
                 }
 
+                /* Ensure we requeue to the expected futex for requeue_pi. */
+                if (requeue_pi && !match_futex(this->requeue_pi_key, &key2)) {
+                        ret = -EINVAL;
+                        break;
+                }
+
                 /*
                  * Requeue nr_requeue waiters and possibly one more in the case
                  * of requeue_pi if we couldn't acquire the lock atomically.
@@ -1254,7 +1302,7 @@ retry_private:
                                                         this->task, 1);
                         if (ret == 1) {
                                 /* We got the lock. */
-                                requeue_pi_wake_futex(this, &key2);
+                                requeue_pi_wake_futex(this, &key2, hb2);
                                 continue;
                         } else if (ret) {
                                 /* -EDEADLK */
@@ -1482,7 +1530,7 @@ retry:
 handle_fault:
         spin_unlock(q->lock_ptr);
 
-        ret = get_user(uval, uaddr);
+        ret = fault_in_user_writeable(uaddr);
 
         spin_lock(q->lock_ptr);
 
@@ -1716,6 +1764,7 @@ static int futex_wait(u32 __user *uaddr, int fshared,
         q.pi_state = NULL;
         q.bitset = bitset;
         q.rt_waiter = NULL;
+        q.requeue_pi_key = NULL;
 
         if (abs_time) {
                 to = &timeout;
@@ -1807,7 +1856,6 @@ static int futex_lock_pi(u32 __user *uaddr, int fshared,
 {
         struct hrtimer_sleeper timeout, *to = NULL;
         struct futex_hash_bucket *hb;
-        u32 uval;
         struct futex_q q;
         int res, ret;
 
@@ -1824,6 +1872,7 @@ static int futex_lock_pi(u32 __user *uaddr, int fshared,
 
         q.pi_state = NULL;
         q.rt_waiter = NULL;
+        q.requeue_pi_key = NULL;
 retry:
         q.key = FUTEX_KEY_INIT;
         ret = get_futex_key(uaddr, fshared, &q.key, VERIFY_WRITE);
@@ -1909,16 +1958,9 @@ out:
         return ret != -EINTR ? ret : -ERESTARTNOINTR;
 
 uaddr_faulted:
-        /*
-         * We have to r/w  *(int __user *)uaddr, and we have to modify it
-         * atomically.  Therefore, if we continue to fault after get_user()
-         * below, we need to handle the fault ourselves, while still holding
-         * the mmap_sem.  This can occur if the uaddr is under contention as
-         * we have to drop the mmap_sem in order to call get_user().
-         */
         queue_unlock(&q, hb);
 
-        ret = get_user(uval, uaddr);
+        ret = fault_in_user_writeable(uaddr);
         if (ret)
                 goto out_put_key;
 
@@ -2013,17 +2055,10 @@ out:
         return ret;
 
 pi_faulted:
-        /*
-         * We have to r/w  *(int __user *)uaddr, and we have to modify it
-         * atomically.  Therefore, if we continue to fault after get_user()
-         * below, we need to handle the fault ourselves, while still holding
-         * the mmap_sem.  This can occur if the uaddr is under contention as
-         * we have to drop the mmap_sem in order to call get_user().
-         */
         spin_unlock(&hb->lock);
         put_futex_key(fshared, &key);
 
-        ret = get_user(uval, uaddr);
+        ret = fault_in_user_writeable(uaddr);
         if (!ret)
                 goto retry;
 
@@ -2098,11 +2133,11 @@ int handle_early_requeue_pi_wakeup(struct futex_hash_bucket *hb,
  * We call schedule in futex_wait_queue_me() when we enqueue and return there
  * via the following:
  * 1) wakeup on uaddr2 after an atomic lock acquisition by futex_requeue()
- * 2) wakeup on uaddr2 after a requeue and subsequent unlock
- * 3) signal (before or after requeue)
- * 4) timeout (before or after requeue)
+ * 2) wakeup on uaddr2 after a requeue
+ * 3) signal
+ * 4) timeout
  *
- * If 3, we setup a restart_block with futex_wait_requeue_pi() as the function.
+ * If 3, cleanup and return -ERESTARTNOINTR.
  *
  * If 2, we may then block on trying to take the rt_mutex and return via:
  * 5) successful lock
@@ -2110,7 +2145,7 @@ int handle_early_requeue_pi_wakeup(struct futex_hash_bucket *hb,
  * 7) timeout
  * 8) other lock acquisition failure
  *
- * If 6, we setup a restart_block with futex_lock_pi() as the function.
+ * If 6, return -EWOULDBLOCK (restarting the syscall would do the same).
  *
  * If 4 or 7, we cleanup and return with -ETIMEDOUT.
  *
@@ -2149,15 +2184,16 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, int fshared,
         debug_rt_mutex_init_waiter(&rt_waiter);
         rt_waiter.task = NULL;
 
-        q.pi_state = NULL;
-        q.bitset = bitset;
-        q.rt_waiter = &rt_waiter;
-
         key2 = FUTEX_KEY_INIT;
         ret = get_futex_key(uaddr2, fshared, &key2, VERIFY_WRITE);
         if (unlikely(ret != 0))
                 goto out;
 
+        q.pi_state = NULL;
+        q.bitset = bitset;
+        q.rt_waiter = &rt_waiter;
+        q.requeue_pi_key = &key2;
+
         /* Prepare to wait on uaddr. */
         ret = futex_wait_setup(uaddr, val, fshared, &q, &hb);
         if (ret)
@@ -2228,14 +2264,11 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, int fshared,
                         rt_mutex_unlock(pi_mutex);
         } else if (ret == -EINTR) {
                 /*
-                 * We've already been requeued, but we have no way to
-                 * restart by calling futex_lock_pi() directly. We
-                 * could restart the syscall, but that will look at
-                 * the user space value and return right away. So we
-                 * drop back with EWOULDBLOCK to tell user space that
-                 * "val" has been changed. That's the same what the
-                 * restart of the syscall would do in
-                 * futex_wait_setup().
+                 * We've already been requeued, but cannot restart by calling
+                 * futex_lock_pi() directly. We could restart this syscall, but
+                 * it would detect that the user space "val" changed and return
+                 * -EWOULDBLOCK.  Save the overhead of the restart and return
+                 * -EWOULDBLOCK directly.
                  */
                 ret = -EWOULDBLOCK;
         }
diff --git a/kernel/futex_compat.c b/kernel/futex_compat.c
index d607a5b9ee29..235716556bf1 100644
--- a/kernel/futex_compat.c
+++ b/kernel/futex_compat.c
@@ -180,7 +180,8 @@ asmlinkage long compat_sys_futex(u32 __user *uaddr, int op, u32 val,
         int cmd = op & FUTEX_CMD_MASK;
 
         if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI ||
-                      cmd == FUTEX_WAIT_BITSET)) {
+                      cmd == FUTEX_WAIT_BITSET ||
+                      cmd == FUTEX_WAIT_REQUEUE_PI)) {
                 if (get_compat_timespec(&ts, utime))
                         return -EFAULT;
                 if (!timespec_valid(&ts))
@@ -191,7 +192,8 @@ asmlinkage long compat_sys_futex(u32 __user *uaddr, int op, u32 val,
                         t = ktime_add_safe(ktime_get(), t);
                 tp = &t;
         }
-        if (cmd == FUTEX_REQUEUE || cmd == FUTEX_CMP_REQUEUE)
+        if (cmd == FUTEX_REQUEUE || cmd == FUTEX_CMP_REQUEUE ||
+            cmd == FUTEX_CMP_REQUEUE_PI || cmd == FUTEX_WAKE_OP)
                 val2 = (int) (unsigned long) utime;
 
         return do_futex(uaddr, op, val, tp, uaddr2, val2, val3);
diff --git a/kernel/gcov/Kconfig b/kernel/gcov/Kconfig
new file mode 100644
index 000000000000..654efd09f6a9
--- /dev/null
+++ b/kernel/gcov/Kconfig
@@ -0,0 +1,48 @@
+menu "GCOV-based kernel profiling"
+
+config GCOV_KERNEL
+        bool "Enable gcov-based kernel profiling"
+        depends on DEBUG_FS && CONSTRUCTORS
+        default n
+        ---help---
+        This option enables gcov-based code profiling (e.g. for code coverage
+        measurements).
+
+        If unsure, say N.
+
+        Additionally specify CONFIG_GCOV_PROFILE_ALL=y to get profiling data
+        for the entire kernel. To enable profiling for specific files or
+        directories, add a line similar to the following to the respective
+        Makefile:
+
+        For a single file (e.g. main.o):
+                GCOV_PROFILE_main.o := y
+
+        For all files in one directory:
+                GCOV_PROFILE := y
+
+        To exclude files from being profiled even when CONFIG_GCOV_PROFILE_ALL
+        is specified, use:
+
+                GCOV_PROFILE_main.o := n
+        and:
+                GCOV_PROFILE := n
+
+        Note that the debugfs filesystem has to be mounted to access
+        profiling data.
+
+config GCOV_PROFILE_ALL
+        bool "Profile entire Kernel"
+        depends on GCOV_KERNEL
+        depends on S390 || X86 || (PPC && EXPERIMENTAL)
+        default n
+        ---help---
+        This options activates profiling for the entire kernel.
+
+        If unsure, say N.
+
+        Note that a kernel compiled with profiling flags will be significantly
+        larger and run slower. Also be sure to exclude files from profiling
+        which are not linked to the kernel image to prevent linker errors.
+
+endmenu
diff --git a/kernel/gcov/Makefile b/kernel/gcov/Makefile
new file mode 100644
index 000000000000..3f761001d517
--- /dev/null
+++ b/kernel/gcov/Makefile
@@ -0,0 +1,3 @@
+EXTRA_CFLAGS := -DSRCTREE='"$(srctree)"' -DOBJTREE='"$(objtree)"'
+
+obj-$(CONFIG_GCOV_KERNEL) := base.o fs.o gcc_3_4.o
diff --git a/kernel/gcov/base.c b/kernel/gcov/base.c
new file mode 100644
index 000000000000..9b22d03cc581
--- /dev/null
+++ b/kernel/gcov/base.c
@@ -0,0 +1,148 @@
+/*
+ *  This code maintains a list of active profiling data structures.
+ *
+ *    Copyright IBM Corp. 2009
+ *    Author(s): Peter Oberparleiter <oberpar@linux.vnet.ibm.com>
+ *
+ *    Uses gcc-internal data definitions.
+ *    Based on the gcov-kernel patch by:
+ *               Hubertus Franke <frankeh@us.ibm.com>
+ *               Nigel Hinds <nhinds@us.ibm.com>
+ *               Rajan Ravindran <rajancr@us.ibm.com>
+ *               Peter Oberparleiter <oberpar@linux.vnet.ibm.com>
+ *               Paul Larson
+ */
+
+#define pr_fmt(fmt)     "gcov: " fmt
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include "gcov.h"
+
+static struct gcov_info *gcov_info_head;
+static int gcov_events_enabled;
+static DEFINE_MUTEX(gcov_lock);
+
+/*
+ * __gcov_init is called by gcc-generated constructor code for each object
+ * file compiled with -fprofile-arcs.
+ */
+void __gcov_init(struct gcov_info *info)
+{
+        static unsigned int gcov_version;
+
+        mutex_lock(&gcov_lock);
+        if (gcov_version == 0) {
+                gcov_version = info->version;
+                /*
+                 * Printing gcc's version magic may prove useful for debugging
+                 * incompatibility reports.
+                 */
+                pr_info("version magic: 0x%x\n", gcov_version);
+        }
+        /*
+         * Add new profiling data structure to list and inform event
+         * listener.
+         */
+        info->next = gcov_info_head;
+        gcov_info_head = info;
+        if (gcov_events_enabled)
+                gcov_event(GCOV_ADD, info);
+        mutex_unlock(&gcov_lock);
+}
+EXPORT_SYMBOL(__gcov_init);
+
+/*
+ * These functions may be referenced by gcc-generated profiling code but serve
+ * no function for kernel profiling.
+ */
+void __gcov_flush(void)
+{
+        /* Unused. */
+}
+EXPORT_SYMBOL(__gcov_flush);
+
+void __gcov_merge_add(gcov_type *counters, unsigned int n_counters)
+{
+        /* Unused. */
+}
+EXPORT_SYMBOL(__gcov_merge_add);
+
+void __gcov_merge_single(gcov_type *counters, unsigned int n_counters)
+{
+        /* Unused. */
+}
+EXPORT_SYMBOL(__gcov_merge_single);
+
+void __gcov_merge_delta(gcov_type *counters, unsigned int n_counters)
+{
+        /* Unused. */
+}
+EXPORT_SYMBOL(__gcov_merge_delta);
+
+/**
+ * gcov_enable_events - enable event reporting through gcov_event()
+ *
+ * Turn on reporting of profiling data load/unload-events through the
+ * gcov_event() callback. Also replay all previous events once. This function
+ * is needed because some events are potentially generated too early for the
+ * callback implementation to handle them initially.
+ */
+void gcov_enable_events(void)
+{
+        struct gcov_info *info;
+
+        mutex_lock(&gcov_lock);
+        gcov_events_enabled = 1;
+        /* Perform event callback for previously registered entries. */
+        for (info = gcov_info_head; info; info = info->next)
+                gcov_event(GCOV_ADD, info);
+        mutex_unlock(&gcov_lock);
+}
+
+#ifdef CONFIG_MODULES
+static inline int within(void *addr, void *start, unsigned long size)
+{
+        return ((addr >= start) && (addr < start + size));
+}
+
+/* Update list and generate events when modules are unloaded. */
+static int gcov_module_notifier(struct notifier_block *nb, unsigned long event,
+                                void *data)
+{
+        struct module *mod = data;
+        struct gcov_info *info;
+        struct gcov_info *prev;
+
+        if (event != MODULE_STATE_GOING)
+                return NOTIFY_OK;
+        mutex_lock(&gcov_lock);
+        prev = NULL;
+        /* Remove entries located in module from linked list. */
+        for (info = gcov_info_head; info; info = info->next) {
+                if (within(info, mod->module_core, mod->core_size)) {
+                        if (prev)
+                                prev->next = info->next;
+                        else
+                                gcov_info_head = info->next;
+                        if (gcov_events_enabled)
+                                gcov_event(GCOV_REMOVE, info);
+                } else
+                        prev = info;
+        }
+        mutex_unlock(&gcov_lock);
+
+        return NOTIFY_OK;
+}
+
+static struct notifier_block gcov_nb = {
+        .notifier_call  = gcov_module_notifier,
+};
+
+static int __init gcov_init(void)
+{
+        return register_module_notifier(&gcov_nb);
+}
+device_initcall(gcov_init);
+#endif /* CONFIG_MODULES */
diff --git a/kernel/gcov/fs.c b/kernel/gcov/fs.c
new file mode 100644
index 000000000000..ef3c3f88a7a3
--- /dev/null
+++ b/kernel/gcov/fs.c
@@ -0,0 +1,673 @@
+/*
+ *  This code exports profiling data as debugfs files to userspace.
+ *
+ *    Copyright IBM Corp. 2009
+ *    Author(s): Peter Oberparleiter <oberpar@linux.vnet.ibm.com>
+ *
+ *    Uses gcc-internal data definitions.
+ *    Based on the gcov-kernel patch by:
+ *               Hubertus Franke <frankeh@us.ibm.com>
+ *               Nigel Hinds <nhinds@us.ibm.com>
+ *               Rajan Ravindran <rajancr@us.ibm.com>
+ *               Peter Oberparleiter <oberpar@linux.vnet.ibm.com>
+ *               Paul Larson
+ *               Yi CDL Yang
+ */
+
+#define pr_fmt(fmt)     "gcov: " fmt
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/debugfs.h>
+#include <linux/fs.h>
+#include <linux/list.h>
+#include <linux/string.h>
+#include <linux/slab.h>
+#include <linux/mutex.h>
+#include <linux/seq_file.h>
+#include "gcov.h"
+
+/**
+ * struct gcov_node - represents a debugfs entry
+ * @list: list head for child node list
+ * @children: child nodes
+ * @all: list head for list of all nodes
+ * @parent: parent node
+ * @info: associated profiling data structure if not a directory
+ * @ghost: when an object file containing profiling data is unloaded we keep a
+ *         copy of the profiling data here to allow collecting coverage data
+ *         for cleanup code. Such a node is called a "ghost".
+ * @dentry: main debugfs entry, either a directory or data file
+ * @links: associated symbolic links
+ * @name: data file basename
+ *
+ * struct gcov_node represents an entity within the gcov/ subdirectory
+ * of debugfs. There are directory and data file nodes. The latter represent
+ * the actual synthesized data file plus any associated symbolic links which
+ * are needed by the gcov tool to work correctly.
+ */
+struct gcov_node {
+        struct list_head list;
+        struct list_head children;
+        struct list_head all;
+        struct gcov_node *parent;
+        struct gcov_info *info;
+        struct gcov_info *ghost;
+        struct dentry *dentry;
+        struct dentry **links;
+        char name[0];
+};
+
+static const char objtree[] = OBJTREE;
+static const char srctree[] = SRCTREE;
+static struct gcov_node root_node;
+static struct dentry *reset_dentry;
+static LIST_HEAD(all_head);
+static DEFINE_MUTEX(node_lock);
+
+/* If non-zero, keep copies of profiling data for unloaded modules. */
+static int gcov_persist = 1;
+
+static int __init gcov_persist_setup(char *str)
+{
+        unsigned long val;
+
+        if (strict_strtoul(str, 0, &val)) {
+                pr_warning("invalid gcov_persist parameter '%s'\n", str);
+                return 0;
+        }
+        gcov_persist = val;
+        pr_info("setting gcov_persist to %d\n", gcov_persist);
+
+        return 1;
+}
+__setup("gcov_persist=", gcov_persist_setup);
+
+/*
+ * seq_file.start() implementation for gcov data files. Note that the
+ * gcov_iterator interface is designed to be more restrictive than seq_file
+ * (no start from arbitrary position, etc.), to simplify the iterator
+ * implementation.
+ */
+static void *gcov_seq_start(struct seq_file *seq, loff_t *pos)
+{
+        loff_t i;
+
+        gcov_iter_start(seq->private);
+        for (i = 0; i < *pos; i++) {
+                if (gcov_iter_next(seq->private))
+                        return NULL;
+        }
+        return seq->private;
+}
+
+/* seq_file.next() implementation for gcov data files. */
+static void *gcov_seq_next(struct seq_file *seq, void *data, loff_t *pos)
+{
+        struct gcov_iterator *iter = data;
+
+        if (gcov_iter_next(iter))
+                return NULL;
+        (*pos)++;
+
+        return iter;
+}
+
+/* seq_file.show() implementation for gcov data files. */
+static int gcov_seq_show(struct seq_file *seq, void *data)
+{
+        struct gcov_iterator *iter = data;
+
+        if (gcov_iter_write(iter, seq))
+                return -EINVAL;
+        return 0;
+}
+
+static void gcov_seq_stop(struct seq_file *seq, void *data)
+{
+        /* Unused. */
+}
+
+static const struct seq_operations gcov_seq_ops = {
+        .start  = gcov_seq_start,
+        .next   = gcov_seq_next,
+        .show   = gcov_seq_show,
+        .stop   = gcov_seq_stop,
+};
+
+/*
+ * Return the profiling data set for a given node. This can either be the
+ * original profiling data structure or a duplicate (also called "ghost")
+ * in case the associated object file has been unloaded.
+ */
+static struct gcov_info *get_node_info(struct gcov_node *node)
+{
+        if (node->info)
+                return node->info;
+
+        return node->ghost;
+}
+
+/*
+ * open() implementation for gcov data files. Create a copy of the profiling
+ * data set and initialize the iterator and seq_file interface.
+ */
+static int gcov_seq_open(struct inode *inode, struct file *file)
+{
+        struct gcov_node *node = inode->i_private;
+        struct gcov_iterator *iter;
+        struct seq_file *seq;
+        struct gcov_info *info;
+        int rc = -ENOMEM;
+
+        mutex_lock(&node_lock);
+        /*
+         * Read from a profiling data copy to minimize reference tracking
+         * complexity and concurrent access.
+         */
+        info = gcov_info_dup(get_node_info(node));
+        if (!info)
+                goto out_unlock;
+        iter = gcov_iter_new(info);
+        if (!iter)
+                goto err_free_info;
+        rc = seq_open(file, &gcov_seq_ops);
+        if (rc)
+                goto err_free_iter_info;
+        seq = file->private_data;
+        seq->private = iter;
+out_unlock:
+        mutex_unlock(&node_lock);
+        return rc;
+
+err_free_iter_info:
+        gcov_iter_free(iter);
+err_free_info:
+        gcov_info_free(info);
+        goto out_unlock;
+}
+
+/*
+ * release() implementation for gcov data files. Release resources allocated
+ * by open().
+ */
+static int gcov_seq_release(struct inode *inode, struct file *file)
+{
+        struct gcov_iterator *iter;
+        struct gcov_info *info;
+        struct seq_file *seq;
+
+        seq = file->private_data;
+        iter = seq->private;
+        info = gcov_iter_get_info(iter);
+        gcov_iter_free(iter);
+        gcov_info_free(info);
+        seq_release(inode, file);
+
+        return 0;
+}
+
+/*
+ * Find a node by the associated data file name. Needs to be called with
+ * node_lock held.
+ */
+static struct gcov_node *get_node_by_name(const char *name)
+{
+        struct gcov_node *node;
+        struct gcov_info *info;
+
+        list_for_each_entry(node, &all_head, all) {
+                info = get_node_info(node);
+                if (info && (strcmp(info->filename, name) == 0))
+                        return node;
+        }
+
+        return NULL;
+}
+
+static void remove_node(struct gcov_node *node);
+
+/*
+ * write() implementation for gcov data files. Reset profiling data for the
+ * associated file. If the object file has been unloaded (i.e. this is
+ * a "ghost" node), remove the debug fs node as well.
+ */
+static ssize_t gcov_seq_write(struct file *file, const char __user *addr,
+                              size_t len, loff_t *pos)
+{
+        struct seq_file *seq;
+        struct gcov_info *info;
+        struct gcov_node *node;
+
+        seq = file->private_data;
+        info = gcov_iter_get_info(seq->private);
+        mutex_lock(&node_lock);
+        node = get_node_by_name(info->filename);
+        if (node) {
+                /* Reset counts or remove node for unloaded modules. */
+                if (node->ghost)
+                        remove_node(node);
+                else
+                        gcov_info_reset(node->info);
+        }
+        /* Reset counts for open file. */
+        gcov_info_reset(info);
+        mutex_unlock(&node_lock);
+
+        return len;
+}
+
+/*
+ * Given a string <path> representing a file path of format:
+ *   path/to/file.gcda
+ * construct and return a new string:
+ *   <dir/>path/to/file.<ext>
+ */
+static char *link_target(const char *dir, const char *path, const char *ext)
+{
+        char *target;
+        char *old_ext;
+        char *copy;
+
+        copy = kstrdup(path, GFP_KERNEL);
+        if (!copy)
+                return NULL;
+        old_ext = strrchr(copy, '.');
+        if (old_ext)
+                *old_ext = '\0';
+        if (dir)
+                target = kasprintf(GFP_KERNEL, "%s/%s.%s", dir, copy, ext);
+        else
+                target = kasprintf(GFP_KERNEL, "%s.%s", copy, ext);
+        kfree(copy);
+
+        return target;
+}
+
+/*
+ * Construct a string representing the symbolic link target for the given
+ * gcov data file name and link type. Depending on the link type and the
+ * location of the data file, the link target can either point to a
+ * subdirectory of srctree, objtree or in an external location.
+ */
+static char *get_link_target(const char *filename, const struct gcov_link *ext)
+{
+        const char *rel;
+        char *result;
+
+        if (strncmp(filename, objtree, strlen(objtree)) == 0) {
+                rel = filename + strlen(objtree) + 1;
+                if (ext->dir == SRC_TREE)
+                        result = link_target(srctree, rel, ext->ext);
+                else
+                        result = link_target(objtree, rel, ext->ext);
+        } else {
+                /* External compilation. */
+                result = link_target(NULL, filename, ext->ext);
+        }
+
+        return result;
+}
+
+#define SKEW_PREFIX     ".tmp_"
+
+/*
+ * For a filename .tmp_filename.ext return filename.ext. Needed to compensate
+ * for filename skewing caused by the mod-versioning mechanism.
+ */
+static const char *deskew(const char *basename)
+{
+        if (strncmp(basename, SKEW_PREFIX, sizeof(SKEW_PREFIX) - 1) == 0)
+                return basename + sizeof(SKEW_PREFIX) - 1;
+        return basename;
+}
+
+/*
+ * Create links to additional files (usually .c and .gcno files) which the
+ * gcov tool expects to find in the same directory as the gcov data file.
+ */
+static void add_links(struct gcov_node *node, struct dentry *parent)
+{
+        char *basename;
+        char *target;
+        int num;
+        int i;
+
+        for (num = 0; gcov_link[num].ext; num++)
+                /* Nothing. */;
+        node->links = kcalloc(num, sizeof(struct dentry *), GFP_KERNEL);
+        if (!node->links)
+                return;
+        for (i = 0; i < num; i++) {
+                target = get_link_target(get_node_info(node)->filename,
+                                         &gcov_link[i]);
+                if (!target)
+                        goto out_err;
+                basename = strrchr(target, '/');
+                if (!basename)
+                        goto out_err;
+                basename++;
+                node->links[i] = debugfs_create_symlink(deskew(basename),
+                                                        parent, target);
+                if (!node->links[i])
+                        goto out_err;
+                kfree(target);
+        }
+
+        return;
+out_err:
+        kfree(target);
+        while (i-- > 0)
+                debugfs_remove(node->links[i]);
+        kfree(node->links);
+        node->links = NULL;
+}
+
+static const struct file_operations gcov_data_fops = {
+        .open           = gcov_seq_open,
+        .release        = gcov_seq_release,
+        .read           = seq_read,
+        .llseek         = seq_lseek,
+        .write          = gcov_seq_write,
+};
+
+/* Basic initialization of a new node. */
+static void init_node(struct gcov_node *node, struct gcov_info *info,
+                      const char *name, struct gcov_node *parent)
+{
+        INIT_LIST_HEAD(&node->list);
+        INIT_LIST_HEAD(&node->children);
+        INIT_LIST_HEAD(&node->all);
+        node->info = info;
+        node->parent = parent;
+        if (name)
+                strcpy(node->name, name);
+}
+
+/*
+ * Create a new node and associated debugfs entry. Needs to be called with
+ * node_lock held.
+ */
+static struct gcov_node *new_node(struct gcov_node *parent,
+                                  struct gcov_info *info, const char *name)
+{
+        struct gcov_node *node;
+
+        node = kzalloc(sizeof(struct gcov_node) + strlen(name) + 1, GFP_KERNEL);
+        if (!node) {
+                pr_warning("out of memory\n");
+                return NULL;
+        }
+        init_node(node, info, name, parent);
+        /* Differentiate between gcov data file nodes and directory nodes. */
+        if (info) {
+                node->dentry = debugfs_create_file(deskew(node->name), 0600,
+                                        parent->dentry, node, &gcov_data_fops);
+        } else
+                node->dentry = debugfs_create_dir(node->name, parent->dentry);
+        if (!node->dentry) {
+                pr_warning("could not create file\n");
+                kfree(node);
+                return NULL;
+        }
+        if (info)
+                add_links(node, parent->dentry);
+        list_add(&node->list, &parent->children);
+        list_add(&node->all, &all_head);
+
+        return node;
+}
+
+/* Remove symbolic links associated with node. */
+static void remove_links(struct gcov_node *node)
+{
+        int i;
+
+        if (!node->links)
+                return;
+        for (i = 0; gcov_link[i].ext; i++)
+                debugfs_remove(node->links[i]);
+        kfree(node->links);
+        node->links = NULL;
+}
+
+/*
+ * Remove node from all lists and debugfs and release associated resources.
+ * Needs to be called with node_lock held.
+ */
+static void release_node(struct gcov_node *node)
+{
+        list_del(&node->list);
+        list_del(&node->all);
+        debugfs_remove(node->dentry);
+        remove_links(node);
+        if (node->ghost)
+                gcov_info_free(node->ghost);
+        kfree(node);
+}
+
+/* Release node and empty parents. Needs to be called with node_lock held. */
+static void remove_node(struct gcov_node *node)
+{
+        struct gcov_node *parent;
+
+        while ((node != &root_node) && list_empty(&node->children)) {
+                parent = node->parent;
+                release_node(node);
+                node = parent;
+        }
+}
+
+/*
+ * Find child node with given basename. Needs to be called with node_lock
+ * held.
+ */
+static struct gcov_node *get_child_by_name(struct gcov_node *parent,
+                                           const char *name)
+{
+        struct gcov_node *node;
+
+        list_for_each_entry(node, &parent->children, list) {
+                if (strcmp(node->name, name) == 0)
+                        return node;
+        }
+
+        return NULL;
+}
+
+/*
+ * write() implementation for reset file. Reset all profiling data to zero
+ * and remove ghost nodes.
+ */
+static ssize_t reset_write(struct file *file, const char __user *addr,
+                           size_t len, loff_t *pos)
+{
+        struct gcov_node *node;
+
+        mutex_lock(&node_lock);
+restart:
+        list_for_each_entry(node, &all_head, all) {
+                if (node->info)
+                        gcov_info_reset(node->info);
+                else if (list_empty(&node->children)) {
+                        remove_node(node);
+                        /* Several nodes may have gone - restart loop. */
+                        goto restart;
+                }
+        }
+        mutex_unlock(&node_lock);
+
+        return len;
+}
+
+/* read() implementation for reset file. Unused. */
+static ssize_t reset_read(struct file *file, char __user *addr, size_t len,
+                          loff_t *pos)
+{
+        /* Allow read operation so that a recursive copy won't fail. */
+        return 0;
+}
+
+static const struct file_operations gcov_reset_fops = {
+        .write  = reset_write,
+        .read   = reset_read,
+};
+
+/*
+ * Create a node for a given profiling data set and add it to all lists and
+ * debugfs. Needs to be called with node_lock held.
+ */
+static void add_node(struct gcov_info *info)
+{
+        char *filename;
+        char *curr;
+        char *next;
+        struct gcov_node *parent;
+        struct gcov_node *node;
+
+        filename = kstrdup(info->filename, GFP_KERNEL);
+        if (!filename)
+                return;
+        parent = &root_node;
+        /* Create directory nodes along the path. */
+        for (curr = filename; (next = strchr(curr, '/')); curr = next + 1) {
+                if (curr == next)
+                        continue;
+                *next = 0;
+                if (strcmp(curr, ".") == 0)
+                        continue;
+                if (strcmp(curr, "..") == 0) {
+                        if (!parent->parent)
+                                goto err_remove;
+                        parent = parent->parent;
+                        continue;
+                }
+                node = get_child_by_name(parent, curr);
+                if (!node) {
+                        node = new_node(parent, NULL, curr);
+                        if (!node)
+                                goto err_remove;
+                }
+                parent = node;
+        }
+        /* Create file node. */
+        node = new_node(parent, info, curr);
+        if (!node)
+                goto err_remove;
+out:
+        kfree(filename);
+        return;
+
+err_remove:
+        remove_node(parent);
+        goto out;
+}
+
+/*
+ * The profiling data set associated with this node is being unloaded. Store a
+ * copy of the profiling data and turn this node into a "ghost".
+ */
+static int ghost_node(struct gcov_node *node)
+{
+        node->ghost = gcov_info_dup(node->info);
+        if (!node->ghost) {
+                pr_warning("could not save data for '%s' (out of memory)\n",
+                           node->info->filename);
+                return -ENOMEM;
+        }
+        node->info = NULL;
+
+        return 0;
+}
+
+/*
+ * Profiling data for this node has been loaded again. Add profiling data
+ * from previous instantiation and turn this node into a regular node.
+ */
+static void revive_node(struct gcov_node *node, struct gcov_info *info)
+{
+        if (gcov_info_is_compatible(node->ghost, info))
+                gcov_info_add(info, node->ghost);
+        else {
+                pr_warning("discarding saved data for '%s' (version changed)\n",
+                           info->filename);
+        }
+        gcov_info_free(node->ghost);
+        node->ghost = NULL;
+        node->info = info;
+}
+
+/*
+ * Callback to create/remove profiling files when code compiled with
+ * -fprofile-arcs is loaded/unloaded.
+ */
+void gcov_event(enum gcov_action action, struct gcov_info *info)
+{
+        struct gcov_node *node;
+
+        mutex_lock(&node_lock);
+        node = get_node_by_name(info->filename);
+        switch (action) {
+        case GCOV_ADD:
+                /* Add new node or revive ghost. */
+                if (!node) {
+                        add_node(info);
+                        break;
+                }
+                if (gcov_persist)
+                        revive_node(node, info);
+                else {
+                        pr_warning("could not add '%s' (already exists)\n",
+                                   info->filename);
+                }
+                break;
+        case GCOV_REMOVE:
+                /* Remove node or turn into ghost. */
+                if (!node) {
+                        pr_warning("could not remove '%s' (not found)\n",
+                                   info->filename);
+                        break;
+                }
+                if (gcov_persist) {
+                        if (!ghost_node(node))
+                                break;
+                }
+                remove_node(node);
+                break;
+        }
+        mutex_unlock(&node_lock);
+}
+
+/* Create debugfs entries. */
+static __init int gcov_fs_init(void)
+{
+        int rc = -EIO;
+
+        init_node(&root_node, NULL, NULL, NULL);
+        /*
+         * /sys/kernel/debug/gcov will be parent for the reset control file
+         * and all profiling files.
+         */
+        root_node.dentry = debugfs_create_dir("gcov", NULL);
+        if (!root_node.dentry)
+                goto err_remove;
+        /*
+         * Create reset file which resets all profiling counts when written
+         * to.
+         */
+        reset_dentry = debugfs_create_file("reset", 0600, root_node.dentry,
+                                           NULL, &gcov_reset_fops);
+        if (!reset_dentry)
+                goto err_remove;
+        /* Replay previous events to get our fs hierarchy up-to-date. */
+        gcov_enable_events();
+        return 0;
+
+err_remove:
+        pr_err("init failed\n");
+        if (root_node.dentry)
+                debugfs_remove(root_node.dentry);
+
+        return rc;
+}
+device_initcall(gcov_fs_init);
diff --git a/kernel/gcov/gcc_3_4.c b/kernel/gcov/gcc_3_4.c
new file mode 100644
index 000000000000..ae5bb4260033
--- /dev/null
+++ b/kernel/gcov/gcc_3_4.c
@@ -0,0 +1,447 @@
+/*
+ *  This code provides functions to handle gcc's profiling data format
+ *  introduced with gcc 3.4. Future versions of gcc may change the gcov
+ *  format (as happened before), so all format-specific information needs
+ *  to be kept modular and easily exchangeable.
+ *
+ *  This file is based on gcc-internal definitions. Functions and data
+ *  structures are defined to be compatible with gcc counterparts.
+ *  For a better understanding, refer to gcc source: gcc/gcov-io.h.
+ *
+ *    Copyright IBM Corp. 2009
+ *    Author(s): Peter Oberparleiter <oberpar@linux.vnet.ibm.com>
+ *
+ *    Uses gcc-internal data definitions.
+ */
+
+#include <linux/errno.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/seq_file.h>
+#include <linux/vmalloc.h>
+#include "gcov.h"
+
+/* Symbolic links to be created for each profiling data file. */
+const struct gcov_link gcov_link[] = {
+        { OBJ_TREE, "gcno" },   /* Link to .gcno file in $(objtree). */
+        { 0, NULL},
+};
+
+/*
+ * Determine whether a counter is active. Based on gcc magic. Doesn't change
+ * at run-time.
+ */
+static int counter_active(struct gcov_info *info, unsigned int type)
+{
+        return (1 << type) & info->ctr_mask;
+}
+
+/* Determine number of active counters. Based on gcc magic. */
+static unsigned int num_counter_active(struct gcov_info *info)
+{
+        unsigned int i;
+        unsigned int result = 0;
+
+        for (i = 0; i < GCOV_COUNTERS; i++) {
+                if (counter_active(info, i))
+                        result++;
+        }
+        return result;
+}
+
+/**
+ * gcov_info_reset - reset profiling data to zero
+ * @info: profiling data set
+ */
+void gcov_info_reset(struct gcov_info *info)
+{
+        unsigned int active = num_counter_active(info);
+        unsigned int i;
+
+        for (i = 0; i < active; i++) {
+                memset(info->counts[i].values, 0,
+                       info->counts[i].num * sizeof(gcov_type));
+        }
+}
+
+/**
+ * gcov_info_is_compatible - check if profiling data can be added
+ * @info1: first profiling data set
+ * @info2: second profiling data set
+ *
+ * Returns non-zero if profiling data can be added, zero otherwise.
+ */
+int gcov_info_is_compatible(struct gcov_info *info1, struct gcov_info *info2)
+{
+        return (info1->stamp == info2->stamp);
+}
+
+/**
+ * gcov_info_add - add up profiling data
+ * @dest: profiling data set to which data is added
+ * @source: profiling data set which is added
+ *
+ * Adds profiling counts of @source to @dest.
+ */
+void gcov_info_add(struct gcov_info *dest, struct gcov_info *source)
+{
+        unsigned int i;
+        unsigned int j;
+
+        for (i = 0; i < num_counter_active(dest); i++) {
+                for (j = 0; j < dest->counts[i].num; j++) {
+                        dest->counts[i].values[j] +=
+                                source->counts[i].values[j];
+                }
+        }
+}
+
+/* Get size of function info entry. Based on gcc magic. */
+static size_t get_fn_size(struct gcov_info *info)
+{
+        size_t size;
+
+        size = sizeof(struct gcov_fn_info) + num_counter_active(info) *
+               sizeof(unsigned int);
+        if (__alignof__(struct gcov_fn_info) > sizeof(unsigned int))
+                size = ALIGN(size, __alignof__(struct gcov_fn_info));
+        return size;
+}
+
+/* Get address of function info entry. Based on gcc magic. */
+static struct gcov_fn_info *get_fn_info(struct gcov_info *info, unsigned int fn)
+{
+        return (struct gcov_fn_info *)
+                ((char *) info->functions + fn * get_fn_size(info));
+}
+
+/**
+ * gcov_info_dup - duplicate profiling data set
+ * @info: profiling data set to duplicate
+ *
+ * Return newly allocated duplicate on success, %NULL on error.
+ */
+struct gcov_info *gcov_info_dup(struct gcov_info *info)
+{
+        struct gcov_info *dup;
+        unsigned int i;
+        unsigned int active;
+
+        /* Duplicate gcov_info. */
+        active = num_counter_active(info);
+        dup = kzalloc(sizeof(struct gcov_info) +
+                      sizeof(struct gcov_ctr_info) * active, GFP_KERNEL);
+        if (!dup)
+                return NULL;
+        dup->version            = info->version;
+        dup->stamp              = info->stamp;
+        dup->n_functions        = info->n_functions;
+        dup->ctr_mask           = info->ctr_mask;
+        /* Duplicate filename. */
+        dup->filename           = kstrdup(info->filename, GFP_KERNEL);
+        if (!dup->filename)
+                goto err_free;
+        /* Duplicate table of functions. */
+        dup->functions = kmemdup(info->functions, info->n_functions *
+                                 get_fn_size(info), GFP_KERNEL);
+        if (!dup->functions)
+                goto err_free;
+        /* Duplicate counter arrays. */
+        for (i = 0; i < active ; i++) {
+                struct gcov_ctr_info *ctr = &info->counts[i];
+                size_t size = ctr->num * sizeof(gcov_type);
+
+                dup->counts[i].num = ctr->num;
+                dup->counts[i].merge = ctr->merge;
+                dup->counts[i].values = vmalloc(size);
+                if (!dup->counts[i].values)
+                        goto err_free;
+                memcpy(dup->counts[i].values, ctr->values, size);
+        }
+        return dup;
+
+err_free:
+        gcov_info_free(dup);
+        return NULL;
+}
+
+/**
+ * gcov_info_free - release memory for profiling data set duplicate
+ * @info: profiling data set duplicate to free
+ */
+void gcov_info_free(struct gcov_info *info)
+{
+        unsigned int active = num_counter_active(info);
+        unsigned int i;
+
+        for (i = 0; i < active ; i++)
+                vfree(info->counts[i].values);
+        kfree(info->functions);
+        kfree(info->filename);
+        kfree(info);
+}
+
+/**
+ * struct type_info - iterator helper array
+ * @ctr_type: counter type
+ * @offset: index of the first value of the current function for this type
+ *
+ * This array is needed to convert the in-memory data format into the in-file
+ * data format:
+ *
+ * In-memory:
+ *   for each counter type
+ *     for each function
+ *       values
+ *
+ * In-file:
+ *   for each function
+ *     for each counter type
+ *       values
+ *
+ * See gcc source gcc/gcov-io.h for more information on data organization.
+ */
+struct type_info {
+        int ctr_type;
+        unsigned int offset;
+};
+
+/**
+ * struct gcov_iterator - specifies current file position in logical records
+ * @info: associated profiling data
+ * @record: record type
+ * @function: function number
+ * @type: counter type
+ * @count: index into values array
+ * @num_types: number of counter types
+ * @type_info: helper array to get values-array offset for current function
+ */
+struct gcov_iterator {
+        struct gcov_info *info;
+
+        int record;
+        unsigned int function;
+        unsigned int type;
+        unsigned int count;
+
+        int num_types;
+        struct type_info type_info[0];
+};
+
+static struct gcov_fn_info *get_func(struct gcov_iterator *iter)
+{
+        return get_fn_info(iter->info, iter->function);
+}
+
+static struct type_info *get_type(struct gcov_iterator *iter)
+{
+        return &iter->type_info[iter->type];
+}
+
+/**
+ * gcov_iter_new - allocate and initialize profiling data iterator
+ * @info: profiling data set to be iterated
+ *
+ * Return file iterator on success, %NULL otherwise.
+ */
+struct gcov_iterator *gcov_iter_new(struct gcov_info *info)
+{
+        struct gcov_iterator *iter;
+
+        iter = kzalloc(sizeof(struct gcov_iterator) +
+                       num_counter_active(info) * sizeof(struct type_info),
+                       GFP_KERNEL);
+        if (iter)
+                iter->info = info;
+
+        return iter;
+}
+
+/**
+ * gcov_iter_free - release memory for iterator
+ * @iter: file iterator to free
+ */
+void gcov_iter_free(struct gcov_iterator *iter)
+{
+        kfree(iter);
+}
+
+/**
+ * gcov_iter_get_info - return profiling data set for given file iterator
+ * @iter: file iterator
+ */
+struct gcov_info *gcov_iter_get_info(struct gcov_iterator *iter)
+{
+        return iter->info;
+}
+
+/**
+ * gcov_iter_start - reset file iterator to starting position
+ * @iter: file iterator
+ */
+void gcov_iter_start(struct gcov_iterator *iter)
+{
+        int i;
+
+        iter->record = 0;
+        iter->function = 0;
+        iter->type = 0;
+        iter->count = 0;
+        iter->num_types = 0;
+        for (i = 0; i < GCOV_COUNTERS; i++) {
+                if (counter_active(iter->info, i)) {
+                        iter->type_info[iter->num_types].ctr_type = i;
+                        iter->type_info[iter->num_types++].offset = 0;
+                }
+        }
+}
+
+/* Mapping of logical record number to actual file content. */
+#define RECORD_FILE_MAGIC       0
+#define RECORD_GCOV_VERSION     1
+#define RECORD_TIME_STAMP       2
+#define RECORD_FUNCTION_TAG     3
+#define RECORD_FUNCTON_TAG_LEN  4
+#define RECORD_FUNCTION_IDENT   5
+#define RECORD_FUNCTION_CHECK   6
+#define RECORD_COUNT_TAG        7
+#define RECORD_COUNT_LEN        8
+#define RECORD_COUNT            9
+
+/**
+ * gcov_iter_next - advance file iterator to next logical record
+ * @iter: file iterator
+ *
+ * Return zero if new position is valid, non-zero if iterator has reached end.
+ */
+int gcov_iter_next(struct gcov_iterator *iter)
+{
+        switch (iter->record) {
+        case RECORD_FILE_MAGIC:
+        case RECORD_GCOV_VERSION:
+        case RECORD_FUNCTION_TAG:
+        case RECORD_FUNCTON_TAG_LEN:
+        case RECORD_FUNCTION_IDENT:
+        case RECORD_COUNT_TAG:
+                /* Advance to next record */
+                iter->record++;
+                break;
+        case RECORD_COUNT:
+                /* Advance to next count */
+                iter->count++;
+                /* fall through */
+        case RECORD_COUNT_LEN:
+                if (iter->count < get_func(iter)->n_ctrs[iter->type]) {
+                        iter->record = 9;
+                        break;
+                }
+                /* Advance to next counter type */
+                get_type(iter)->offset += iter->count;
+                iter->count = 0;
+                iter->type++;
+                /* fall through */
+        case RECORD_FUNCTION_CHECK:
+                if (iter->type < iter->num_types) {
+                        iter->record = 7;
+                        break;
+                }
+                /* Advance to next function */
+                iter->type = 0;
+                iter->function++;
+                /* fall through */
+        case RECORD_TIME_STAMP:
+                if (iter->function < iter->info->n_functions)
+                        iter->record = 3;
+                else
+                        iter->record = -1;
+                break;
+        }
+        /* Check for EOF. */
+        if (iter->record == -1)
+                return -EINVAL;
+        else
+                return 0;
+}
+
+/**
+ * seq_write_gcov_u32 - write 32 bit number in gcov format to seq_file
+ * @seq: seq_file handle
+ * @v: value to be stored
+ *
+ * Number format defined by gcc: numbers are recorded in the 32 bit
+ * unsigned binary form of the endianness of the machine generating the
+ * file.
+ */
+static int seq_write_gcov_u32(struct seq_file *seq, u32 v)
+{
+        return seq_write(seq, &v, sizeof(v));
+}
+
+/**
+ * seq_write_gcov_u64 - write 64 bit number in gcov format to seq_file
+ * @seq: seq_file handle
+ * @v: value to be stored
+ *
+ * Number format defined by gcc: numbers are recorded in the 32 bit
+ * unsigned binary form of the endianness of the machine generating the
+ * file. 64 bit numbers are stored as two 32 bit numbers, the low part
+ * first.
+ */
+static int seq_write_gcov_u64(struct seq_file *seq, u64 v)
+{
+        u32 data[2];
+
+        data[0] = (v & 0xffffffffUL);
+        data[1] = (v >> 32);
+        return seq_write(seq, data, sizeof(data));
+}
+
+/**
+ * gcov_iter_write - write data for current pos to seq_file
+ * @iter: file iterator
+ * @seq: seq_file handle
+ *
+ * Return zero on success, non-zero otherwise.
+ */
+int gcov_iter_write(struct gcov_iterator *iter, struct seq_file *seq)
+{
+        int rc = -EINVAL;
+
+        switch (iter->record) {
+        case RECORD_FILE_MAGIC:
+                rc = seq_write_gcov_u32(seq, GCOV_DATA_MAGIC);
+                break;
+        case RECORD_GCOV_VERSION:
+                rc = seq_write_gcov_u32(seq, iter->info->version);
+                break;
+        case RECORD_TIME_STAMP:
+                rc = seq_write_gcov_u32(seq, iter->info->stamp);
+                break;
+        case RECORD_FUNCTION_TAG:
+                rc = seq_write_gcov_u32(seq, GCOV_TAG_FUNCTION);
+                break;
+        case RECORD_FUNCTON_TAG_LEN:
+                rc = seq_write_gcov_u32(seq, 2);
+                break;
+        case RECORD_FUNCTION_IDENT:
+                rc = seq_write_gcov_u32(seq, get_func(iter)->ident);
+                break;
+        case RECORD_FUNCTION_CHECK:
+                rc = seq_write_gcov_u32(seq, get_func(iter)->checksum);
+                break;
+        case RECORD_COUNT_TAG:
+                rc = seq_write_gcov_u32(seq,
+                        GCOV_TAG_FOR_COUNTER(get_type(iter)->ctr_type));
+                break;
+        case RECORD_COUNT_LEN:
+                rc = seq_write_gcov_u32(seq,
+                                get_func(iter)->n_ctrs[iter->type] * 2);
+                break;
+        case RECORD_COUNT:
+                rc = seq_write_gcov_u64(seq,
+                        iter->info->counts[iter->type].
+                                values[iter->count + get_type(iter)->offset]);
+                break;
+        }
+        return rc;
+}
diff --git a/kernel/gcov/gcov.h b/kernel/gcov/gcov.h
new file mode 100644
index 000000000000..060073ebf7a6
--- /dev/null
+++ b/kernel/gcov/gcov.h
@@ -0,0 +1,128 @@
+/*
+ *  Profiling infrastructure declarations.
+ *
+ *  This file is based on gcc-internal definitions. Data structures are
+ *  defined to be compatible with gcc counterparts. For a better
+ *  understanding, refer to gcc source: gcc/gcov-io.h.
+ *
+ *    Copyright IBM Corp. 2009
+ *    Author(s): Peter Oberparleiter <oberpar@linux.vnet.ibm.com>
+ *
+ *    Uses gcc-internal data definitions.
+ */
+
+#ifndef GCOV_H
+#define GCOV_H GCOV_H
+
+#include <linux/types.h>
+
+/*
+ * Profiling data types used for gcc 3.4 and above - these are defined by
+ * gcc and need to be kept as close to the original definition as possible to
+ * remain compatible.
+ */
+#define GCOV_COUNTERS           5
+#define GCOV_DATA_MAGIC         ((unsigned int) 0x67636461)
+#define GCOV_TAG_FUNCTION       ((unsigned int) 0x01000000)
+#define GCOV_TAG_COUNTER_BASE   ((unsigned int) 0x01a10000)
+#define GCOV_TAG_FOR_COUNTER(count)                                     \
+        (GCOV_TAG_COUNTER_BASE + ((unsigned int) (count) << 17))
+
+#if BITS_PER_LONG >= 64
+typedef long gcov_type;
+#else
+typedef long long gcov_type;
+#endif
+
+/**
+ * struct gcov_fn_info - profiling meta data per function
+ * @ident: object file-unique function identifier
+ * @checksum: function checksum
+ * @n_ctrs: number of values per counter type belonging to this function
+ *
+ * This data is generated by gcc during compilation and doesn't change
+ * at run-time.
+ */
+struct gcov_fn_info {
+        unsigned int ident;
+        unsigned int checksum;
+        unsigned int n_ctrs[0];
+};
+
+/**
+ * struct gcov_ctr_info - profiling data per counter type
+ * @num: number of counter values for this type
+ * @values: array of counter values for this type
+ * @merge: merge function for counter values of this type (unused)
+ *
+ * This data is generated by gcc during compilation and doesn't change
+ * at run-time with the exception of the values array.
+ */
+struct gcov_ctr_info {
+        unsigned int    num;
+        gcov_type       *values;
+        void            (*merge)(gcov_type *, unsigned int);
+};
+
+/**
+ * struct gcov_info - profiling data per object file
+ * @version: gcov version magic indicating the gcc version used for compilation
+ * @next: list head for a singly-linked list
+ * @stamp: time stamp
+ * @filename: name of the associated gcov data file
+ * @n_functions: number of instrumented functions
+ * @functions: function data
+ * @ctr_mask: mask specifying which counter types are active
+ * @counts: counter data per counter type
+ *
+ * This data is generated by gcc during compilation and doesn't change
+ * at run-time with the exception of the next pointer.
+ */
+struct gcov_info {
+        unsigned int                    version;
+        struct gcov_info                *next;
+        unsigned int                    stamp;
+        const char                      *filename;
+        unsigned int                    n_functions;
+        const struct gcov_fn_info       *functions;
+        unsigned int                    ctr_mask;
+        struct gcov_ctr_info            counts[0];
+};
+
+/* Base interface. */
+enum gcov_action {
+        GCOV_ADD,
+        GCOV_REMOVE,
+};
+
+void gcov_event(enum gcov_action action, struct gcov_info *info);
+void gcov_enable_events(void);
+
+/* Iterator control. */
+struct seq_file;
+struct gcov_iterator;
+
+struct gcov_iterator *gcov_iter_new(struct gcov_info *info);
+void gcov_iter_free(struct gcov_iterator *iter);
+void gcov_iter_start(struct gcov_iterator *iter);
+int gcov_iter_next(struct gcov_iterator *iter);
+int gcov_iter_write(struct gcov_iterator *iter, struct seq_file *seq);
+struct gcov_info *gcov_iter_get_info(struct gcov_iterator *iter);
+
+/* gcov_info control. */
+void gcov_info_reset(struct gcov_info *info);
+int gcov_info_is_compatible(struct gcov_info *info1, struct gcov_info *info2);
+void gcov_info_add(struct gcov_info *dest, struct gcov_info *source);
+struct gcov_info *gcov_info_dup(struct gcov_info *info);
+void gcov_info_free(struct gcov_info *info);
+
+struct gcov_link {
+        enum {
+                OBJ_TREE,
+                SRC_TREE,
+        } dir;
+        const char *ext;
+};
+extern const struct gcov_link gcov_link[];
+
+#endif /* GCOV_H */
diff --git a/kernel/groups.c b/kernel/groups.c
new file mode 100644
index 000000000000..2b45b2ee3964
--- /dev/null
+++ b/kernel/groups.c
@@ -0,0 +1,288 @@
+/*
+ * Supplementary group IDs
+ */
+#include <linux/cred.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/security.h>
+#include <linux/syscalls.h>
+#include <asm/uaccess.h>
+
+/* init to 2 - one for init_task, one to ensure it is never freed */
+struct group_info init_groups = { .usage = ATOMIC_INIT(2) };
+
+struct group_info *groups_alloc(int gidsetsize)
+{
+        struct group_info *group_info;
+        int nblocks;
+        int i;
+
+        nblocks = (gidsetsize + NGROUPS_PER_BLOCK - 1) / NGROUPS_PER_BLOCK;
+        /* Make sure we always allocate at least one indirect block pointer */
+        nblocks = nblocks ? : 1;
+        group_info = kmalloc(sizeof(*group_info) + nblocks*sizeof(gid_t *), GFP_USER);
+        if (!group_info)
+                return NULL;
+        group_info->ngroups = gidsetsize;
+        group_info->nblocks = nblocks;
+        atomic_set(&group_info->usage, 1);
+
+        if (gidsetsize <= NGROUPS_SMALL)
+                group_info->blocks[0] = group_info->small_block;
+        else {
+                for (i = 0; i < nblocks; i++) {
+                        gid_t *b;
+                        b = (void *)__get_free_page(GFP_USER);
+                        if (!b)
+                                goto out_undo_partial_alloc;
+                        group_info->blocks[i] = b;
+                }
+        }
+        return group_info;
+
+out_undo_partial_alloc:
+        while (--i >= 0) {
+                free_page((unsigned long)group_info->blocks[i]);
+        }
+        kfree(group_info);
+        return NULL;
+}
+
+EXPORT_SYMBOL(groups_alloc);
+
+void groups_free(struct group_info *group_info)
+{
+        if (group_info->blocks[0] != group_info->small_block) {
+                int i;
+                for (i = 0; i < group_info->nblocks; i++)
+                        free_page((unsigned long)group_info->blocks[i]);
+        }
+        kfree(group_info);
+}
+
+EXPORT_SYMBOL(groups_free);
+
+/* export the group_info to a user-space array */
+static int groups_to_user(gid_t __user *grouplist,
+                          const struct group_info *group_info)
+{
+        int i;
+        unsigned int count = group_info->ngroups;
+
+        for (i = 0; i < group_info->nblocks; i++) {
+                unsigned int cp_count = min(NGROUPS_PER_BLOCK, count);
+                unsigned int len = cp_count * sizeof(*grouplist);
+
+                if (copy_to_user(grouplist, group_info->blocks[i], len))
+                        return -EFAULT;
+
+                grouplist += NGROUPS_PER_BLOCK;
+                count -= cp_count;
+        }
+        return 0;
+}
+
+/* fill a group_info from a user-space array - it must be allocated already */
+static int groups_from_user(struct group_info *group_info,
+    gid_t __user *grouplist)
+{
+        int i;
+        unsigned int count = group_info->ngroups;
+
+        for (i = 0; i < group_info->nblocks; i++) {
+                unsigned int cp_count = min(NGROUPS_PER_BLOCK, count);
+                unsigned int len = cp_count * sizeof(*grouplist);
+
+                if (copy_from_user(group_info->blocks[i], grouplist, len))
+                        return -EFAULT;
+
+                grouplist += NGROUPS_PER_BLOCK;
+                count -= cp_count;
+        }
+        return 0;
+}
+
+/* a simple Shell sort */
+static void groups_sort(struct group_info *group_info)
+{
+        int base, max, stride;
+        int gidsetsize = group_info->ngroups;
+
+        for (stride = 1; stride < gidsetsize; stride = 3 * stride + 1)
+                ; /* nothing */
+        stride /= 3;
+
+        while (stride) {
+                max = gidsetsize - stride;
+                for (base = 0; base < max; base++) {
+                        int left = base;
+                        int right = left + stride;
+                        gid_t tmp = GROUP_AT(group_info, right);
+
+                        while (left >= 0 && GROUP_AT(group_info, left) > tmp) {
+                                GROUP_AT(group_info, right) =
+                                    GROUP_AT(group_info, left);
+                                right = left;
+                                left -= stride;
+                        }
+                        GROUP_AT(group_info, right) = tmp;
+                }
+                stride /= 3;
+        }
+}
+
+/* a simple bsearch */
+int groups_search(const struct group_info *group_info, gid_t grp)
+{
+        unsigned int left, right;
+
+        if (!group_info)
+                return 0;
+
+        left = 0;
+        right = group_info->ngroups;
+        while (left < right) {
+                unsigned int mid = (left+right)/2;
+                int cmp = grp - GROUP_AT(group_info, mid);
+                if (cmp > 0)
+                        left = mid + 1;
+                else if (cmp < 0)
+                        right = mid;
+                else
+                        return 1;
+        }
+        return 0;
+}
+
+/**
+ * set_groups - Change a group subscription in a set of credentials
+ * @new: The newly prepared set of credentials to alter
+ * @group_info: The group list to install
+ *
+ * Validate a group subscription and, if valid, insert it into a set
+ * of credentials.
+ */
+int set_groups(struct cred *new, struct group_info *group_info)
+{
+        int retval;
+
+        retval = security_task_setgroups(group_info);
+        if (retval)
+                return retval;
+
+        put_group_info(new->group_info);
+        groups_sort(group_info);
+        get_group_info(group_info);
+        new->group_info = group_info;
+        return 0;
+}
+
+EXPORT_SYMBOL(set_groups);
+
+/**
+ * set_current_groups - Change current's group subscription
+ * @group_info: The group list to impose
+ *
+ * Validate a group subscription and, if valid, impose it upon current's task
+ * security record.
+ */
+int set_current_groups(struct group_info *group_info)
+{
+        struct cred *new;
+        int ret;
+
+        new = prepare_creds();
+        if (!new)
+                return -ENOMEM;
+
+        ret = set_groups(new, group_info);
+        if (ret < 0) {
+                abort_creds(new);
+                return ret;
+        }
+
+        return commit_creds(new);
+}
+
+EXPORT_SYMBOL(set_current_groups);
+
+SYSCALL_DEFINE2(getgroups, int, gidsetsize, gid_t __user *, grouplist)
+{
+        const struct cred *cred = current_cred();
+        int i;
+
+        if (gidsetsize < 0)
+                return -EINVAL;
+
+        /* no need to grab task_lock here; it cannot change */
+        i = cred->group_info->ngroups;
+        if (gidsetsize) {
+                if (i > gidsetsize) {
+                        i = -EINVAL;
+                        goto out;
+                }
+                if (groups_to_user(grouplist, cred->group_info)) {
+                        i = -EFAULT;
+                        goto out;
+                }
+        }
+out:
+        return i;
+}
+
+/*
+ *      SMP: Our groups are copy-on-write. We can set them safely
+ *      without another task interfering.
+ */
+
+SYSCALL_DEFINE2(setgroups, int, gidsetsize, gid_t __user *, grouplist)
+{
+        struct group_info *group_info;
+        int retval;
+
+        if (!capable(CAP_SETGID))
+                return -EPERM;
+        if ((unsigned)gidsetsize > NGROUPS_MAX)
+                return -EINVAL;
+
+        group_info = groups_alloc(gidsetsize);
+        if (!group_info)
+                return -ENOMEM;
+        retval = groups_from_user(group_info, grouplist);
+        if (retval) {
+                put_group_info(group_info);
+                return retval;
+        }
+
+        retval = set_current_groups(group_info);
+        put_group_info(group_info);
+
+        return retval;
+}
+
+/*
+ * Check whether we're fsgid/egid or in the supplemental group..
+ */
+int in_group_p(gid_t grp)
+{
+        const struct cred *cred = current_cred();
+        int retval = 1;
+
+        if (grp != cred->fsgid)
+                retval = groups_search(cred->group_info, grp);
+        return retval;
+}
+
+EXPORT_SYMBOL(in_group_p);
+
+int in_egroup_p(gid_t grp)
+{
+        const struct cred *cred = current_cred();
+        int retval = 1;
+
+        if (grp != cred->egid)
+                retval = groups_search(cred->group_info, grp);
+        return retval;
+}
+
+EXPORT_SYMBOL(in_egroup_p);
diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c
index b675a67c9ac3..05071bf6a37b 100644
--- a/kernel/hrtimer.c
+++ b/kernel/hrtimer.c
@@ -191,6 +191,46 @@ struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,
         }
 }
 
+
+/*
+ * Get the preferred target CPU for NOHZ
+ */
+static int hrtimer_get_target(int this_cpu, int pinned)
+{
+#ifdef CONFIG_NO_HZ
+        if (!pinned && get_sysctl_timer_migration() && idle_cpu(this_cpu)) {
+                int preferred_cpu = get_nohz_load_balancer();
+
+                if (preferred_cpu >= 0)
+                        return preferred_cpu;
+        }
+#endif
+        return this_cpu;
+}
+
+/*
+ * With HIGHRES=y we do not migrate the timer when it is expiring
+ * before the next event on the target cpu because we cannot reprogram
+ * the target cpu hardware and we would cause it to fire late.
+ *
+ * Called with cpu_base->lock of target cpu held.
+ */
+static int
+hrtimer_check_target(struct hrtimer *timer, struct hrtimer_clock_base *new_base)
+{
+#ifdef CONFIG_HIGH_RES_TIMERS
+        ktime_t expires;
+
+        if (!new_base->cpu_base->hres_active)
+                return 0;
+
+        expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset);
+        return expires.tv64 <= new_base->cpu_base->expires_next.tv64;
+#else
+        return 0;
+#endif
+}
+
 /*
  * Switch the timer base to the current CPU when possible.
  */
@@ -200,16 +240,8 @@ switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base,
 {
         struct hrtimer_clock_base *new_base;
         struct hrtimer_cpu_base *new_cpu_base;
-        int cpu, preferred_cpu = -1;
-
-        cpu = smp_processor_id();
-#if defined(CONFIG_NO_HZ) && defined(CONFIG_SMP)
-        if (!pinned && get_sysctl_timer_migration() && idle_cpu(cpu)) {
-                preferred_cpu = get_nohz_load_balancer();
-                if (preferred_cpu >= 0)
-                        cpu = preferred_cpu;
-        }
-#endif
+        int this_cpu = smp_processor_id();
+        int cpu = hrtimer_get_target(this_cpu, pinned);
 
 again:
         new_cpu_base = &per_cpu(hrtimer_bases, cpu);
@@ -217,7 +249,7 @@ again:
 
         if (base != new_base) {
                 /*
-                 * We are trying to schedule the timer on the local CPU.
+                 * We are trying to move timer to new_base.
                  * However we can't change timer's base while it is running,
                  * so we keep it on the same CPU. No hassle vs. reprogramming
                  * the event source in the high resolution case. The softirq
@@ -233,38 +265,12 @@ again:
                 spin_unlock(&base->cpu_base->lock);
                 spin_lock(&new_base->cpu_base->lock);
 
-                /* Optimized away for NOHZ=n SMP=n */
-                if (cpu == preferred_cpu) {
-                        /* Calculate clock monotonic expiry time */
-#ifdef CONFIG_HIGH_RES_TIMERS
-                        ktime_t expires = ktime_sub(hrtimer_get_expires(timer),
-                                                        new_base->offset);
-#else
-                        ktime_t expires = hrtimer_get_expires(timer);
-#endif
-
-                        /*
-                         * Get the next event on target cpu from the
-                         * clock events layer.
-                         * This covers the highres=off nohz=on case as well.
-                         */
-                        ktime_t next = clockevents_get_next_event(cpu);
-
-                        ktime_t delta = ktime_sub(expires, next);
-
-                        /*
-                         * We do not migrate the timer when it is expiring
-                         * before the next event on the target cpu because
-                         * we cannot reprogram the target cpu hardware and
-                         * we would cause it to fire late.
-                         */
-                        if (delta.tv64 < 0) {
-                                cpu = smp_processor_id();
-                                spin_unlock(&new_base->cpu_base->lock);
-                                spin_lock(&base->cpu_base->lock);
-                                timer->base = base;
-                                goto again;
-                        }
+                if (cpu != this_cpu && hrtimer_check_target(timer, new_base)) {
+                        cpu = this_cpu;
+                        spin_unlock(&new_base->cpu_base->lock);
+                        spin_lock(&base->cpu_base->lock);
+                        timer->base = base;
+                        goto again;
                 }
                 timer->base = new_base;
         }
@@ -380,6 +386,8 @@ ktime_t ktime_add_safe(const ktime_t lhs, const ktime_t rhs)
         return res;
 }
 
+EXPORT_SYMBOL_GPL(ktime_add_safe);
+
 #ifdef CONFIG_DEBUG_OBJECTS_TIMERS
 
 static struct debug_obj_descr hrtimer_debug_descr;
@@ -477,6 +485,7 @@ void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t clock_id,
         debug_object_init_on_stack(timer, &hrtimer_debug_descr);
         __hrtimer_init(timer, clock_id, mode);
 }
+EXPORT_SYMBOL_GPL(hrtimer_init_on_stack);
 
 void destroy_hrtimer_on_stack(struct hrtimer *timer)
 {
@@ -1274,14 +1283,22 @@ void hrtimer_interrupt(struct clock_event_device *dev)
 
         expires_next.tv64 = KTIME_MAX;
 
+        spin_lock(&cpu_base->lock);
+        /*
+         * We set expires_next to KTIME_MAX here with cpu_base->lock
+         * held to prevent that a timer is enqueued in our queue via
+         * the migration code. This does not affect enqueueing of
+         * timers which run their callback and need to be requeued on
+         * this CPU.
+         */
+        cpu_base->expires_next.tv64 = KTIME_MAX;
+
         base = cpu_base->clock_base;
 
         for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
                 ktime_t basenow;
                 struct rb_node *node;
 
-                spin_lock(&cpu_base->lock);
-
                 basenow = ktime_add(now, base->offset);
 
                 while ((node = base->first)) {
@@ -1314,11 +1331,15 @@ void hrtimer_interrupt(struct clock_event_device *dev)
 
                         __run_hrtimer(timer);
                 }
-                spin_unlock(&cpu_base->lock);
                 base++;
         }
 
+        /*
+         * Store the new expiry value so the migration code can verify
+         * against it.
+         */
         cpu_base->expires_next = expires_next;
+        spin_unlock(&cpu_base->lock);
 
         /* Reprogramming necessary ? */
         if (expires_next.tv64 != KTIME_MAX) {
@@ -1457,6 +1478,7 @@ void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task)
         sl->timer.function = hrtimer_wakeup;
         sl->task = task;
 }
+EXPORT_SYMBOL_GPL(hrtimer_init_sleeper);
 
 static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode)
 {
diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c
index 13c68e71b726..c1660194d115 100644
--- a/kernel/irq/chip.c
+++ b/kernel/irq/chip.c
@@ -222,6 +222,34 @@ int set_irq_chip_data(unsigned int irq, void *data)
 }
 EXPORT_SYMBOL(set_irq_chip_data);
 
+/**
+ *      set_irq_nested_thread - Set/Reset the IRQ_NESTED_THREAD flag of an irq
+ *
+ *      @irq:   Interrupt number
+ *      @nest:  0 to clear / 1 to set the IRQ_NESTED_THREAD flag
+ *
+ *      The IRQ_NESTED_THREAD flag indicates that on
+ *      request_threaded_irq() no separate interrupt thread should be
+ *      created for the irq as the handler are called nested in the
+ *      context of a demultiplexing interrupt handler thread.
+ */
+void set_irq_nested_thread(unsigned int irq, int nest)
+{
+        struct irq_desc *desc = irq_to_desc(irq);
+        unsigned long flags;
+
+        if (!desc)
+                return;
+
+        spin_lock_irqsave(&desc->lock, flags);
+        if (nest)
+                desc->status |= IRQ_NESTED_THREAD;
+        else
+                desc->status &= ~IRQ_NESTED_THREAD;
+        spin_unlock_irqrestore(&desc->lock, flags);
+}
+EXPORT_SYMBOL_GPL(set_irq_nested_thread);
+
 /*
  * default enable function
  */
@@ -299,6 +327,45 @@ static inline void mask_ack_irq(struct irq_desc *desc, int irq)
         }
 }
 
+/*
+ *      handle_nested_irq - Handle a nested irq from a irq thread
+ *      @irq:   the interrupt number
+ *
+ *      Handle interrupts which are nested into a threaded interrupt
+ *      handler. The handler function is called inside the calling
+ *      threads context.
+ */
+void handle_nested_irq(unsigned int irq)
+{
+        struct irq_desc *desc = irq_to_desc(irq);
+        struct irqaction *action;
+        irqreturn_t action_ret;
+
+        might_sleep();
+
+        spin_lock_irq(&desc->lock);
+
+        kstat_incr_irqs_this_cpu(irq, desc);
+
+        action = desc->action;
+        if (unlikely(!action || (desc->status & IRQ_DISABLED)))
+                goto out_unlock;
+
+        desc->status |= IRQ_INPROGRESS;
+        spin_unlock_irq(&desc->lock);
+
+        action_ret = action->thread_fn(action->irq, action->dev_id);
+        if (!noirqdebug)
+                note_interrupt(irq, desc, action_ret);
+
+        spin_lock_irq(&desc->lock);
+        desc->status &= ~IRQ_INPROGRESS;
+
+out_unlock:
+        spin_unlock_irq(&desc->lock);
+}
+EXPORT_SYMBOL_GPL(handle_nested_irq);
+
 /**
  *      handle_simple_irq - Simple and software-decoded IRQs.
  *      @irq:   the interrupt number
@@ -382,7 +449,10 @@ handle_level_irq(unsigned int irq, struct irq_desc *desc)
 
         spin_lock(&desc->lock);
         desc->status &= ~IRQ_INPROGRESS;
-        if (!(desc->status & IRQ_DISABLED) && desc->chip->unmask)
+
+        if (unlikely(desc->status & IRQ_ONESHOT))
+                desc->status |= IRQ_MASKED;
+        else if (!(desc->status & IRQ_DISABLED) && desc->chip->unmask)
                 desc->chip->unmask(irq);
 out_unlock:
         spin_unlock(&desc->lock);
@@ -572,6 +642,7 @@ __set_irq_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
                 desc->chip = &dummy_irq_chip;
         }
 
+        chip_bus_lock(irq, desc);
         spin_lock_irqsave(&desc->lock, flags);
 
         /* Uninstall? */
@@ -591,6 +662,7 @@ __set_irq_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
                 desc->chip->startup(irq);
         }
         spin_unlock_irqrestore(&desc->lock, flags);
+        chip_bus_sync_unlock(irq, desc);
 }
 EXPORT_SYMBOL_GPL(__set_irq_handler);
 
diff --git a/kernel/irq/handle.c b/kernel/irq/handle.c
index 065205bdd920..a81cf80554db 100644
--- a/kernel/irq/handle.c
+++ b/kernel/irq/handle.c
@@ -161,7 +161,7 @@ int __init early_irq_init(void)
 
         desc = irq_desc_legacy;
         legacy_count = ARRAY_SIZE(irq_desc_legacy);
-        node = first_online_node;
+        node = first_online_node;
 
         /* allocate irq_desc_ptrs array based on nr_irqs */
         irq_desc_ptrs = kcalloc(nr_irqs, sizeof(void *), GFP_NOWAIT);
@@ -172,6 +172,9 @@ int __init early_irq_init(void)
 
         for (i = 0; i < legacy_count; i++) {
                 desc[i].irq = i;
+#ifdef CONFIG_SMP
+                desc[i].node = node;
+#endif
                 desc[i].kstat_irqs = kstat_irqs_legacy + i * nr_cpu_ids;
                 lockdep_set_class(&desc[i].lock, &irq_desc_lock_class);
                 alloc_desc_masks(&desc[i], node, true);
diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h
index 73468253143b..1b5d742c6a77 100644
--- a/kernel/irq/internals.h
+++ b/kernel/irq/internals.h
@@ -42,8 +42,20 @@ static inline void unregister_handler_proc(unsigned int irq,
 
 extern int irq_select_affinity_usr(unsigned int irq);
 
-extern void
-irq_set_thread_affinity(struct irq_desc *desc, const struct cpumask *cpumask);
+extern void irq_set_thread_affinity(struct irq_desc *desc);
+
+/* Inline functions for support of irq chips on slow busses */
+static inline void chip_bus_lock(unsigned int irq, struct irq_desc *desc)
+{
+        if (unlikely(desc->chip->bus_lock))
+                desc->chip->bus_lock(irq);
+}
+
+static inline void chip_bus_sync_unlock(unsigned int irq, struct irq_desc *desc)
+{
+        if (unlikely(desc->chip->bus_sync_unlock))
+                desc->chip->bus_sync_unlock(irq);
+}
 
 /*
  * Debugging printout:
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c
index aaf5c9d05770..bde4c667d24d 100644
--- a/kernel/irq/manage.c
+++ b/kernel/irq/manage.c
@@ -80,14 +80,22 @@ int irq_can_set_affinity(unsigned int irq)
         return 1;
 }
 
-void
-irq_set_thread_affinity(struct irq_desc *desc, const struct cpumask *cpumask)
+/**
+ *      irq_set_thread_affinity - Notify irq threads to adjust affinity
+ *      @desc:          irq descriptor which has affitnity changed
+ *
+ *      We just set IRQTF_AFFINITY and delegate the affinity setting
+ *      to the interrupt thread itself. We can not call
+ *      set_cpus_allowed_ptr() here as we hold desc->lock and this
+ *      code can be called from hard interrupt context.
+ */
+void irq_set_thread_affinity(struct irq_desc *desc)
 {
         struct irqaction *action = desc->action;
 
         while (action) {
                 if (action->thread)
-                        set_cpus_allowed_ptr(action->thread, cpumask);
+                        set_bit(IRQTF_AFFINITY, &action->thread_flags);
                 action = action->next;
         }
 }
@@ -112,7 +120,7 @@ int irq_set_affinity(unsigned int irq, const struct cpumask *cpumask)
         if (desc->status & IRQ_MOVE_PCNTXT) {
                 if (!desc->chip->set_affinity(irq, cpumask)) {
                         cpumask_copy(desc->affinity, cpumask);
-                        irq_set_thread_affinity(desc, cpumask);
+                        irq_set_thread_affinity(desc);
                 }
         }
         else {
@@ -122,7 +130,7 @@ int irq_set_affinity(unsigned int irq, const struct cpumask *cpumask)
 #else
         if (!desc->chip->set_affinity(irq, cpumask)) {
                 cpumask_copy(desc->affinity, cpumask);
-                irq_set_thread_affinity(desc, cpumask);
+                irq_set_thread_affinity(desc);
         }
 #endif
         desc->status |= IRQ_AFFINITY_SET;
@@ -176,7 +184,7 @@ int irq_select_affinity_usr(unsigned int irq)
         spin_lock_irqsave(&desc->lock, flags);
         ret = setup_affinity(irq, desc);
         if (!ret)
-                irq_set_thread_affinity(desc, desc->affinity);
+                irq_set_thread_affinity(desc);
         spin_unlock_irqrestore(&desc->lock, flags);
 
         return ret;
@@ -222,9 +230,11 @@ void disable_irq_nosync(unsigned int irq)
         if (!desc)
                 return;
 
+        chip_bus_lock(irq, desc);
         spin_lock_irqsave(&desc->lock, flags);
         __disable_irq(desc, irq, false);
         spin_unlock_irqrestore(&desc->lock, flags);
+        chip_bus_sync_unlock(irq, desc);
 }
 EXPORT_SYMBOL(disable_irq_nosync);
 
@@ -286,7 +296,8 @@ void __enable_irq(struct irq_desc *desc, unsigned int irq, bool resume)
  *      matches the last disable, processing of interrupts on this
  *      IRQ line is re-enabled.
  *
- *      This function may be called from IRQ context.
+ *      This function may be called from IRQ context only when
+ *      desc->chip->bus_lock and desc->chip->bus_sync_unlock are NULL !
  */
 void enable_irq(unsigned int irq)
 {
@@ -296,9 +307,11 @@ void enable_irq(unsigned int irq)
         if (!desc)
                 return;
 
+        chip_bus_lock(irq, desc);
         spin_lock_irqsave(&desc->lock, flags);
         __enable_irq(desc, irq, false);
         spin_unlock_irqrestore(&desc->lock, flags);
+        chip_bus_sync_unlock(irq, desc);
 }
 EXPORT_SYMBOL(enable_irq);
 
@@ -428,6 +441,26 @@ int __irq_set_trigger(struct irq_desc *desc, unsigned int irq,
         return ret;
 }
 
+/*
+ * Default primary interrupt handler for threaded interrupts. Is
+ * assigned as primary handler when request_threaded_irq is called
+ * with handler == NULL. Useful for oneshot interrupts.
+ */
+static irqreturn_t irq_default_primary_handler(int irq, void *dev_id)
+{
+        return IRQ_WAKE_THREAD;
+}
+
+/*
+ * Primary handler for nested threaded interrupts. Should never be
+ * called.
+ */
+static irqreturn_t irq_nested_primary_handler(int irq, void *dev_id)
+{
+        WARN(1, "Primary handler called for nested irq %d\n", irq);
+        return IRQ_NONE;
+}
+
 static int irq_wait_for_interrupt(struct irqaction *action)
 {
         while (!kthread_should_stop()) {
@@ -444,6 +477,56 @@ static int irq_wait_for_interrupt(struct irqaction *action)
 }
 
 /*
+ * Oneshot interrupts keep the irq line masked until the threaded
+ * handler finished. unmask if the interrupt has not been disabled and
+ * is marked MASKED.
+ */
+static void irq_finalize_oneshot(unsigned int irq, struct irq_desc *desc)
+{
+        chip_bus_lock(irq, desc);
+        spin_lock_irq(&desc->lock);
+        if (!(desc->status & IRQ_DISABLED) && (desc->status & IRQ_MASKED)) {
+                desc->status &= ~IRQ_MASKED;
+                desc->chip->unmask(irq);
+        }
+        spin_unlock_irq(&desc->lock);
+        chip_bus_sync_unlock(irq, desc);
+}
+
+#ifdef CONFIG_SMP
+/*
+ * Check whether we need to change the affinity of the interrupt thread.
+ */
+static void
+irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action)
+{
+        cpumask_var_t mask;
+
+        if (!test_and_clear_bit(IRQTF_AFFINITY, &action->thread_flags))
+                return;
+
+        /*
+         * In case we are out of memory we set IRQTF_AFFINITY again and
+         * try again next time
+         */
+        if (!alloc_cpumask_var(&mask, GFP_KERNEL)) {
+                set_bit(IRQTF_AFFINITY, &action->thread_flags);
+                return;
+        }
+
+        spin_lock_irq(&desc->lock);
+        cpumask_copy(mask, desc->affinity);
+        spin_unlock_irq(&desc->lock);
+
+        set_cpus_allowed_ptr(current, mask);
+        free_cpumask_var(mask);
+}
+#else
+static inline void
+irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action) { }
+#endif
+
+/*
  * Interrupt handler thread
  */
 static int irq_thread(void *data)
@@ -451,13 +534,15 @@ static int irq_thread(void *data)
         struct sched_param param = { .sched_priority = MAX_USER_RT_PRIO/2, };
         struct irqaction *action = data;
         struct irq_desc *desc = irq_to_desc(action->irq);
-        int wake;
+        int wake, oneshot = desc->status & IRQ_ONESHOT;
 
         sched_setscheduler(current, SCHED_FIFO, &param);
         current->irqaction = action;
 
         while (!irq_wait_for_interrupt(action)) {
 
+                irq_thread_check_affinity(desc, action);
+
                 atomic_inc(&desc->threads_active);
 
                 spin_lock_irq(&desc->lock);
@@ -475,6 +560,9 @@ static int irq_thread(void *data)
                         spin_unlock_irq(&desc->lock);
 
                         action->thread_fn(action->irq, action->dev_id);
+
+                        if (oneshot)
+                                irq_finalize_oneshot(action->irq, desc);
                 }
 
                 wake = atomic_dec_and_test(&desc->threads_active);
@@ -522,7 +610,7 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
         struct irqaction *old, **old_ptr;
         const char *old_name = NULL;
         unsigned long flags;
-        int shared = 0;
+        int nested, shared = 0;
         int ret;
 
         if (!desc)
@@ -547,10 +635,32 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
                 rand_initialize_irq(irq);
         }
 
+        /* Oneshot interrupts are not allowed with shared */
+        if ((new->flags & IRQF_ONESHOT) && (new->flags & IRQF_SHARED))
+                return -EINVAL;
+
+        /*
+         * Check whether the interrupt nests into another interrupt
+         * thread.
+         */
+        nested = desc->status & IRQ_NESTED_THREAD;
+        if (nested) {
+                if (!new->thread_fn)
+                        return -EINVAL;
+                /*
+                 * Replace the primary handler which was provided from
+                 * the driver for non nested interrupt handling by the
+                 * dummy function which warns when called.
+                 */
+                new->handler = irq_nested_primary_handler;
+        }
+
         /*
-         * Threaded handler ?
+         * Create a handler thread when a thread function is supplied
+         * and the interrupt does not nest into another interrupt
+         * thread.
          */
-        if (new->thread_fn) {
+        if (new->thread_fn && !nested) {
                 struct task_struct *t;
 
                 t = kthread_create(irq_thread, new, "irq/%d-%s", irq,
@@ -564,7 +674,6 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
                  */
                 get_task_struct(t);
                 new->thread = t;
-                wake_up_process(t);
         }
 
         /*
@@ -620,9 +729,12 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
                         desc->status |= IRQ_PER_CPU;
 #endif
 
-                desc->status &= ~(IRQ_AUTODETECT | IRQ_WAITING |
+                desc->status &= ~(IRQ_AUTODETECT | IRQ_WAITING | IRQ_ONESHOT |
                                   IRQ_INPROGRESS | IRQ_SPURIOUS_DISABLED);
 
+                if (new->flags & IRQF_ONESHOT)
+                        desc->status |= IRQ_ONESHOT;
+
                 if (!(desc->status & IRQ_NOAUTOEN)) {
                         desc->depth = 0;
                         desc->status &= ~IRQ_DISABLED;
@@ -647,6 +759,7 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
                                 (int)(new->flags & IRQF_TRIGGER_MASK));
         }
 
+        new->irq = irq;
         *old_ptr = new;
 
         /* Reset broken irq detection when installing new handler */
@@ -664,7 +777,13 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
 
         spin_unlock_irqrestore(&desc->lock, flags);
 
-        new->irq = irq;
+        /*
+         * Strictly no need to wake it up, but hung_task complains
+         * when no hard interrupt wakes the thread up.
+         */
+        if (new->thread)
+                wake_up_process(new->thread);
+
         register_irq_proc(irq, desc);
         new->dir = NULL;
         register_handler_proc(irq, new);
@@ -718,7 +837,6 @@ static struct irqaction *__free_irq(unsigned int irq, void *dev_id)
 {
         struct irq_desc *desc = irq_to_desc(irq);
         struct irqaction *action, **action_ptr;
-        struct task_struct *irqthread;
         unsigned long flags;
 
         WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
@@ -766,9 +884,6 @@ static struct irqaction *__free_irq(unsigned int irq, void *dev_id)
                         desc->chip->disable(irq);
         }
 
-        irqthread = action->thread;
-        action->thread = NULL;
-
         spin_unlock_irqrestore(&desc->lock, flags);
 
         unregister_handler_proc(irq, action);
@@ -776,12 +891,6 @@ static struct irqaction *__free_irq(unsigned int irq, void *dev_id)
         /* Make sure it's not being used on another CPU: */
         synchronize_irq(irq);
 
-        if (irqthread) {
-                if (!test_bit(IRQTF_DIED, &action->thread_flags))
-                        kthread_stop(irqthread);
-                put_task_struct(irqthread);
-        }
-
 #ifdef CONFIG_DEBUG_SHIRQ
         /*
          * It's a shared IRQ -- the driver ought to be prepared for an IRQ
@@ -797,6 +906,13 @@ static struct irqaction *__free_irq(unsigned int irq, void *dev_id)
                 local_irq_restore(flags);
         }
 #endif
+
+        if (action->thread) {
+                if (!test_bit(IRQTF_DIED, &action->thread_flags))
+                        kthread_stop(action->thread);
+                put_task_struct(action->thread);
+        }
+
         return action;
 }
 
@@ -829,7 +945,14 @@ EXPORT_SYMBOL_GPL(remove_irq);
  */
 void free_irq(unsigned int irq, void *dev_id)
 {
+        struct irq_desc *desc = irq_to_desc(irq);
+
+        if (!desc)
+                return;
+
+        chip_bus_lock(irq, desc);
         kfree(__free_irq(irq, dev_id));
+        chip_bus_sync_unlock(irq, desc);
 }
 EXPORT_SYMBOL(free_irq);
 
@@ -838,6 +961,8 @@ EXPORT_SYMBOL(free_irq);
  *      @irq: Interrupt line to allocate
  *      @handler: Function to be called when the IRQ occurs.
  *                Primary handler for threaded interrupts
+ *                If NULL and thread_fn != NULL the default
+ *                primary handler is installed
  *      @thread_fn: Function called from the irq handler thread
  *                  If NULL, no irq thread is created
  *      @irqflags: Interrupt type flags
@@ -856,7 +981,7 @@ EXPORT_SYMBOL(free_irq);
  *      still called in hard interrupt context and has to check
  *      whether the interrupt originates from the device. If yes it
  *      needs to disable the interrupt on the device and return
- *      IRQ_THREAD_WAKE which will wake up the handler thread and run
+ *      IRQ_WAKE_THREAD which will wake up the handler thread and run
  *      @thread_fn. This split handler design is necessary to support
  *      shared interrupts.
  *
@@ -917,8 +1042,12 @@ int request_threaded_irq(unsigned int irq, irq_handler_t handler,
 
         if (desc->status & IRQ_NOREQUEST)
                 return -EINVAL;
-        if (!handler)
-                return -EINVAL;
+
+        if (!handler) {
+                if (!thread_fn)
+                        return -EINVAL;
+                handler = irq_default_primary_handler;
+        }
 
         action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
         if (!action)
@@ -930,7 +1059,10 @@ int request_threaded_irq(unsigned int irq, irq_handler_t handler,
         action->name = devname;
         action->dev_id = dev_id;
 
+        chip_bus_lock(irq, desc);
         retval = __setup_irq(irq, desc, action);
+        chip_bus_sync_unlock(irq, desc);
+
         if (retval)
                 kfree(action);
 
diff --git a/kernel/irq/migration.c b/kernel/irq/migration.c
index cfe767ca1545..fcb6c96f2627 100644
--- a/kernel/irq/migration.c
+++ b/kernel/irq/migration.c
@@ -45,7 +45,7 @@ void move_masked_irq(int irq)
                    < nr_cpu_ids))
                 if (!desc->chip->set_affinity(irq, desc->pending_mask)) {
                         cpumask_copy(desc->affinity, desc->pending_mask);
-                        irq_set_thread_affinity(desc, desc->pending_mask);
+                        irq_set_thread_affinity(desc);
                 }
 
         cpumask_clear(desc->pending_mask);
diff --git a/kernel/irq/numa_migrate.c b/kernel/irq/numa_migrate.c
index 2f69bee57bf2..3fd30197da2e 100644
--- a/kernel/irq/numa_migrate.c
+++ b/kernel/irq/numa_migrate.c
@@ -107,8 +107,8 @@ out_unlock:
 
 struct irq_desc *move_irq_desc(struct irq_desc *desc, int node)
 {
-        /* those all static, do move them */
-        if (desc->irq < NR_IRQS_LEGACY)
+        /* those static or target node is -1, do not move them */
+        if (desc->irq < NR_IRQS_LEGACY || node == -1)
                 return desc;
 
         if (desc->node != node)
diff --git a/kernel/irq/pm.c b/kernel/irq/pm.c
index 638d8bedec14..a0bb09e79867 100644
--- a/kernel/irq/pm.c
+++ b/kernel/irq/pm.c
@@ -15,10 +15,10 @@
 /**
  * suspend_device_irqs - disable all currently enabled interrupt lines
  *
- * During system-wide suspend or hibernation device interrupts need to be
- * disabled at the chip level and this function is provided for this purpose.
- * It disables all interrupt lines that are enabled at the moment and sets the
- * IRQ_SUSPENDED flag for them.
+ * During system-wide suspend or hibernation device drivers need to be prevented
+ * from receiving interrupts and this function is provided for this purpose.
+ * It marks all interrupt lines in use, except for the timer ones, as disabled
+ * and sets the IRQ_SUSPENDED flag for each of them.
  */
 void suspend_device_irqs(void)
 {
diff --git a/kernel/irq/resend.c b/kernel/irq/resend.c
index 89c7117acf2b..090c3763f3a2 100644
--- a/kernel/irq/resend.c
+++ b/kernel/irq/resend.c
@@ -70,8 +70,7 @@ void check_irq_resend(struct irq_desc *desc, unsigned int irq)
         if ((status & (IRQ_LEVEL | IRQ_PENDING | IRQ_REPLAY)) == IRQ_PENDING) {
                 desc->status = (status & ~IRQ_PENDING) | IRQ_REPLAY;
 
-                if (!desc->chip || !desc->chip->retrigger ||
-                                        !desc->chip->retrigger(irq)) {
+                if (!desc->chip->retrigger || !desc->chip->retrigger(irq)) {
 #ifdef CONFIG_HARDIRQS_SW_RESEND
                         /* Set it pending and activate the softirq: */
                         set_bit(irq, irqs_resend);
diff --git a/kernel/irq/spurious.c b/kernel/irq/spurious.c
index 4d568294de3e..114e704760fe 100644
--- a/kernel/irq/spurious.c
+++ b/kernel/irq/spurious.c
@@ -297,7 +297,6 @@ static int __init irqfixup_setup(char *str)
 
 __setup("irqfixup", irqfixup_setup);
 module_param(irqfixup, int, 0644);
-MODULE_PARM_DESC("irqfixup", "0: No fixup, 1: irqfixup mode, 2: irqpoll mode");
 
 static int __init irqpoll_setup(char *str)
 {
diff --git a/kernel/kexec.c b/kernel/kexec.c
index ae1c35201cc8..f336e2107f98 100644
--- a/kernel/kexec.c
+++ b/kernel/kexec.c
@@ -1228,7 +1228,7 @@ static int __init parse_crashkernel_mem(char 			*cmdline,
         } while (*cur++ == ',');
 
         if (*crash_size > 0) {
-                while (*cur != ' ' && *cur != '@')
+                while (*cur && *cur != ' ' && *cur != '@')
                         cur++;
                 if (*cur == '@') {
                         cur++;
diff --git a/kernel/kfifo.c b/kernel/kfifo.c
index bc41ad0f24f8..26539e3228e5 100644
--- a/kernel/kfifo.c
+++ b/kernel/kfifo.c
@@ -72,9 +72,9 @@ struct kfifo *kfifo_alloc(unsigned int size, gfp_t gfp_mask, spinlock_t *lock)
 
         /*
          * round up to the next power of 2, since our 'let the indices
-         * wrap' tachnique works only in this case.
+         * wrap' technique works only in this case.
          */
-        if (size & (size - 1)) {
+        if (!is_power_of_2(size)) {
                 BUG_ON(size > 0x80000000);
                 size = roundup_pow_of_two(size);
         }
diff --git a/kernel/kmod.c b/kernel/kmod.c
index 7e95bedb2bfc..9fcb53a11f87 100644
--- a/kernel/kmod.c
+++ b/kernel/kmod.c
@@ -24,7 +24,6 @@
 #include <linux/unistd.h>
 #include <linux/kmod.h>
 #include <linux/slab.h>
-#include <linux/mnt_namespace.h>
 #include <linux/completion.h>
 #include <linux/file.h>
 #include <linux/fdtable.h>
@@ -38,6 +37,8 @@
 #include <linux/suspend.h>
 #include <asm/uaccess.h>
 
+#include <trace/events/module.h>
+
 extern int max_threads;
 
 static struct workqueue_struct *khelper_wq;
@@ -79,6 +80,10 @@ int __request_module(bool wait, const char *fmt, ...)
 #define MAX_KMOD_CONCURRENT 50  /* Completely arbitrary value - KAO */
         static int kmod_loop_msg;
 
+        ret = security_kernel_module_request();
+        if (ret)
+                return ret;
+
         va_start(args, fmt);
         ret = vsnprintf(module_name, MODULE_NAME_LEN, fmt, args);
         va_end(args);
@@ -109,6 +114,8 @@ int __request_module(bool wait, const char *fmt, ...)
                 return -ENOMEM;
         }
 
+        trace_module_request(module_name, wait, _RET_IP_);
+
         ret = call_usermodehelper(modprobe_path, argv, envp,
                         wait ? UMH_WAIT_PROC : UMH_WAIT_EXEC);
         atomic_dec(&kmod_concurrent);
@@ -463,6 +470,7 @@ int call_usermodehelper_exec(struct subprocess_info *sub_info,
         int retval = 0;
 
         BUG_ON(atomic_read(&sub_info->cred->usage) != 1);
+        validate_creds(sub_info->cred);
 
         helper_lock();
         if (sub_info->path[0] == '\0')
diff --git a/kernel/kprobes.c b/kernel/kprobes.c
index c0fa54b276d9..ef177d653b2c 100644
--- a/kernel/kprobes.c
+++ b/kernel/kprobes.c
@@ -103,7 +103,7 @@ static struct kprobe_blackpoint kprobe_blacklist[] = {
 #define INSNS_PER_PAGE  (PAGE_SIZE/(MAX_INSN_SIZE * sizeof(kprobe_opcode_t)))
 
 struct kprobe_insn_page {
-        struct hlist_node hlist;
+        struct list_head list;
         kprobe_opcode_t *insns;         /* Page of instruction slots */
         char slot_used[INSNS_PER_PAGE];
         int nused;
@@ -117,7 +117,7 @@ enum kprobe_slot_state {
 };
 
 static DEFINE_MUTEX(kprobe_insn_mutex); /* Protects kprobe_insn_pages */
-static struct hlist_head kprobe_insn_pages;
+static LIST_HEAD(kprobe_insn_pages);
 static int kprobe_garbage_slots;
 static int collect_garbage_slots(void);
 
@@ -152,10 +152,9 @@ loop_end:
 static kprobe_opcode_t __kprobes *__get_insn_slot(void)
 {
         struct kprobe_insn_page *kip;
-        struct hlist_node *pos;
 
  retry:
-        hlist_for_each_entry(kip, pos, &kprobe_insn_pages, hlist) {
+        list_for_each_entry(kip, &kprobe_insn_pages, list) {
                 if (kip->nused < INSNS_PER_PAGE) {
                         int i;
                         for (i = 0; i < INSNS_PER_PAGE; i++) {
@@ -189,8 +188,8 @@ static kprobe_opcode_t __kprobes *__get_insn_slot(void)
                 kfree(kip);
                 return NULL;
         }
-        INIT_HLIST_NODE(&kip->hlist);
-        hlist_add_head(&kip->hlist, &kprobe_insn_pages);
+        INIT_LIST_HEAD(&kip->list);
+        list_add(&kip->list, &kprobe_insn_pages);
         memset(kip->slot_used, SLOT_CLEAN, INSNS_PER_PAGE);
         kip->slot_used[0] = SLOT_USED;
         kip->nused = 1;
@@ -219,12 +218,8 @@ static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx)
                  * so as not to have to set it up again the
                  * next time somebody inserts a probe.
                  */
-                hlist_del(&kip->hlist);
-                if (hlist_empty(&kprobe_insn_pages)) {
-                        INIT_HLIST_NODE(&kip->hlist);
-                        hlist_add_head(&kip->hlist,
-                                       &kprobe_insn_pages);
-                } else {
+                if (!list_is_singular(&kprobe_insn_pages)) {
+                        list_del(&kip->list);
                         module_free(NULL, kip->insns);
                         kfree(kip);
                 }
@@ -235,18 +230,13 @@ static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx)
 
 static int __kprobes collect_garbage_slots(void)
 {
-        struct kprobe_insn_page *kip;
-        struct hlist_node *pos, *next;
-        int safety;
+        struct kprobe_insn_page *kip, *next;
 
         /* Ensure no-one is preepmted on the garbages */
-        mutex_unlock(&kprobe_insn_mutex);
-        safety = check_safety();
-        mutex_lock(&kprobe_insn_mutex);
-        if (safety != 0)
+        if (check_safety())
                 return -EAGAIN;
 
-        hlist_for_each_entry_safe(kip, pos, next, &kprobe_insn_pages, hlist) {
+        list_for_each_entry_safe(kip, next, &kprobe_insn_pages, list) {
                 int i;
                 if (kip->ngarbage == 0)
                         continue;
@@ -264,19 +254,17 @@ static int __kprobes collect_garbage_slots(void)
 void __kprobes free_insn_slot(kprobe_opcode_t * slot, int dirty)
 {
         struct kprobe_insn_page *kip;
-        struct hlist_node *pos;
 
         mutex_lock(&kprobe_insn_mutex);
-        hlist_for_each_entry(kip, pos, &kprobe_insn_pages, hlist) {
+        list_for_each_entry(kip, &kprobe_insn_pages, list) {
                 if (kip->insns <= slot &&
                     slot < kip->insns + (INSNS_PER_PAGE * MAX_INSN_SIZE)) {
                         int i = (slot - kip->insns) / MAX_INSN_SIZE;
                         if (dirty) {
                                 kip->slot_used[i] = SLOT_DIRTY;
                                 kip->ngarbage++;
-                        } else {
+                        } else
                                 collect_one_slot(kip, i);
-                        }
                         break;
                 }
         }
@@ -698,7 +686,7 @@ int __kprobes register_kprobe(struct kprobe *p)
         p->addr = addr;
 
         preempt_disable();
-        if (!__kernel_text_address((unsigned long) p->addr) ||
+        if (!kernel_text_address((unsigned long) p->addr) ||
             in_kprobes_functions((unsigned long) p->addr)) {
                 preempt_enable();
                 return -EINVAL;
diff --git a/kernel/kthread.c b/kernel/kthread.c
index 41c88fe40500..5fe709982caa 100644
--- a/kernel/kthread.c
+++ b/kernel/kthread.c
@@ -9,14 +9,13 @@
 #include <linux/kthread.h>
 #include <linux/completion.h>
 #include <linux/err.h>
+#include <linux/cpuset.h>
 #include <linux/unistd.h>
 #include <linux/file.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <trace/events/sched.h>
 
-#define KTHREAD_NICE_LEVEL (-5)
-
 static DEFINE_SPINLOCK(kthread_create_lock);
 static LIST_HEAD(kthread_create_list);
 struct task_struct *kthreadd_task;
@@ -26,7 +25,6 @@ struct kthread_create_info
         /* Information passed to kthread() from kthreadd. */
         int (*threadfn)(void *data);
         void *data;
-        struct completion started;
 
         /* Result passed back to kthread_create() from kthreadd. */
         struct task_struct *result;
@@ -35,17 +33,13 @@ struct kthread_create_info
         struct list_head list;
 };
 
-struct kthread_stop_info
-{
-        struct task_struct *k;
-        int err;
-        struct completion done;
+struct kthread {
+        int should_stop;
+        struct completion exited;
 };
 
-/* Thread stopping is done by setthing this var: lock serializes
- * multiple kthread_stop calls. */
-static DEFINE_MUTEX(kthread_stop_lock);
-static struct kthread_stop_info kthread_stop_info;
+#define to_kthread(tsk) \
+        container_of((tsk)->vfork_done, struct kthread, exited)
 
 /**
  * kthread_should_stop - should this kthread return now?
@@ -56,36 +50,35 @@ static struct kthread_stop_info kthread_stop_info;
  */
 int kthread_should_stop(void)
 {
-        return (kthread_stop_info.k == current);
+        return to_kthread(current)->should_stop;
 }
 EXPORT_SYMBOL(kthread_should_stop);
 
 static int kthread(void *_create)
 {
+        /* Copy data: it's on kthread's stack */
         struct kthread_create_info *create = _create;
-        int (*threadfn)(void *data);
-        void *data;
-        int ret = -EINTR;
+        int (*threadfn)(void *data) = create->threadfn;
+        void *data = create->data;
+        struct kthread self;
+        int ret;
 
-        /* Copy data: it's on kthread's stack */
-        threadfn = create->threadfn;
-        data = create->data;
+        self.should_stop = 0;
+        init_completion(&self.exited);
+        current->vfork_done = &self.exited;
 
         /* OK, tell user we're spawned, wait for stop or wakeup */
         __set_current_state(TASK_UNINTERRUPTIBLE);
         create->result = current;
-        complete(&create->started);
+        complete(&create->done);
         schedule();
 
-        if (!kthread_should_stop())
+        ret = -EINTR;
+        if (!self.should_stop)
                 ret = threadfn(data);
 
-        /* It might have exited on its own, w/o kthread_stop.  Check. */
-        if (kthread_should_stop()) {
-                kthread_stop_info.err = ret;
-                complete(&kthread_stop_info.done);
-        }
-        return 0;
+        /* we can't just return, we must preserve "self" on stack */
+        do_exit(ret);
 }
 
 static void create_kthread(struct kthread_create_info *create)
@@ -94,11 +87,10 @@ static void create_kthread(struct kthread_create_info *create)
 
         /* We want our own signal handler (we take no signals by default). */
         pid = kernel_thread(kthread, create, CLONE_FS | CLONE_FILES | SIGCHLD);
-        if (pid < 0)
+        if (pid < 0) {
                 create->result = ERR_PTR(pid);
-        else
-                wait_for_completion(&create->started);
-        complete(&create->done);
+                complete(&create->done);
+        }
 }
 
 /**
@@ -129,7 +121,6 @@ struct task_struct *kthread_create(int (*threadfn)(void *data),
 
         create.threadfn = threadfn;
         create.data = data;
-        init_completion(&create.started);
         init_completion(&create.done);
 
         spin_lock(&kthread_create_lock);
@@ -152,7 +143,6 @@ struct task_struct *kthread_create(int (*threadfn)(void *data),
                  * The kernel thread should not inherit these properties.
                  */
                 sched_setscheduler_nocheck(create.result, SCHED_NORMAL, &param);
-                set_user_nice(create.result, KTHREAD_NICE_LEVEL);
                 set_cpus_allowed_ptr(create.result, cpu_all_mask);
         }
         return create.result;
@@ -187,40 +177,34 @@ EXPORT_SYMBOL(kthread_bind);
  * @k: thread created by kthread_create().
  *
  * Sets kthread_should_stop() for @k to return true, wakes it, and
- * waits for it to exit.  Your threadfn() must not call do_exit()
- * itself if you use this function!  This can also be called after
- * kthread_create() instead of calling wake_up_process(): the thread
- * will exit without calling threadfn().
+ * waits for it to exit. This can also be called after kthread_create()
+ * instead of calling wake_up_process(): the thread will exit without
+ * calling threadfn().
+ *
+ * If threadfn() may call do_exit() itself, the caller must ensure
+ * task_struct can't go away.
  *
  * Returns the result of threadfn(), or %-EINTR if wake_up_process()
  * was never called.
  */
 int kthread_stop(struct task_struct *k)
 {
+        struct kthread *kthread;
         int ret;
 
-        mutex_lock(&kthread_stop_lock);
-
-        /* It could exit after stop_info.k set, but before wake_up_process. */
-        get_task_struct(k);
-
         trace_sched_kthread_stop(k);
+        get_task_struct(k);
 
-        /* Must init completion *before* thread sees kthread_stop_info.k */
-        init_completion(&kthread_stop_info.done);
-        smp_wmb();
+        kthread = to_kthread(k);
+        barrier(); /* it might have exited */
+        if (k->vfork_done != NULL) {
+                kthread->should_stop = 1;
+                wake_up_process(k);
+                wait_for_completion(&kthread->exited);
+        }
+        ret = k->exit_code;
 
-        /* Now set kthread_should_stop() to true, and wake it up. */
-        kthread_stop_info.k = k;
-        wake_up_process(k);
         put_task_struct(k);
-
-        /* Once it dies, reset stop ptr, gather result and we're done. */
-        wait_for_completion(&kthread_stop_info.done);
-        kthread_stop_info.k = NULL;
-        ret = kthread_stop_info.err;
-        mutex_unlock(&kthread_stop_lock);
-
         trace_sched_kthread_stop_ret(ret);
 
         return ret;
@@ -234,8 +218,8 @@ int kthreadd(void *unused)
         /* Setup a clean context for our children to inherit. */
         set_task_comm(tsk, "kthreadd");
         ignore_signals(tsk);
-        set_user_nice(tsk, KTHREAD_NICE_LEVEL);
         set_cpus_allowed_ptr(tsk, cpu_all_mask);
+        set_mems_allowed(node_possible_map);
 
         current->flags |= PF_NOFREEZE | PF_FREEZER_NOSIG;
 
diff --git a/kernel/lockdep.c b/kernel/lockdep.c
index 8bbeef996c76..f74d2d7aa605 100644
--- a/kernel/lockdep.c
+++ b/kernel/lockdep.c
@@ -42,6 +42,7 @@
 #include <linux/hash.h>
 #include <linux/ftrace.h>
 #include <linux/stringify.h>
+#include <linux/bitops.h>
 
 #include <asm/sections.h>
 
@@ -366,11 +367,21 @@ static int save_trace(struct stack_trace *trace)
 
         save_stack_trace(trace);
 
+        /*
+         * Some daft arches put -1 at the end to indicate its a full trace.
+         *
+         * <rant> this is buggy anyway, since it takes a whole extra entry so a
+         * complete trace that maxes out the entries provided will be reported
+         * as incomplete, friggin useless </rant>
+         */
+        if (trace->entries[trace->nr_entries-1] == ULONG_MAX)
+                trace->nr_entries--;
+
         trace->max_entries = trace->nr_entries;
 
         nr_stack_trace_entries += trace->nr_entries;
 
-        if (nr_stack_trace_entries == MAX_STACK_TRACE_ENTRIES) {
+        if (nr_stack_trace_entries >= MAX_STACK_TRACE_ENTRIES-1) {
                 if (!debug_locks_off_graph_unlock())
                         return 0;
 
@@ -388,20 +399,6 @@ unsigned int nr_hardirq_chains;
 unsigned int nr_softirq_chains;
 unsigned int nr_process_chains;
 unsigned int max_lockdep_depth;
-unsigned int max_recursion_depth;
-
-static unsigned int lockdep_dependency_gen_id;
-
-static bool lockdep_dependency_visit(struct lock_class *source,
-                                     unsigned int depth)
-{
-        if (!depth)
-                lockdep_dependency_gen_id++;
-        if (source->dep_gen_id == lockdep_dependency_gen_id)
-                return true;
-        source->dep_gen_id = lockdep_dependency_gen_id;
-        return false;
-}
 
 #ifdef CONFIG_DEBUG_LOCKDEP
 /*
@@ -431,11 +428,8 @@ atomic_t redundant_softirqs_on;
 atomic_t redundant_softirqs_off;
 atomic_t nr_unused_locks;
 atomic_t nr_cyclic_checks;
-atomic_t nr_cyclic_check_recursions;
 atomic_t nr_find_usage_forwards_checks;
-atomic_t nr_find_usage_forwards_recursions;
 atomic_t nr_find_usage_backwards_checks;
-atomic_t nr_find_usage_backwards_recursions;
 #endif
 
 /*
@@ -551,58 +545,6 @@ static void lockdep_print_held_locks(struct task_struct *curr)
         }
 }
 
-static void print_lock_class_header(struct lock_class *class, int depth)
-{
-        int bit;
-
-        printk("%*s->", depth, "");
-        print_lock_name(class);
-        printk(" ops: %lu", class->ops);
-        printk(" {\n");
-
-        for (bit = 0; bit < LOCK_USAGE_STATES; bit++) {
-                if (class->usage_mask & (1 << bit)) {
-                        int len = depth;
-
-                        len += printk("%*s   %s", depth, "", usage_str[bit]);
-                        len += printk(" at:\n");
-                        print_stack_trace(class->usage_traces + bit, len);
-                }
-        }
-        printk("%*s }\n", depth, "");
-
-        printk("%*s ... key      at: ",depth,"");
-        print_ip_sym((unsigned long)class->key);
-}
-
-/*
- * printk all lock dependencies starting at <entry>:
- */
-static void __used
-print_lock_dependencies(struct lock_class *class, int depth)
-{
-        struct lock_list *entry;
-
-        if (lockdep_dependency_visit(class, depth))
-                return;
-
-        if (DEBUG_LOCKS_WARN_ON(depth >= 20))
-                return;
-
-        print_lock_class_header(class, depth);
-
-        list_for_each_entry(entry, &class->locks_after, entry) {
-                if (DEBUG_LOCKS_WARN_ON(!entry->class))
-                        return;
-
-                print_lock_dependencies(entry->class, depth + 1);
-
-                printk("%*s ... acquired at:\n",depth,"");
-                print_stack_trace(&entry->trace, 2);
-                printk("\n");
-        }
-}
-
 static void print_kernel_version(void)
 {
         printk("%s %.*s\n", init_utsname()->release,
@@ -898,22 +840,203 @@ static int add_lock_to_list(struct lock_class *class, struct lock_class *this,
 }
 
 /*
+ * For good efficiency of modular, we use power of 2
+ */
+#define MAX_CIRCULAR_QUEUE_SIZE         4096UL
+#define CQ_MASK                         (MAX_CIRCULAR_QUEUE_SIZE-1)
+
+/*
+ * The circular_queue and helpers is used to implement the
+ * breadth-first search(BFS)algorithem, by which we can build
+ * the shortest path from the next lock to be acquired to the
+ * previous held lock if there is a circular between them.
+ */
+struct circular_queue {
+        unsigned long element[MAX_CIRCULAR_QUEUE_SIZE];
+        unsigned int  front, rear;
+};
+
+static struct circular_queue lock_cq;
+
+unsigned int max_bfs_queue_depth;
+
+static unsigned int lockdep_dependency_gen_id;
+
+static inline void __cq_init(struct circular_queue *cq)
+{
+        cq->front = cq->rear = 0;
+        lockdep_dependency_gen_id++;
+}
+
+static inline int __cq_empty(struct circular_queue *cq)
+{
+        return (cq->front == cq->rear);
+}
+
+static inline int __cq_full(struct circular_queue *cq)
+{
+        return ((cq->rear + 1) & CQ_MASK) == cq->front;
+}
+
+static inline int __cq_enqueue(struct circular_queue *cq, unsigned long elem)
+{
+        if (__cq_full(cq))
+                return -1;
+
+        cq->element[cq->rear] = elem;
+        cq->rear = (cq->rear + 1) & CQ_MASK;
+        return 0;
+}
+
+static inline int __cq_dequeue(struct circular_queue *cq, unsigned long *elem)
+{
+        if (__cq_empty(cq))
+                return -1;
+
+        *elem = cq->element[cq->front];
+        cq->front = (cq->front + 1) & CQ_MASK;
+        return 0;
+}
+
+static inline unsigned int  __cq_get_elem_count(struct circular_queue *cq)
+{
+        return (cq->rear - cq->front) & CQ_MASK;
+}
+
+static inline void mark_lock_accessed(struct lock_list *lock,
+                                        struct lock_list *parent)
+{
+        unsigned long nr;
+
+        nr = lock - list_entries;
+        WARN_ON(nr >= nr_list_entries);
+        lock->parent = parent;
+        lock->class->dep_gen_id = lockdep_dependency_gen_id;
+}
+
+static inline unsigned long lock_accessed(struct lock_list *lock)
+{
+        unsigned long nr;
+
+        nr = lock - list_entries;
+        WARN_ON(nr >= nr_list_entries);
+        return lock->class->dep_gen_id == lockdep_dependency_gen_id;
+}
+
+static inline struct lock_list *get_lock_parent(struct lock_list *child)
+{
+        return child->parent;
+}
+
+static inline int get_lock_depth(struct lock_list *child)
+{
+        int depth = 0;
+        struct lock_list *parent;
+
+        while ((parent = get_lock_parent(child))) {
+                child = parent;
+                depth++;
+        }
+        return depth;
+}
+
+static int __bfs(struct lock_list *source_entry,
+                 void *data,
+                 int (*match)(struct lock_list *entry, void *data),
+                 struct lock_list **target_entry,
+                 int forward)
+{
+        struct lock_list *entry;
+        struct list_head *head;
+        struct circular_queue *cq = &lock_cq;
+        int ret = 1;
+
+        if (match(source_entry, data)) {
+                *target_entry = source_entry;
+                ret = 0;
+                goto exit;
+        }
+
+        if (forward)
+                head = &source_entry->class->locks_after;
+        else
+                head = &source_entry->class->locks_before;
+
+        if (list_empty(head))
+                goto exit;
+
+        __cq_init(cq);
+        __cq_enqueue(cq, (unsigned long)source_entry);
+
+        while (!__cq_empty(cq)) {
+                struct lock_list *lock;
+
+                __cq_dequeue(cq, (unsigned long *)&lock);
+
+                if (!lock->class) {
+                        ret = -2;
+                        goto exit;
+                }
+
+                if (forward)
+                        head = &lock->class->locks_after;
+                else
+                        head = &lock->class->locks_before;
+
+                list_for_each_entry(entry, head, entry) {
+                        if (!lock_accessed(entry)) {
+                                unsigned int cq_depth;
+                                mark_lock_accessed(entry, lock);
+                                if (match(entry, data)) {
+                                        *target_entry = entry;
+                                        ret = 0;
+                                        goto exit;
+                                }
+
+                                if (__cq_enqueue(cq, (unsigned long)entry)) {
+                                        ret = -1;
+                                        goto exit;
+                                }
+                                cq_depth = __cq_get_elem_count(cq);
+                                if (max_bfs_queue_depth < cq_depth)
+                                        max_bfs_queue_depth = cq_depth;
+                        }
+                }
+        }
+exit:
+        return ret;
+}
+
+static inline int __bfs_forwards(struct lock_list *src_entry,
+                        void *data,
+                        int (*match)(struct lock_list *entry, void *data),
+                        struct lock_list **target_entry)
+{
+        return __bfs(src_entry, data, match, target_entry, 1);
+
+}
+
+static inline int __bfs_backwards(struct lock_list *src_entry,
+                        void *data,
+                        int (*match)(struct lock_list *entry, void *data),
+                        struct lock_list **target_entry)
+{
+        return __bfs(src_entry, data, match, target_entry, 0);
+
+}
+
+/*
  * Recursive, forwards-direction lock-dependency checking, used for
  * both noncyclic checking and for hardirq-unsafe/softirq-unsafe
  * checking.
- *
- * (to keep the stackframe of the recursive functions small we
- *  use these global variables, and we also mark various helper
- *  functions as noinline.)
  */
-static struct held_lock *check_source, *check_target;
 
 /*
  * Print a dependency chain entry (this is only done when a deadlock
  * has been detected):
  */
 static noinline int
-print_circular_bug_entry(struct lock_list *target, unsigned int depth)
+print_circular_bug_entry(struct lock_list *target, int depth)
 {
         if (debug_locks_silent)
                 return 0;
@@ -930,11 +1053,13 @@ print_circular_bug_entry(struct lock_list *target, unsigned int depth)
  * header first:
  */
 static noinline int
-print_circular_bug_header(struct lock_list *entry, unsigned int depth)
+print_circular_bug_header(struct lock_list *entry, unsigned int depth,
+                        struct held_lock *check_src,
+                        struct held_lock *check_tgt)
 {
         struct task_struct *curr = current;
 
-        if (!debug_locks_off_graph_unlock() || debug_locks_silent)
+        if (debug_locks_silent)
                 return 0;
 
         printk("\n=======================================================\n");
@@ -943,9 +1068,9 @@ print_circular_bug_header(struct lock_list *entry, unsigned int depth)
         printk(  "-------------------------------------------------------\n");
         printk("%s/%d is trying to acquire lock:\n",
                 curr->comm, task_pid_nr(curr));
-        print_lock(check_source);
+        print_lock(check_src);
         printk("\nbut task is already holding lock:\n");
-        print_lock(check_target);
+        print_lock(check_tgt);
         printk("\nwhich lock already depends on the new lock.\n\n");
         printk("\nthe existing dependency chain (in reverse order) is:\n");
 
@@ -954,19 +1079,36 @@ print_circular_bug_header(struct lock_list *entry, unsigned int depth)
         return 0;
 }
 
-static noinline int print_circular_bug_tail(void)
+static inline int class_equal(struct lock_list *entry, void *data)
+{
+        return entry->class == data;
+}
+
+static noinline int print_circular_bug(struct lock_list *this,
+                                struct lock_list *target,
+                                struct held_lock *check_src,
+                                struct held_lock *check_tgt)
 {
         struct task_struct *curr = current;
-        struct lock_list this;
+        struct lock_list *parent;
+        int depth;
 
-        if (debug_locks_silent)
+        if (!debug_locks_off_graph_unlock() || debug_locks_silent)
                 return 0;
 
-        this.class = hlock_class(check_source);
-        if (!save_trace(&this.trace))
+        if (!save_trace(&this->trace))
                 return 0;
 
-        print_circular_bug_entry(&this, 0);
+        depth = get_lock_depth(target);
+
+        print_circular_bug_header(target, depth, check_src, check_tgt);
+
+        parent = get_lock_parent(target);
+
+        while (parent) {
+                print_circular_bug_entry(parent, --depth);
+                parent = get_lock_parent(parent);
+        }
 
         printk("\nother info that might help us debug this:\n\n");
         lockdep_print_held_locks(curr);
@@ -977,73 +1119,69 @@ static noinline int print_circular_bug_tail(void)
         return 0;
 }
 
-#define RECURSION_LIMIT 40
-
-static int noinline print_infinite_recursion_bug(void)
+static noinline int print_bfs_bug(int ret)
 {
         if (!debug_locks_off_graph_unlock())
                 return 0;
 
-        WARN_ON(1);
+        WARN(1, "lockdep bfs error:%d\n", ret);
 
         return 0;
 }
 
-unsigned long __lockdep_count_forward_deps(struct lock_class *class,
-                                           unsigned int depth)
+static int noop_count(struct lock_list *entry, void *data)
 {
-        struct lock_list *entry;
-        unsigned long ret = 1;
+        (*(unsigned long *)data)++;
+        return 0;
+}
 
-        if (lockdep_dependency_visit(class, depth))
-                return 0;
+unsigned long __lockdep_count_forward_deps(struct lock_list *this)
+{
+        unsigned long  count = 0;
+        struct lock_list *uninitialized_var(target_entry);
 
-        /*
-         * Recurse this class's dependency list:
-         */
-        list_for_each_entry(entry, &class->locks_after, entry)
-                ret += __lockdep_count_forward_deps(entry->class, depth + 1);
+        __bfs_forwards(this, (void *)&count, noop_count, &target_entry);
 
-        return ret;
+        return count;
 }
-
 unsigned long lockdep_count_forward_deps(struct lock_class *class)
 {
         unsigned long ret, flags;
+        struct lock_list this;
+
+        this.parent = NULL;
+        this.class = class;
 
         local_irq_save(flags);
         __raw_spin_lock(&lockdep_lock);
-        ret = __lockdep_count_forward_deps(class, 0);
+        ret = __lockdep_count_forward_deps(&this);
         __raw_spin_unlock(&lockdep_lock);
         local_irq_restore(flags);
 
         return ret;
 }
 
-unsigned long __lockdep_count_backward_deps(struct lock_class *class,
-                                            unsigned int depth)
+unsigned long __lockdep_count_backward_deps(struct lock_list *this)
 {
-        struct lock_list *entry;
-        unsigned long ret = 1;
+        unsigned long  count = 0;
+        struct lock_list *uninitialized_var(target_entry);
 
-        if (lockdep_dependency_visit(class, depth))
-                return 0;
-        /*
-         * Recurse this class's dependency list:
-         */
-        list_for_each_entry(entry, &class->locks_before, entry)
-                ret += __lockdep_count_backward_deps(entry->class, depth + 1);
+        __bfs_backwards(this, (void *)&count, noop_count, &target_entry);
 
-        return ret;
+        return count;
 }
 
 unsigned long lockdep_count_backward_deps(struct lock_class *class)
 {
         unsigned long ret, flags;
+        struct lock_list this;
+
+        this.parent = NULL;
+        this.class = class;
 
         local_irq_save(flags);
         __raw_spin_lock(&lockdep_lock);
-        ret = __lockdep_count_backward_deps(class, 0);
+        ret = __lockdep_count_backward_deps(&this);
         __raw_spin_unlock(&lockdep_lock);
         local_irq_restore(flags);
 
@@ -1055,29 +1193,16 @@ unsigned long lockdep_count_backward_deps(struct lock_class *class)
  * lead to <target>. Print an error and return 0 if it does.
  */
 static noinline int
-check_noncircular(struct lock_class *source, unsigned int depth)
+check_noncircular(struct lock_list *root, struct lock_class *target,
+                struct lock_list **target_entry)
 {
-        struct lock_list *entry;
+        int result;
 
-        if (lockdep_dependency_visit(source, depth))
-                return 1;
+        debug_atomic_inc(&nr_cyclic_checks);
 
-        debug_atomic_inc(&nr_cyclic_check_recursions);
-        if (depth > max_recursion_depth)
-                max_recursion_depth = depth;
-        if (depth >= RECURSION_LIMIT)
-                return print_infinite_recursion_bug();
-        /*
-         * Check this lock's dependency list:
-         */
-        list_for_each_entry(entry, &source->locks_after, entry) {
-                if (entry->class == hlock_class(check_target))
-                        return print_circular_bug_header(entry, depth+1);
-                debug_atomic_inc(&nr_cyclic_checks);
-                if (!check_noncircular(entry->class, depth+1))
-                        return print_circular_bug_entry(entry, depth+1);
-        }
-        return 1;
+        result = __bfs_forwards(root, target, class_equal, target_entry);
+
+        return result;
 }
 
 #if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_PROVE_LOCKING)
@@ -1086,103 +1211,121 @@ check_noncircular(struct lock_class *source, unsigned int depth)
  * proving that two subgraphs can be connected by a new dependency
  * without creating any illegal irq-safe -> irq-unsafe lock dependency.
  */
-static enum lock_usage_bit find_usage_bit;
-static struct lock_class *forwards_match, *backwards_match;
+
+static inline int usage_match(struct lock_list *entry, void *bit)
+{
+        return entry->class->usage_mask & (1 << (enum lock_usage_bit)bit);
+}
+
+
 
 /*
  * Find a node in the forwards-direction dependency sub-graph starting
- * at <source> that matches <find_usage_bit>.
+ * at @root->class that matches @bit.
  *
- * Return 2 if such a node exists in the subgraph, and put that node
- * into <forwards_match>.
+ * Return 0 if such a node exists in the subgraph, and put that node
+ * into *@target_entry.
  *
- * Return 1 otherwise and keep <forwards_match> unchanged.
- * Return 0 on error.
+ * Return 1 otherwise and keep *@target_entry unchanged.
+ * Return <0 on error.
  */
-static noinline int
-find_usage_forwards(struct lock_class *source, unsigned int depth)
+static int
+find_usage_forwards(struct lock_list *root, enum lock_usage_bit bit,
+                        struct lock_list **target_entry)
 {
-        struct lock_list *entry;
-        int ret;
-
-        if (lockdep_dependency_visit(source, depth))
-                return 1;
-
-        if (depth > max_recursion_depth)
-                max_recursion_depth = depth;
-        if (depth >= RECURSION_LIMIT)
-                return print_infinite_recursion_bug();
+        int result;
 
         debug_atomic_inc(&nr_find_usage_forwards_checks);
-        if (source->usage_mask & (1 << find_usage_bit)) {
-                forwards_match = source;
-                return 2;
-        }
 
-        /*
-         * Check this lock's dependency list:
-         */
-        list_for_each_entry(entry, &source->locks_after, entry) {
-                debug_atomic_inc(&nr_find_usage_forwards_recursions);
-                ret = find_usage_forwards(entry->class, depth+1);
-                if (ret == 2 || ret == 0)
-                        return ret;
-        }
-        return 1;
+        result = __bfs_forwards(root, (void *)bit, usage_match, target_entry);
+
+        return result;
 }
 
 /*
  * Find a node in the backwards-direction dependency sub-graph starting
- * at <source> that matches <find_usage_bit>.
+ * at @root->class that matches @bit.
  *
- * Return 2 if such a node exists in the subgraph, and put that node
- * into <backwards_match>.
+ * Return 0 if such a node exists in the subgraph, and put that node
+ * into *@target_entry.
  *
- * Return 1 otherwise and keep <backwards_match> unchanged.
- * Return 0 on error.
+ * Return 1 otherwise and keep *@target_entry unchanged.
+ * Return <0 on error.
  */
-static noinline int
-find_usage_backwards(struct lock_class *source, unsigned int depth)
+static int
+find_usage_backwards(struct lock_list *root, enum lock_usage_bit bit,
+                        struct lock_list **target_entry)
 {
-        struct lock_list *entry;
-        int ret;
+        int result;
 
-        if (lockdep_dependency_visit(source, depth))
-                return 1;
+        debug_atomic_inc(&nr_find_usage_backwards_checks);
 
-        if (!__raw_spin_is_locked(&lockdep_lock))
-                return DEBUG_LOCKS_WARN_ON(1);
+        result = __bfs_backwards(root, (void *)bit, usage_match, target_entry);
 
-        if (depth > max_recursion_depth)
-                max_recursion_depth = depth;
-        if (depth >= RECURSION_LIMIT)
-                return print_infinite_recursion_bug();
+        return result;
+}
 
-        debug_atomic_inc(&nr_find_usage_backwards_checks);
-        if (source->usage_mask & (1 << find_usage_bit)) {
-                backwards_match = source;
-                return 2;
-        }
+static void print_lock_class_header(struct lock_class *class, int depth)
+{
+        int bit;
 
-        if (!source && debug_locks_off_graph_unlock()) {
-                WARN_ON(1);
-                return 0;
-        }
+        printk("%*s->", depth, "");
+        print_lock_name(class);
+        printk(" ops: %lu", class->ops);
+        printk(" {\n");
 
-        /*
-         * Check this lock's dependency list:
-         */
-        list_for_each_entry(entry, &source->locks_before, entry) {
-                debug_atomic_inc(&nr_find_usage_backwards_recursions);
-                ret = find_usage_backwards(entry->class, depth+1);
-                if (ret == 2 || ret == 0)
-                        return ret;
+        for (bit = 0; bit < LOCK_USAGE_STATES; bit++) {
+                if (class->usage_mask & (1 << bit)) {
+                        int len = depth;
+
+                        len += printk("%*s   %s", depth, "", usage_str[bit]);
+                        len += printk(" at:\n");
+                        print_stack_trace(class->usage_traces + bit, len);
+                }
         }
-        return 1;
+        printk("%*s }\n", depth, "");
+
+        printk("%*s ... key      at: ",depth,"");
+        print_ip_sym((unsigned long)class->key);
+}
+
+/*
+ * printk the shortest lock dependencies from @start to @end in reverse order:
+ */
+static void __used
+print_shortest_lock_dependencies(struct lock_list *leaf,
+                                struct lock_list *root)
+{
+        struct lock_list *entry = leaf;
+        int depth;
+
+        /*compute depth from generated tree by BFS*/
+        depth = get_lock_depth(leaf);
+
+        do {
+                print_lock_class_header(entry->class, depth);
+                printk("%*s ... acquired at:\n", depth, "");
+                print_stack_trace(&entry->trace, 2);
+                printk("\n");
+
+                if (depth == 0 && (entry != root)) {
+                        printk("lockdep:%s bad BFS generated tree\n", __func__);
+                        break;
+                }
+
+                entry = get_lock_parent(entry);
+                depth--;
+        } while (entry && (depth >= 0));
+
+        return;
 }
 
 static int
 print_bad_irq_dependency(struct task_struct *curr,
+                         struct lock_list *prev_root,
+                         struct lock_list *next_root,
+                         struct lock_list *backwards_entry,
+                         struct lock_list *forwards_entry,
                          struct held_lock *prev,
                          struct held_lock *next,
                          enum lock_usage_bit bit1,
@@ -1215,26 +1358,32 @@ print_bad_irq_dependency(struct task_struct *curr,
 
         printk("\nbut this new dependency connects a %s-irq-safe lock:\n",
                 irqclass);
-        print_lock_name(backwards_match);
+        print_lock_name(backwards_entry->class);
         printk("\n... which became %s-irq-safe at:\n", irqclass);
 
-        print_stack_trace(backwards_match->usage_traces + bit1, 1);
+        print_stack_trace(backwards_entry->class->usage_traces + bit1, 1);
 
         printk("\nto a %s-irq-unsafe lock:\n", irqclass);
-        print_lock_name(forwards_match);
+        print_lock_name(forwards_entry->class);
         printk("\n... which became %s-irq-unsafe at:\n", irqclass);
         printk("...");
 
-        print_stack_trace(forwards_match->usage_traces + bit2, 1);
+        print_stack_trace(forwards_entry->class->usage_traces + bit2, 1);
 
         printk("\nother info that might help us debug this:\n\n");
         lockdep_print_held_locks(curr);
 
-        printk("\nthe %s-irq-safe lock's dependencies:\n", irqclass);
-        print_lock_dependencies(backwards_match, 0);
+        printk("\nthe dependencies between %s-irq-safe lock", irqclass);
+        printk(" and the holding lock:\n");
+        if (!save_trace(&prev_root->trace))
+                return 0;
+        print_shortest_lock_dependencies(backwards_entry, prev_root);
 
-        printk("\nthe %s-irq-unsafe lock's dependencies:\n", irqclass);
-        print_lock_dependencies(forwards_match, 0);
+        printk("\nthe dependencies between the lock to be acquired");
+        printk(" and %s-irq-unsafe lock:\n", irqclass);
+        if (!save_trace(&next_root->trace))
+                return 0;
+        print_shortest_lock_dependencies(forwards_entry, next_root);
 
         printk("\nstack backtrace:\n");
         dump_stack();
@@ -1248,19 +1397,30 @@ check_usage(struct task_struct *curr, struct held_lock *prev,
             enum lock_usage_bit bit_forwards, const char *irqclass)
 {
         int ret;
+        struct lock_list this, that;
+        struct lock_list *uninitialized_var(target_entry);
+        struct lock_list *uninitialized_var(target_entry1);
 
-        find_usage_bit = bit_backwards;
-        /* fills in <backwards_match> */
-        ret = find_usage_backwards(hlock_class(prev), 0);
-        if (!ret || ret == 1)
+        this.parent = NULL;
+
+        this.class = hlock_class(prev);
+        ret = find_usage_backwards(&this, bit_backwards, &target_entry);
+        if (ret < 0)
+                return print_bfs_bug(ret);
+        if (ret == 1)
                 return ret;
 
-        find_usage_bit = bit_forwards;
-        ret = find_usage_forwards(hlock_class(next), 0);
-        if (!ret || ret == 1)
+        that.parent = NULL;
+        that.class = hlock_class(next);
+        ret = find_usage_forwards(&that, bit_forwards, &target_entry1);
+        if (ret < 0)
+                return print_bfs_bug(ret);
+        if (ret == 1)
                 return ret;
-        /* ret == 2 */
-        return print_bad_irq_dependency(curr, prev, next,
+
+        return print_bad_irq_dependency(curr, &this, &that,
+                        target_entry, target_entry1,
+                        prev, next,
                         bit_backwards, bit_forwards, irqclass);
 }
 
@@ -1472,6 +1632,8 @@ check_prev_add(struct task_struct *curr, struct held_lock *prev,
 {
         struct lock_list *entry;
         int ret;
+        struct lock_list this;
+        struct lock_list *uninitialized_var(target_entry);
 
         /*
          * Prove that the new <prev> -> <next> dependency would not
@@ -1482,10 +1644,13 @@ check_prev_add(struct task_struct *curr, struct held_lock *prev,
          * We are using global variables to control the recursion, to
          * keep the stackframe size of the recursive functions low:
          */
-        check_source = next;
-        check_target = prev;
-        if (!(check_noncircular(hlock_class(next), 0)))
-                return print_circular_bug_tail();
+        this.class = hlock_class(next);
+        this.parent = NULL;
+        ret = check_noncircular(&this, hlock_class(prev), &target_entry);
+        if (unlikely(!ret))
+                return print_circular_bug(&this, target_entry, next, prev);
+        else if (unlikely(ret < 0))
+                return print_bfs_bug(ret);
 
         if (!check_prev_add_irq(curr, prev, next))
                 return 0;
@@ -1884,7 +2049,8 @@ static int mark_lock(struct task_struct *curr, struct held_lock *this,
  * print irq inversion bug:
  */
 static int
-print_irq_inversion_bug(struct task_struct *curr, struct lock_class *other,
+print_irq_inversion_bug(struct task_struct *curr,
+                        struct lock_list *root, struct lock_list *other,
                         struct held_lock *this, int forwards,
                         const char *irqclass)
 {
@@ -1902,17 +2068,16 @@ print_irq_inversion_bug(struct task_struct *curr, struct lock_class *other,
                 printk("but this lock took another, %s-unsafe lock in the past:\n", irqclass);
         else
                 printk("but this lock was taken by another, %s-safe lock in the past:\n", irqclass);
-        print_lock_name(other);
+        print_lock_name(other->class);
         printk("\n\nand interrupts could create inverse lock ordering between them.\n\n");
 
         printk("\nother info that might help us debug this:\n");
         lockdep_print_held_locks(curr);
 
-        printk("\nthe first lock's dependencies:\n");
-        print_lock_dependencies(hlock_class(this), 0);
-
-        printk("\nthe second lock's dependencies:\n");
-        print_lock_dependencies(other, 0);
+        printk("\nthe shortest dependencies between 2nd lock and 1st lock:\n");
+        if (!save_trace(&root->trace))
+                return 0;
+        print_shortest_lock_dependencies(other, root);
 
         printk("\nstack backtrace:\n");
         dump_stack();
@@ -1929,14 +2094,19 @@ check_usage_forwards(struct task_struct *curr, struct held_lock *this,
                      enum lock_usage_bit bit, const char *irqclass)
 {
         int ret;
-
-        find_usage_bit = bit;
-        /* fills in <forwards_match> */
-        ret = find_usage_forwards(hlock_class(this), 0);
-        if (!ret || ret == 1)
+        struct lock_list root;
+        struct lock_list *uninitialized_var(target_entry);
+
+        root.parent = NULL;
+        root.class = hlock_class(this);
+        ret = find_usage_forwards(&root, bit, &target_entry);
+        if (ret < 0)
+                return print_bfs_bug(ret);
+        if (ret == 1)
                 return ret;
 
-        return print_irq_inversion_bug(curr, forwards_match, this, 1, irqclass);
+        return print_irq_inversion_bug(curr, &root, target_entry,
+                                        this, 1, irqclass);
 }
 
 /*
@@ -1948,14 +2118,19 @@ check_usage_backwards(struct task_struct *curr, struct held_lock *this,
                       enum lock_usage_bit bit, const char *irqclass)
 {
         int ret;
-
-        find_usage_bit = bit;
-        /* fills in <backwards_match> */
-        ret = find_usage_backwards(hlock_class(this), 0);
-        if (!ret || ret == 1)
+        struct lock_list root;
+        struct lock_list *uninitialized_var(target_entry);
+
+        root.parent = NULL;
+        root.class = hlock_class(this);
+        ret = find_usage_backwards(&root, bit, &target_entry);
+        if (ret < 0)
+                return print_bfs_bug(ret);
+        if (ret == 1)
                 return ret;
 
-        return print_irq_inversion_bug(curr, backwards_match, this, 0, irqclass);
+        return print_irq_inversion_bug(curr, &root, target_entry,
+                                        this, 1, irqclass);
 }
 
 void print_irqtrace_events(struct task_struct *curr)
@@ -2530,13 +2705,15 @@ EXPORT_SYMBOL_GPL(lockdep_init_map);
  */
 static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
                           int trylock, int read, int check, int hardirqs_off,
-                          struct lockdep_map *nest_lock, unsigned long ip)
+                          struct lockdep_map *nest_lock, unsigned long ip,
+                          int references)
 {
         struct task_struct *curr = current;
         struct lock_class *class = NULL;
         struct held_lock *hlock;
         unsigned int depth, id;
         int chain_head = 0;
+        int class_idx;
         u64 chain_key;
 
         if (!prove_locking)
@@ -2584,10 +2761,24 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
         if (DEBUG_LOCKS_WARN_ON(depth >= MAX_LOCK_DEPTH))
                 return 0;
 
+        class_idx = class - lock_classes + 1;
+
+        if (depth) {
+                hlock = curr->held_locks + depth - 1;
+                if (hlock->class_idx == class_idx && nest_lock) {
+                        if (hlock->references)
+                                hlock->references++;
+                        else
+                                hlock->references = 2;
+
+                        return 1;
+                }
+        }
+
         hlock = curr->held_locks + depth;
         if (DEBUG_LOCKS_WARN_ON(!class))
                 return 0;
-        hlock->class_idx = class - lock_classes + 1;
+        hlock->class_idx = class_idx;
         hlock->acquire_ip = ip;
         hlock->instance = lock;
         hlock->nest_lock = nest_lock;
@@ -2595,6 +2786,7 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
         hlock->read = read;
         hlock->check = check;
         hlock->hardirqs_off = !!hardirqs_off;
+        hlock->references = references;
 #ifdef CONFIG_LOCK_STAT
         hlock->waittime_stamp = 0;
         hlock->holdtime_stamp = sched_clock();
@@ -2703,6 +2895,30 @@ static int check_unlock(struct task_struct *curr, struct lockdep_map *lock,
         return 1;
 }
 
+static int match_held_lock(struct held_lock *hlock, struct lockdep_map *lock)
+{
+        if (hlock->instance == lock)
+                return 1;
+
+        if (hlock->references) {
+                struct lock_class *class = lock->class_cache;
+
+                if (!class)
+                        class = look_up_lock_class(lock, 0);
+
+                if (DEBUG_LOCKS_WARN_ON(!class))
+                        return 0;
+
+                if (DEBUG_LOCKS_WARN_ON(!hlock->nest_lock))
+                        return 0;
+
+                if (hlock->class_idx == class - lock_classes + 1)
+                        return 1;
+        }
+
+        return 0;
+}
+
 static int
 __lock_set_class(struct lockdep_map *lock, const char *name,
                  struct lock_class_key *key, unsigned int subclass,
@@ -2726,7 +2942,7 @@ __lock_set_class(struct lockdep_map *lock, const char *name,
                  */
                 if (prev_hlock && prev_hlock->irq_context != hlock->irq_context)
                         break;
-                if (hlock->instance == lock)
+                if (match_held_lock(hlock, lock))
                         goto found_it;
                 prev_hlock = hlock;
         }
@@ -2745,7 +2961,8 @@ found_it:
                 if (!__lock_acquire(hlock->instance,
                         hlock_class(hlock)->subclass, hlock->trylock,
                                 hlock->read, hlock->check, hlock->hardirqs_off,
-                                hlock->nest_lock, hlock->acquire_ip))
+                                hlock->nest_lock, hlock->acquire_ip,
+                                hlock->references))
                         return 0;
         }
 
@@ -2784,20 +3001,34 @@ lock_release_non_nested(struct task_struct *curr,
                  */
                 if (prev_hlock && prev_hlock->irq_context != hlock->irq_context)
                         break;
-                if (hlock->instance == lock)
+                if (match_held_lock(hlock, lock))
                         goto found_it;
                 prev_hlock = hlock;
         }
         return print_unlock_inbalance_bug(curr, lock, ip);
 
 found_it:
-        lock_release_holdtime(hlock);
+        if (hlock->instance == lock)
+                lock_release_holdtime(hlock);
+
+        if (hlock->references) {
+                hlock->references--;
+                if (hlock->references) {
+                        /*
+                         * We had, and after removing one, still have
+                         * references, the current lock stack is still
+                         * valid. We're done!
+                         */
+                        return 1;
+                }
+        }
 
         /*
          * We have the right lock to unlock, 'hlock' points to it.
          * Now we remove it from the stack, and add back the other
          * entries (if any), recalculating the hash along the way:
          */
+
         curr->lockdep_depth = i;
         curr->curr_chain_key = hlock->prev_chain_key;
 
@@ -2806,7 +3037,8 @@ found_it:
                 if (!__lock_acquire(hlock->instance,
                         hlock_class(hlock)->subclass, hlock->trylock,
                                 hlock->read, hlock->check, hlock->hardirqs_off,
-                                hlock->nest_lock, hlock->acquire_ip))
+                                hlock->nest_lock, hlock->acquire_ip,
+                                hlock->references))
                         return 0;
         }
 
@@ -2836,7 +3068,7 @@ static int lock_release_nested(struct task_struct *curr,
         /*
          * Is the unlock non-nested:
          */
-        if (hlock->instance != lock)
+        if (hlock->instance != lock || hlock->references)
                 return lock_release_non_nested(curr, lock, ip);
         curr->lockdep_depth--;
 
@@ -2881,6 +3113,21 @@ __lock_release(struct lockdep_map *lock, int nested, unsigned long ip)
         check_chain_key(curr);
 }
 
+static int __lock_is_held(struct lockdep_map *lock)
+{
+        struct task_struct *curr = current;
+        int i;
+
+        for (i = 0; i < curr->lockdep_depth; i++) {
+                struct held_lock *hlock = curr->held_locks + i;
+
+                if (match_held_lock(hlock, lock))
+                        return 1;
+        }
+
+        return 0;
+}
+
 /*
  * Check whether we follow the irq-flags state precisely:
  */
@@ -2957,7 +3204,7 @@ void lock_acquire(struct lockdep_map *lock, unsigned int subclass,
 
         current->lockdep_recursion = 1;
         __lock_acquire(lock, subclass, trylock, read, check,
-                       irqs_disabled_flags(flags), nest_lock, ip);
+                       irqs_disabled_flags(flags), nest_lock, ip, 0);
         current->lockdep_recursion = 0;
         raw_local_irq_restore(flags);
 }
@@ -2982,6 +3229,26 @@ void lock_release(struct lockdep_map *lock, int nested,
 }
 EXPORT_SYMBOL_GPL(lock_release);
 
+int lock_is_held(struct lockdep_map *lock)
+{
+        unsigned long flags;
+        int ret = 0;
+
+        if (unlikely(current->lockdep_recursion))
+                return ret;
+
+        raw_local_irq_save(flags);
+        check_flags(flags);
+
+        current->lockdep_recursion = 1;
+        ret = __lock_is_held(lock);
+        current->lockdep_recursion = 0;
+        raw_local_irq_restore(flags);
+
+        return ret;
+}
+EXPORT_SYMBOL_GPL(lock_is_held);
+
 void lockdep_set_current_reclaim_state(gfp_t gfp_mask)
 {
         current->lockdep_reclaim_gfp = gfp_mask;
@@ -3041,7 +3308,7 @@ __lock_contended(struct lockdep_map *lock, unsigned long ip)
                  */
                 if (prev_hlock && prev_hlock->irq_context != hlock->irq_context)
                         break;
-                if (hlock->instance == lock)
+                if (match_held_lock(hlock, lock))
                         goto found_it;
                 prev_hlock = hlock;
         }
@@ -3049,6 +3316,9 @@ __lock_contended(struct lockdep_map *lock, unsigned long ip)
         return;
 
 found_it:
+        if (hlock->instance != lock)
+                return;
+
         hlock->waittime_stamp = sched_clock();
 
         contention_point = lock_point(hlock_class(hlock)->contention_point, ip);
@@ -3088,7 +3358,7 @@ __lock_acquired(struct lockdep_map *lock, unsigned long ip)
                  */
                 if (prev_hlock && prev_hlock->irq_context != hlock->irq_context)
                         break;
-                if (hlock->instance == lock)
+                if (match_held_lock(hlock, lock))
                         goto found_it;
                 prev_hlock = hlock;
         }
@@ -3096,6 +3366,9 @@ __lock_acquired(struct lockdep_map *lock, unsigned long ip)
         return;
 
 found_it:
+        if (hlock->instance != lock)
+                return;
+
         cpu = smp_processor_id();
         if (hlock->waittime_stamp) {
                 now = sched_clock();
@@ -3326,7 +3599,12 @@ void __init lockdep_info(void)
                 sizeof(struct list_head) * CLASSHASH_SIZE +
                 sizeof(struct lock_list) * MAX_LOCKDEP_ENTRIES +
                 sizeof(struct lock_chain) * MAX_LOCKDEP_CHAINS +
-                sizeof(struct list_head) * CHAINHASH_SIZE) / 1024);
+                sizeof(struct list_head) * CHAINHASH_SIZE
+#ifdef CONFIG_PROVE_LOCKING
+                + sizeof(struct circular_queue)
+#endif
+                ) / 1024
+                );
 
         printk(" per task-struct memory footprint: %lu bytes\n",
                 sizeof(struct held_lock) * MAX_LOCK_DEPTH);
diff --git a/kernel/lockdep_internals.h b/kernel/lockdep_internals.h
index 699a2ac3a0d7..a2ee95ad1313 100644
--- a/kernel/lockdep_internals.h
+++ b/kernel/lockdep_internals.h
@@ -91,6 +91,8 @@ extern unsigned int nr_process_chains;
 extern unsigned int max_lockdep_depth;
 extern unsigned int max_recursion_depth;
 
+extern unsigned int max_bfs_queue_depth;
+
 #ifdef CONFIG_PROVE_LOCKING
 extern unsigned long lockdep_count_forward_deps(struct lock_class *);
 extern unsigned long lockdep_count_backward_deps(struct lock_class *);
diff --git a/kernel/lockdep_proc.c b/kernel/lockdep_proc.c
index d7135aa2d2c4..d4b3dbc79fdb 100644
--- a/kernel/lockdep_proc.c
+++ b/kernel/lockdep_proc.c
@@ -25,38 +25,12 @@
 
 static void *l_next(struct seq_file *m, void *v, loff_t *pos)
 {
-        struct lock_class *class;
-
-        (*pos)++;
-
-        if (v == SEQ_START_TOKEN)
-                class = m->private;
-        else {
-                class = v;
-
-                if (class->lock_entry.next != &all_lock_classes)
-                        class = list_entry(class->lock_entry.next,
-                                           struct lock_class, lock_entry);
-                else
-                        class = NULL;
-        }
-
-        return class;
+        return seq_list_next(v, &all_lock_classes, pos);
 }
 
 static void *l_start(struct seq_file *m, loff_t *pos)
 {
-        struct lock_class *class;
-        loff_t i = 0;
-
-        if (*pos == 0)
-                return SEQ_START_TOKEN;
-
-        list_for_each_entry(class, &all_lock_classes, lock_entry) {
-                if (++i == *pos)
-                return class;
-        }
-        return NULL;
+        return seq_list_start_head(&all_lock_classes, *pos);
 }
 
 static void l_stop(struct seq_file *m, void *v)
@@ -82,11 +56,11 @@ static void print_name(struct seq_file *m, struct lock_class *class)
 
 static int l_show(struct seq_file *m, void *v)
 {
-        struct lock_class *class = v;
+        struct lock_class *class = list_entry(v, struct lock_class, lock_entry);
         struct lock_list *entry;
         char usage[LOCK_USAGE_CHARS];
 
-        if (v == SEQ_START_TOKEN) {
+        if (v == &all_lock_classes) {
                 seq_printf(m, "all lock classes:\n");
                 return 0;
         }
@@ -128,17 +102,7 @@ static const struct seq_operations lockdep_ops = {
 
 static int lockdep_open(struct inode *inode, struct file *file)
 {
-        int res = seq_open(file, &lockdep_ops);
-        if (!res) {
-                struct seq_file *m = file->private_data;
-
-                if (!list_empty(&all_lock_classes))
-                        m->private = list_entry(all_lock_classes.next,
-                                        struct lock_class, lock_entry);
-                else
-                        m->private = NULL;
-        }
-        return res;
+        return seq_open(file, &lockdep_ops);
 }
 
 static const struct file_operations proc_lockdep_operations = {
@@ -149,37 +113,23 @@ static const struct file_operations proc_lockdep_operations = {
 };
 
 #ifdef CONFIG_PROVE_LOCKING
-static void *lc_next(struct seq_file *m, void *v, loff_t *pos)
-{
-        struct lock_chain *chain;
-
-        (*pos)++;
-
-        if (v == SEQ_START_TOKEN)
-                chain = m->private;
-        else {
-                chain = v;
-
-                if (*pos < nr_lock_chains)
-                        chain = lock_chains + *pos;
-                else
-                        chain = NULL;
-        }
-
-        return chain;
-}
-
 static void *lc_start(struct seq_file *m, loff_t *pos)
 {
         if (*pos == 0)
                 return SEQ_START_TOKEN;
 
-        if (*pos < nr_lock_chains)
-                return lock_chains + *pos;
+        if (*pos - 1 < nr_lock_chains)
+                return lock_chains + (*pos - 1);
 
         return NULL;
 }
 
+static void *lc_next(struct seq_file *m, void *v, loff_t *pos)
+{
+        (*pos)++;
+        return lc_start(m, pos);
+}
+
 static void lc_stop(struct seq_file *m, void *v)
 {
 }
@@ -220,16 +170,7 @@ static const struct seq_operations lockdep_chains_ops = {
 
 static int lockdep_chains_open(struct inode *inode, struct file *file)
 {
-        int res = seq_open(file, &lockdep_chains_ops);
-        if (!res) {
-                struct seq_file *m = file->private_data;
-
-                if (nr_lock_chains)
-                        m->private = lock_chains;
-                else
-                        m->private = NULL;
-        }
-        return res;
+        return seq_open(file, &lockdep_chains_ops);
 }
 
 static const struct file_operations proc_lockdep_chains_operations = {
@@ -258,16 +199,10 @@ static void lockdep_stats_debug_show(struct seq_file *m)
                 debug_atomic_read(&chain_lookup_hits));
         seq_printf(m, " cyclic checks:                 %11u\n",
                 debug_atomic_read(&nr_cyclic_checks));
-        seq_printf(m, " cyclic-check recursions:       %11u\n",
-                debug_atomic_read(&nr_cyclic_check_recursions));
         seq_printf(m, " find-mask forwards checks:     %11u\n",
                 debug_atomic_read(&nr_find_usage_forwards_checks));
-        seq_printf(m, " find-mask forwards recursions: %11u\n",
-                debug_atomic_read(&nr_find_usage_forwards_recursions));
         seq_printf(m, " find-mask backwards checks:    %11u\n",
                 debug_atomic_read(&nr_find_usage_backwards_checks));
-        seq_printf(m, " find-mask backwards recursions:%11u\n",
-                debug_atomic_read(&nr_find_usage_backwards_recursions));
 
         seq_printf(m, " hardirq on events:             %11u\n", hi1);
         seq_printf(m, " hardirq off events:            %11u\n", hi2);
@@ -409,8 +344,10 @@ static int lockdep_stats_show(struct seq_file *m, void *v)
                         nr_unused);
         seq_printf(m, " max locking depth:             %11u\n",
                         max_lockdep_depth);
-        seq_printf(m, " max recursion depth:           %11u\n",
-                        max_recursion_depth);
+#ifdef CONFIG_PROVE_LOCKING
+        seq_printf(m, " max bfs queue depth:           %11u\n",
+                        max_bfs_queue_depth);
+#endif
         lockdep_stats_debug_show(m);
         seq_printf(m, " debug_locks:                   %11u\n",
                         debug_locks);
@@ -438,7 +375,6 @@ struct lock_stat_data {
 };
 
 struct lock_stat_seq {
-        struct lock_stat_data *iter;
         struct lock_stat_data *iter_end;
         struct lock_stat_data stats[MAX_LOCKDEP_KEYS];
 };
@@ -626,34 +562,22 @@ static void seq_header(struct seq_file *m)
 static void *ls_start(struct seq_file *m, loff_t *pos)
 {
         struct lock_stat_seq *data = m->private;
+        struct lock_stat_data *iter;
 
         if (*pos == 0)
                 return SEQ_START_TOKEN;
 
-        data->iter = data->stats + *pos;
-        if (data->iter >= data->iter_end)
-                data->iter = NULL;
+        iter = data->stats + (*pos - 1);
+        if (iter >= data->iter_end)
+                iter = NULL;
 
-        return data->iter;
+        return iter;
 }
 
 static void *ls_next(struct seq_file *m, void *v, loff_t *pos)
 {
-        struct lock_stat_seq *data = m->private;
-
         (*pos)++;
-
-        if (v == SEQ_START_TOKEN)
-                data->iter = data->stats;
-        else {
-                data->iter = v;
-                data->iter++;
-        }
-
-        if (data->iter == data->iter_end)
-                data->iter = NULL;
-
-        return data->iter;
+        return ls_start(m, pos);
 }
 
 static void ls_stop(struct seq_file *m, void *v)
@@ -691,7 +615,6 @@ static int lock_stat_open(struct inode *inode, struct file *file)
                 struct lock_stat_data *iter = data->stats;
                 struct seq_file *m = file->private_data;
 
-                data->iter = iter;
                 list_for_each_entry(class, &all_lock_classes, lock_entry) {
                         iter->class = class;
                         iter->stats = lock_stats(class);
@@ -699,7 +622,7 @@ static int lock_stat_open(struct inode *inode, struct file *file)
                 }
                 data->iter_end = iter;
 
-                sort(data->stats, data->iter_end - data->iter,
+                sort(data->stats, data->iter_end - data->stats,
                                 sizeof(struct lock_stat_data),
                                 lock_stat_cmp, NULL);
 
@@ -734,7 +657,6 @@ static int lock_stat_release(struct inode *inode, struct file *file)
         struct seq_file *seq = file->private_data;
 
         vfree(seq->private);
-        seq->private = NULL;
         return seq_release(inode, file);
 }
 
@@ -758,7 +680,8 @@ static int __init lockdep_proc_init(void)
                     &proc_lockdep_stats_operations);
 
 #ifdef CONFIG_LOCK_STAT
-        proc_create("lock_stat", S_IRUSR, NULL, &proc_lock_stat_operations);
+        proc_create("lock_stat", S_IRUSR | S_IWUSR, NULL,
+                    &proc_lock_stat_operations);
 #endif
 
         return 0;
diff --git a/kernel/module.c b/kernel/module.c
index 215aaab09e91..05ce49ced8f6 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -55,6 +55,11 @@
 #include <linux/percpu.h>
 #include <linux/kmemleak.h>
 
+#define CREATE_TRACE_POINTS
+#include <trace/events/module.h>
+
+EXPORT_TRACEPOINT_SYMBOL(module_get);
+
 #if 0
 #define DEBUGP printk
 #else
@@ -364,7 +369,7 @@ EXPORT_SYMBOL_GPL(find_module);
 
 #ifdef CONFIG_SMP
 
-#ifdef CONFIG_HAVE_DYNAMIC_PER_CPU_AREA
+#ifndef CONFIG_HAVE_LEGACY_PER_CPU_AREA
 
 static void *percpu_modalloc(unsigned long size, unsigned long align,
                              const char *name)
@@ -389,7 +394,7 @@ static void percpu_modfree(void *freeme)
         free_percpu(freeme);
 }
 
-#else /* ... !CONFIG_HAVE_DYNAMIC_PER_CPU_AREA */
+#else /* ... CONFIG_HAVE_LEGACY_PER_CPU_AREA */
 
 /* Number of blocks used and allocated. */
 static unsigned int pcpu_num_used, pcpu_num_allocated;
@@ -535,7 +540,7 @@ static int percpu_modinit(void)
 }
 __initcall(percpu_modinit);
 
-#endif /* CONFIG_HAVE_DYNAMIC_PER_CPU_AREA */
+#endif /* CONFIG_HAVE_LEGACY_PER_CPU_AREA */
 
 static unsigned int find_pcpusec(Elf_Ehdr *hdr,
                                  Elf_Shdr *sechdrs,
@@ -909,16 +914,18 @@ void __symbol_put(const char *symbol)
 }
 EXPORT_SYMBOL(__symbol_put);
 
+/* Note this assumes addr is a function, which it currently always is. */
 void symbol_put_addr(void *addr)
 {
         struct module *modaddr;
+        unsigned long a = (unsigned long)dereference_function_descriptor(addr);
 
-        if (core_kernel_text((unsigned long)addr))
+        if (core_kernel_text(a))
                 return;
 
         /* module_text_address is safe here: we're supposed to have reference
          * to module from symbol_get, so it can't go away. */
-        modaddr = __module_text_address((unsigned long)addr);
+        modaddr = __module_text_address(a);
         BUG_ON(!modaddr);
         module_put(modaddr);
 }
@@ -940,6 +947,8 @@ void module_put(struct module *module)
         if (module) {
                 unsigned int cpu = get_cpu();
                 local_dec(__module_ref_addr(module, cpu));
+                trace_module_put(module, _RET_IP_,
+                                 local_read(__module_ref_addr(module, cpu)));
                 /* Maybe they're waiting for us to drop reference? */
                 if (unlikely(!module_is_live(module)))
                         wake_up_process(module->waiter);
@@ -1068,7 +1077,8 @@ static inline int check_modstruct_version(Elf_Shdr *sechdrs,
 {
         const unsigned long *crc;
 
-        if (!find_symbol("module_layout", NULL, &crc, true, false))
+        if (!find_symbol(MODULE_SYMBOL_PREFIX "module_layout", NULL,
+                         &crc, true, false))
                 BUG();
         return check_version(sechdrs, versindex, "module_layout", mod, crc);
 }
@@ -1271,6 +1281,10 @@ static void add_notes_attrs(struct module *mod, unsigned int nsect,
         struct module_notes_attrs *notes_attrs;
         struct bin_attribute *nattr;
 
+        /* failed to create section attributes, so can't create notes */
+        if (!mod->sect_attrs)
+                return;
+
         /* Count notes sections and allocate structures.  */
         notes = 0;
         for (i = 0; i < nsect; i++)
@@ -1490,6 +1504,8 @@ static int __unlink_module(void *_mod)
 /* Free a module, remove from lists, etc (must hold module_mutex). */
 static void free_module(struct module *mod)
 {
+        trace_module_free(mod);
+
         /* Delete from various lists */
         stop_machine(__unlink_module, mod, NULL);
         remove_notes_attrs(mod);
@@ -2216,6 +2232,10 @@ static noinline struct module *load_module(void __user *umod,
         mod->unused_gpl_crcs = section_addr(hdr, sechdrs, secstrings,
                                             "__kcrctab_unused_gpl");
 #endif
+#ifdef CONFIG_CONSTRUCTORS
+        mod->ctors = section_objs(hdr, sechdrs, secstrings, ".ctors",
+                                  sizeof(*mod->ctors), &mod->num_ctors);
+#endif
 
 #ifdef CONFIG_MARKERS
         mod->markers = section_objs(hdr, sechdrs, secstrings, "__markers",
@@ -2353,6 +2373,8 @@ static noinline struct module *load_module(void __user *umod,
         /* Get rid of temporary copy */
         vfree(hdr);
 
+        trace_module_load(mod);
+
         /* Done! */
         return mod;
 
@@ -2389,6 +2411,17 @@ static noinline struct module *load_module(void __user *umod,
         goto free_hdr;
 }
 
+/* Call module constructors. */
+static void do_mod_ctors(struct module *mod)
+{
+#ifdef CONFIG_CONSTRUCTORS
+        unsigned long i;
+
+        for (i = 0; i < mod->num_ctors; i++)
+                mod->ctors[i]();
+#endif
+}
+
 /* This is where the real work happens */
 SYSCALL_DEFINE3(init_module, void __user *, umod,
                 unsigned long, len, const char __user *, uargs)
@@ -2417,6 +2450,7 @@ SYSCALL_DEFINE3(init_module, void __user *, umod,
         blocking_notifier_call_chain(&module_notify_list,
                         MODULE_STATE_COMING, mod);
 
+        do_mod_ctors(mod);
         /* Start the module */
         if (mod->init != NULL)
                 ret = do_one_initcall(mod->init);
@@ -2435,9 +2469,9 @@ SYSCALL_DEFINE3(init_module, void __user *, umod,
                 return ret;
         }
         if (ret > 0) {
-                printk(KERN_WARNING "%s: '%s'->init suspiciously returned %d, "
-                                    "it should follow 0/-E convention\n"
-                       KERN_WARNING "%s: loading module anyway...\n",
+                printk(KERN_WARNING
+"%s: '%s'->init suspiciously returned %d, it should follow 0/-E convention\n"
+"%s: loading module anyway...\n",
                        __func__, mod->name, ret,
                        __func__);
                 dump_stack();
diff --git a/kernel/nsproxy.c b/kernel/nsproxy.c
index 63598dca2d0c..09b4ff9711b2 100644
--- a/kernel/nsproxy.c
+++ b/kernel/nsproxy.c
@@ -26,19 +26,14 @@ static struct kmem_cache *nsproxy_cachep;
 
 struct nsproxy init_nsproxy = INIT_NSPROXY(init_nsproxy);
 
-/*
- * creates a copy of "orig" with refcount 1.
- */
-static inline struct nsproxy *clone_nsproxy(struct nsproxy *orig)
+static inline struct nsproxy *create_nsproxy(void)
 {
-        struct nsproxy *ns;
+        struct nsproxy *nsproxy;
 
-        ns = kmem_cache_alloc(nsproxy_cachep, GFP_KERNEL);
-        if (ns) {
-                memcpy(ns, orig, sizeof(struct nsproxy));
-                atomic_set(&ns->count, 1);
-        }
-        return ns;
+        nsproxy = kmem_cache_alloc(nsproxy_cachep, GFP_KERNEL);
+        if (nsproxy)
+                atomic_set(&nsproxy->count, 1);
+        return nsproxy;
 }
 
 /*
@@ -52,7 +47,7 @@ static struct nsproxy *create_new_namespaces(unsigned long flags,
         struct nsproxy *new_nsp;
         int err;
 
-        new_nsp = clone_nsproxy(tsk->nsproxy);
+        new_nsp = create_nsproxy();
         if (!new_nsp)
                 return ERR_PTR(-ENOMEM);
 
diff --git a/kernel/panic.c b/kernel/panic.c
index 984b3ecbd72c..512ab73b0ca3 100644
--- a/kernel/panic.c
+++ b/kernel/panic.c
@@ -301,6 +301,7 @@ int oops_may_print(void)
  */
 void oops_enter(void)
 {
+        tracing_off();
         /* can't trust the integrity of the kernel anymore: */
         debug_locks_off();
         do_oops_enter_exit();
diff --git a/kernel/perf_counter.c b/kernel/perf_counter.c
index 29b685f551aa..8cb94a52d1bb 100644
--- a/kernel/perf_counter.c
+++ b/kernel/perf_counter.c
@@ -42,14 +42,21 @@ static int perf_overcommit __read_mostly = 1;
 static atomic_t nr_counters __read_mostly;
 static atomic_t nr_mmap_counters __read_mostly;
 static atomic_t nr_comm_counters __read_mostly;
+static atomic_t nr_task_counters __read_mostly;
 
 /*
  * perf counter paranoia level:
- *  0 - not paranoid
- *  1 - disallow cpu counters to unpriv
- *  2 - disallow kernel profiling to unpriv
+ *  -1 - not paranoid at all
+ *   0 - disallow raw tracepoint access for unpriv
+ *   1 - disallow cpu counters for unpriv
+ *   2 - disallow kernel profiling for unpriv
  */
-int sysctl_perf_counter_paranoid __read_mostly;
+int sysctl_perf_counter_paranoid __read_mostly = 1;
+
+static inline bool perf_paranoid_tracepoint_raw(void)
+{
+        return sysctl_perf_counter_paranoid > -1;
+}
 
 static inline bool perf_paranoid_cpu(void)
 {
@@ -87,6 +94,7 @@ void __weak hw_perf_disable(void)		{ barrier(); }
 void __weak hw_perf_enable(void)                { barrier(); }
 
 void __weak hw_perf_counter_setup(int cpu)      { barrier(); }
+void __weak hw_perf_counter_setup_online(int cpu)       { barrier(); }
 
 int __weak
 hw_perf_group_sched_in(struct perf_counter *group_leader,
@@ -98,16 +106,16 @@ hw_perf_group_sched_in(struct perf_counter *group_leader,
 
 void __weak perf_counter_print_debug(void)      { }
 
-static DEFINE_PER_CPU(int, disable_count);
+static DEFINE_PER_CPU(int, perf_disable_count);
 
 void __perf_disable(void)
 {
-        __get_cpu_var(disable_count)++;
+        __get_cpu_var(perf_disable_count)++;
 }
 
 bool __perf_enable(void)
 {
-        return !--__get_cpu_var(disable_count);
+        return !--__get_cpu_var(perf_disable_count);
 }
 
 void perf_disable(void)
@@ -124,7 +132,7 @@ void perf_enable(void)
 
 static void get_ctx(struct perf_counter_context *ctx)
 {
-        atomic_inc(&ctx->refcount);
+        WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
 }
 
 static void free_ctx(struct rcu_head *head)
@@ -146,6 +154,28 @@ static void put_ctx(struct perf_counter_context *ctx)
         }
 }
 
+static void unclone_ctx(struct perf_counter_context *ctx)
+{
+        if (ctx->parent_ctx) {
+                put_ctx(ctx->parent_ctx);
+                ctx->parent_ctx = NULL;
+        }
+}
+
+/*
+ * If we inherit counters we want to return the parent counter id
+ * to userspace.
+ */
+static u64 primary_counter_id(struct perf_counter *counter)
+{
+        u64 id = counter->id;
+
+        if (counter->parent)
+                id = counter->parent->id;
+
+        return id;
+}
+
 /*
  * Get the perf_counter_context for a task and lock it.
  * This has to cope with with the fact that until it is locked,
@@ -175,6 +205,11 @@ perf_lock_task_context(struct task_struct *task, unsigned long *flags)
                         spin_unlock_irqrestore(&ctx->lock, *flags);
                         goto retry;
                 }
+
+                if (!atomic_inc_not_zero(&ctx->refcount)) {
+                        spin_unlock_irqrestore(&ctx->lock, *flags);
+                        ctx = NULL;
+                }
         }
         rcu_read_unlock();
         return ctx;
@@ -193,7 +228,6 @@ static struct perf_counter_context *perf_pin_task_context(struct task_struct *ta
         ctx = perf_lock_task_context(task, &flags);
         if (ctx) {
                 ++ctx->pin_count;
-                get_ctx(ctx);
                 spin_unlock_irqrestore(&ctx->lock, flags);
         }
         return ctx;
@@ -232,6 +266,8 @@ list_add_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
 
         list_add_rcu(&counter->event_entry, &ctx->event_list);
         ctx->nr_counters++;
+        if (counter->attr.inherit_stat)
+                ctx->nr_stat++;
 }
 
 /*
@@ -246,6 +282,8 @@ list_del_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
         if (list_empty(&counter->list_entry))
                 return;
         ctx->nr_counters--;
+        if (counter->attr.inherit_stat)
+                ctx->nr_stat--;
 
         list_del_init(&counter->list_entry);
         list_del_rcu(&counter->event_entry);
@@ -275,6 +313,10 @@ counter_sched_out(struct perf_counter *counter,
                 return;
 
         counter->state = PERF_COUNTER_STATE_INACTIVE;
+        if (counter->pending_disable) {
+                counter->pending_disable = 0;
+                counter->state = PERF_COUNTER_STATE_OFF;
+        }
         counter->tstamp_stopped = ctx->time;
         counter->pmu->disable(counter);
         counter->oncpu = -1;
@@ -433,7 +475,8 @@ static void update_counter_times(struct perf_counter *counter)
         struct perf_counter_context *ctx = counter->ctx;
         u64 run_end;
 
-        if (counter->state < PERF_COUNTER_STATE_INACTIVE)
+        if (counter->state < PERF_COUNTER_STATE_INACTIVE ||
+            counter->group_leader->state < PERF_COUNTER_STATE_INACTIVE)
                 return;
 
         counter->total_time_enabled = ctx->time - counter->tstamp_enabled;
@@ -482,7 +525,7 @@ static void __perf_counter_disable(void *info)
          */
         if (counter->state >= PERF_COUNTER_STATE_INACTIVE) {
                 update_context_time(ctx);
-                update_counter_times(counter);
+                update_group_times(counter);
                 if (counter == counter->group_leader)
                         group_sched_out(counter, cpuctx, ctx);
                 else
@@ -537,7 +580,7 @@ static void perf_counter_disable(struct perf_counter *counter)
          * in, so we can change the state safely.
          */
         if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
-                update_counter_times(counter);
+                update_group_times(counter);
                 counter->state = PERF_COUNTER_STATE_OFF;
         }
 
@@ -815,6 +858,27 @@ retry:
 }
 
 /*
+ * Put a counter into inactive state and update time fields.
+ * Enabling the leader of a group effectively enables all
+ * the group members that aren't explicitly disabled, so we
+ * have to update their ->tstamp_enabled also.
+ * Note: this works for group members as well as group leaders
+ * since the non-leader members' sibling_lists will be empty.
+ */
+static void __perf_counter_mark_enabled(struct perf_counter *counter,
+                                        struct perf_counter_context *ctx)
+{
+        struct perf_counter *sub;
+
+        counter->state = PERF_COUNTER_STATE_INACTIVE;
+        counter->tstamp_enabled = ctx->time - counter->total_time_enabled;
+        list_for_each_entry(sub, &counter->sibling_list, list_entry)
+                if (sub->state >= PERF_COUNTER_STATE_INACTIVE)
+                        sub->tstamp_enabled =
+                                ctx->time - sub->total_time_enabled;
+}
+
+/*
  * Cross CPU call to enable a performance counter
  */
 static void __perf_counter_enable(void *info)
@@ -841,8 +905,7 @@ static void __perf_counter_enable(void *info)
 
         if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
                 goto unlock;
-        counter->state = PERF_COUNTER_STATE_INACTIVE;
-        counter->tstamp_enabled = ctx->time - counter->total_time_enabled;
+        __perf_counter_mark_enabled(counter, ctx);
 
         /*
          * If the counter is in a group and isn't the group leader,
@@ -935,11 +998,9 @@ static void perf_counter_enable(struct perf_counter *counter)
          * Since we have the lock this context can't be scheduled
          * in, so we can change the state safely.
          */
-        if (counter->state == PERF_COUNTER_STATE_OFF) {
-                counter->state = PERF_COUNTER_STATE_INACTIVE;
-                counter->tstamp_enabled =
-                        ctx->time - counter->total_time_enabled;
-        }
+        if (counter->state == PERF_COUNTER_STATE_OFF)
+                __perf_counter_mark_enabled(counter, ctx);
+
  out:
         spin_unlock_irq(&ctx->lock);
 }
@@ -1002,6 +1063,81 @@ static int context_equiv(struct perf_counter_context *ctx1,
                 && !ctx1->pin_count && !ctx2->pin_count;
 }
 
+static void __perf_counter_read(void *counter);
+
+static void __perf_counter_sync_stat(struct perf_counter *counter,
+                                     struct perf_counter *next_counter)
+{
+        u64 value;
+
+        if (!counter->attr.inherit_stat)
+                return;
+
+        /*
+         * Update the counter value, we cannot use perf_counter_read()
+         * because we're in the middle of a context switch and have IRQs
+         * disabled, which upsets smp_call_function_single(), however
+         * we know the counter must be on the current CPU, therefore we
+         * don't need to use it.
+         */
+        switch (counter->state) {
+        case PERF_COUNTER_STATE_ACTIVE:
+                __perf_counter_read(counter);
+                break;
+
+        case PERF_COUNTER_STATE_INACTIVE:
+                update_counter_times(counter);
+                break;
+
+        default:
+                break;
+        }
+
+        /*
+         * In order to keep per-task stats reliable we need to flip the counter
+         * values when we flip the contexts.
+         */
+        value = atomic64_read(&next_counter->count);
+        value = atomic64_xchg(&counter->count, value);
+        atomic64_set(&next_counter->count, value);
+
+        swap(counter->total_time_enabled, next_counter->total_time_enabled);
+        swap(counter->total_time_running, next_counter->total_time_running);
+
+        /*
+         * Since we swizzled the values, update the user visible data too.
+         */
+        perf_counter_update_userpage(counter);
+        perf_counter_update_userpage(next_counter);
+}
+
+#define list_next_entry(pos, member) \
+        list_entry(pos->member.next, typeof(*pos), member)
+
+static void perf_counter_sync_stat(struct perf_counter_context *ctx,
+                                   struct perf_counter_context *next_ctx)
+{
+        struct perf_counter *counter, *next_counter;
+
+        if (!ctx->nr_stat)
+                return;
+
+        counter = list_first_entry(&ctx->event_list,
+                                   struct perf_counter, event_entry);
+
+        next_counter = list_first_entry(&next_ctx->event_list,
+                                        struct perf_counter, event_entry);
+
+        while (&counter->event_entry != &ctx->event_list &&
+               &next_counter->event_entry != &next_ctx->event_list) {
+
+                __perf_counter_sync_stat(counter, next_counter);
+
+                counter = list_next_entry(counter, event_entry);
+                next_counter = list_next_entry(next_counter, event_entry);
+        }
+}
+
 /*
  * Called from scheduler to remove the counters of the current task,
  * with interrupts disabled.
@@ -1057,6 +1193,8 @@ void perf_counter_task_sched_out(struct task_struct *task,
                         ctx->task = next;
                         next_ctx->task = task;
                         do_switch = 0;
+
+                        perf_counter_sync_stat(ctx, next_ctx);
                 }
                 spin_unlock(&next_ctx->lock);
                 spin_unlock(&ctx->lock);
@@ -1203,7 +1341,6 @@ static void perf_counter_cpu_sched_in(struct perf_cpu_context *cpuctx, int cpu)
 #define MAX_INTERRUPTS (~0ULL)
 
 static void perf_log_throttle(struct perf_counter *counter, int enable);
-static void perf_log_period(struct perf_counter *counter, u64 period);
 
 static void perf_adjust_period(struct perf_counter *counter, u64 events)
 {
@@ -1222,8 +1359,6 @@ static void perf_adjust_period(struct perf_counter *counter, u64 events)
         if (!sample_period)
                 sample_period = 1;
 
-        perf_log_period(counter, sample_period);
-
         hwc->sample_period = sample_period;
 }
 
@@ -1283,7 +1418,7 @@ static void perf_ctx_adjust_freq(struct perf_counter_context *ctx)
                 if (!interrupts) {
                         perf_disable();
                         counter->pmu->disable(counter);
-                        atomic_set(&hwc->period_left, 0);
+                        atomic64_set(&hwc->period_left, 0);
                         counter->pmu->enable(counter);
                         perf_enable();
                 }
@@ -1344,14 +1479,68 @@ void perf_counter_task_tick(struct task_struct *curr, int cpu)
 }
 
 /*
+ * Enable all of a task's counters that have been marked enable-on-exec.
+ * This expects task == current.
+ */
+static void perf_counter_enable_on_exec(struct task_struct *task)
+{
+        struct perf_counter_context *ctx;
+        struct perf_counter *counter;
+        unsigned long flags;
+        int enabled = 0;
+
+        local_irq_save(flags);
+        ctx = task->perf_counter_ctxp;
+        if (!ctx || !ctx->nr_counters)
+                goto out;
+
+        __perf_counter_task_sched_out(ctx);
+
+        spin_lock(&ctx->lock);
+
+        list_for_each_entry(counter, &ctx->counter_list, list_entry) {
+                if (!counter->attr.enable_on_exec)
+                        continue;
+                counter->attr.enable_on_exec = 0;
+                if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
+                        continue;
+                __perf_counter_mark_enabled(counter, ctx);
+                enabled = 1;
+        }
+
+        /*
+         * Unclone this context if we enabled any counter.
+         */
+        if (enabled)
+                unclone_ctx(ctx);
+
+        spin_unlock(&ctx->lock);
+
+        perf_counter_task_sched_in(task, smp_processor_id());
+ out:
+        local_irq_restore(flags);
+}
+
+/*
  * Cross CPU call to read the hardware counter
  */
-static void __read(void *info)
+static void __perf_counter_read(void *info)
 {
+        struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
         struct perf_counter *counter = info;
         struct perf_counter_context *ctx = counter->ctx;
         unsigned long flags;
 
+        /*
+         * If this is a task context, we need to check whether it is
+         * the current task context of this cpu.  If not it has been
+         * scheduled out before the smp call arrived.  In that case
+         * counter->count would have been updated to a recent sample
+         * when the counter was scheduled out.
+         */
+        if (ctx->task && cpuctx->task_ctx != ctx)
+                return;
+
         local_irq_save(flags);
         if (ctx->is_active)
                 update_context_time(ctx);
@@ -1368,7 +1557,7 @@ static u64 perf_counter_read(struct perf_counter *counter)
          */
         if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
                 smp_call_function_single(counter->oncpu,
-                                         __read, counter, 1);
+                                         __perf_counter_read, counter, 1);
         } else if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
                 update_counter_times(counter);
         }
@@ -1394,7 +1583,6 @@ __perf_counter_init_context(struct perf_counter_context *ctx,
 
 static struct perf_counter_context *find_get_context(pid_t pid, int cpu)
 {
-        struct perf_counter_context *parent_ctx;
         struct perf_counter_context *ctx;
         struct perf_cpu_context *cpuctx;
         struct task_struct *task;
@@ -1454,16 +1642,7 @@ static struct perf_counter_context *find_get_context(pid_t pid, int cpu)
  retry:
         ctx = perf_lock_task_context(task, &flags);
         if (ctx) {
-                parent_ctx = ctx->parent_ctx;
-                if (parent_ctx) {
-                        put_ctx(parent_ctx);
-                        ctx->parent_ctx = NULL;         /* no longer a clone */
-                }
-                /*
-                 * Get an extra reference before dropping the lock so that
-                 * this context won't get freed if the task exits.
-                 */
-                get_ctx(ctx);
+                unclone_ctx(ctx);
                 spin_unlock_irqrestore(&ctx->lock, flags);
         }
 
@@ -1509,11 +1688,20 @@ static void free_counter(struct perf_counter *counter)
 {
         perf_pending_sync(counter);
 
-        atomic_dec(&nr_counters);
-        if (counter->attr.mmap)
-                atomic_dec(&nr_mmap_counters);
-        if (counter->attr.comm)
-                atomic_dec(&nr_comm_counters);
+        if (!counter->parent) {
+                atomic_dec(&nr_counters);
+                if (counter->attr.mmap)
+                        atomic_dec(&nr_mmap_counters);
+                if (counter->attr.comm)
+                        atomic_dec(&nr_comm_counters);
+                if (counter->attr.task)
+                        atomic_dec(&nr_task_counters);
+        }
+
+        if (counter->output) {
+                fput(counter->output->filp);
+                counter->output = NULL;
+        }
 
         if (counter->destroy)
                 counter->destroy(counter);
@@ -1547,14 +1735,133 @@ static int perf_release(struct inode *inode, struct file *file)
         return 0;
 }
 
+static int perf_counter_read_size(struct perf_counter *counter)
+{
+        int entry = sizeof(u64); /* value */
+        int size = 0;
+        int nr = 1;
+
+        if (counter->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
+                size += sizeof(u64);
+
+        if (counter->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
+                size += sizeof(u64);
+
+        if (counter->attr.read_format & PERF_FORMAT_ID)
+                entry += sizeof(u64);
+
+        if (counter->attr.read_format & PERF_FORMAT_GROUP) {
+                nr += counter->group_leader->nr_siblings;
+                size += sizeof(u64);
+        }
+
+        size += entry * nr;
+
+        return size;
+}
+
+static u64 perf_counter_read_value(struct perf_counter *counter)
+{
+        struct perf_counter *child;
+        u64 total = 0;
+
+        total += perf_counter_read(counter);
+        list_for_each_entry(child, &counter->child_list, child_list)
+                total += perf_counter_read(child);
+
+        return total;
+}
+
+static int perf_counter_read_entry(struct perf_counter *counter,
+                                   u64 read_format, char __user *buf)
+{
+        int n = 0, count = 0;
+        u64 values[2];
+
+        values[n++] = perf_counter_read_value(counter);
+        if (read_format & PERF_FORMAT_ID)
+                values[n++] = primary_counter_id(counter);
+
+        count = n * sizeof(u64);
+
+        if (copy_to_user(buf, values, count))
+                return -EFAULT;
+
+        return count;
+}
+
+static int perf_counter_read_group(struct perf_counter *counter,
+                                   u64 read_format, char __user *buf)
+{
+        struct perf_counter *leader = counter->group_leader, *sub;
+        int n = 0, size = 0, err = -EFAULT;
+        u64 values[3];
+
+        values[n++] = 1 + leader->nr_siblings;
+        if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
+                values[n++] = leader->total_time_enabled +
+                        atomic64_read(&leader->child_total_time_enabled);
+        }
+        if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
+                values[n++] = leader->total_time_running +
+                        atomic64_read(&leader->child_total_time_running);
+        }
+
+        size = n * sizeof(u64);
+
+        if (copy_to_user(buf, values, size))
+                return -EFAULT;
+
+        err = perf_counter_read_entry(leader, read_format, buf + size);
+        if (err < 0)
+                return err;
+
+        size += err;
+
+        list_for_each_entry(sub, &leader->sibling_list, list_entry) {
+                err = perf_counter_read_entry(sub, read_format,
+                                buf + size);
+                if (err < 0)
+                        return err;
+
+                size += err;
+        }
+
+        return size;
+}
+
+static int perf_counter_read_one(struct perf_counter *counter,
+                                 u64 read_format, char __user *buf)
+{
+        u64 values[4];
+        int n = 0;
+
+        values[n++] = perf_counter_read_value(counter);
+        if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
+                values[n++] = counter->total_time_enabled +
+                        atomic64_read(&counter->child_total_time_enabled);
+        }
+        if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
+                values[n++] = counter->total_time_running +
+                        atomic64_read(&counter->child_total_time_running);
+        }
+        if (read_format & PERF_FORMAT_ID)
+                values[n++] = primary_counter_id(counter);
+
+        if (copy_to_user(buf, values, n * sizeof(u64)))
+                return -EFAULT;
+
+        return n * sizeof(u64);
+}
+
 /*
  * Read the performance counter - simple non blocking version for now
  */
 static ssize_t
 perf_read_hw(struct perf_counter *counter, char __user *buf, size_t count)
 {
-        u64 values[3];
-        int n;
+        u64 read_format = counter->attr.read_format;
+        int ret;
 
         /*
          * Return end-of-file for a read on a counter that is in
@@ -1564,28 +1871,18 @@ perf_read_hw(struct perf_counter *counter, char __user *buf, size_t count)
         if (counter->state == PERF_COUNTER_STATE_ERROR)
                 return 0;
 
+        if (count < perf_counter_read_size(counter))
+                return -ENOSPC;
+
         WARN_ON_ONCE(counter->ctx->parent_ctx);
         mutex_lock(&counter->child_mutex);
-        values[0] = perf_counter_read(counter);
-        n = 1;
-        if (counter->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
-                values[n++] = counter->total_time_enabled +
-                        atomic64_read(&counter->child_total_time_enabled);
-        if (counter->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
-                values[n++] = counter->total_time_running +
-                        atomic64_read(&counter->child_total_time_running);
-        if (counter->attr.read_format & PERF_FORMAT_ID)
-                values[n++] = counter->id;
+        if (read_format & PERF_FORMAT_GROUP)
+                ret = perf_counter_read_group(counter, read_format, buf);
+        else
+                ret = perf_counter_read_one(counter, read_format, buf);
         mutex_unlock(&counter->child_mutex);
 
-        if (count < n * sizeof(u64))
-                return -EINVAL;
-        count = n * sizeof(u64);
-
-        if (copy_to_user(buf, values, count))
-                return -EFAULT;
-
-        return count;
+        return ret;
 }
 
 static ssize_t
@@ -1620,22 +1917,6 @@ static void perf_counter_reset(struct perf_counter *counter)
         perf_counter_update_userpage(counter);
 }
 
-static void perf_counter_for_each_sibling(struct perf_counter *counter,
-                                          void (*func)(struct perf_counter *))
-{
-        struct perf_counter_context *ctx = counter->ctx;
-        struct perf_counter *sibling;
-
-        WARN_ON_ONCE(ctx->parent_ctx);
-        mutex_lock(&ctx->mutex);
-        counter = counter->group_leader;
-
-        func(counter);
-        list_for_each_entry(sibling, &counter->sibling_list, list_entry)
-                func(sibling);
-        mutex_unlock(&ctx->mutex);
-}
-
 /*
  * Holding the top-level counter's child_mutex means that any
  * descendant process that has inherited this counter will block
@@ -1658,14 +1939,18 @@ static void perf_counter_for_each_child(struct perf_counter *counter,
 static void perf_counter_for_each(struct perf_counter *counter,
                                   void (*func)(struct perf_counter *))
 {
-        struct perf_counter *child;
+        struct perf_counter_context *ctx = counter->ctx;
+        struct perf_counter *sibling;
 
-        WARN_ON_ONCE(counter->ctx->parent_ctx);
-        mutex_lock(&counter->child_mutex);
-        perf_counter_for_each_sibling(counter, func);
-        list_for_each_entry(child, &counter->child_list, child_list)
-                perf_counter_for_each_sibling(child, func);
-        mutex_unlock(&counter->child_mutex);
+        WARN_ON_ONCE(ctx->parent_ctx);
+        mutex_lock(&ctx->mutex);
+        counter = counter->group_leader;
+
+        perf_counter_for_each_child(counter, func);
+        func(counter);
+        list_for_each_entry(sibling, &counter->sibling_list, list_entry)
+                perf_counter_for_each_child(counter, func);
+        mutex_unlock(&ctx->mutex);
 }
 
 static int perf_counter_period(struct perf_counter *counter, u64 __user *arg)
@@ -1694,8 +1979,6 @@ static int perf_counter_period(struct perf_counter *counter, u64 __user *arg)
 
                 counter->attr.sample_freq = value;
         } else {
-                perf_log_period(counter, value);
-
                 counter->attr.sample_period = value;
                 counter->hw.sample_period = value;
         }
@@ -1705,6 +1988,8 @@ unlock:
         return ret;
 }
 
+int perf_counter_set_output(struct perf_counter *counter, int output_fd);
+
 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 {
         struct perf_counter *counter = file->private_data;
@@ -1728,6 +2013,9 @@ static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
         case PERF_COUNTER_IOC_PERIOD:
                 return perf_counter_period(counter, (u64 __user *)arg);
 
+        case PERF_COUNTER_IOC_SET_OUTPUT:
+                return perf_counter_set_output(counter, arg);
+
         default:
                 return -ENOTTY;
         }
@@ -1764,6 +2052,18 @@ int perf_counter_task_disable(void)
         return 0;
 }
 
+#ifndef PERF_COUNTER_INDEX_OFFSET
+# define PERF_COUNTER_INDEX_OFFSET 0
+#endif
+
+static int perf_counter_index(struct perf_counter *counter)
+{
+        if (counter->state != PERF_COUNTER_STATE_ACTIVE)
+                return 0;
+
+        return counter->hw.idx + 1 - PERF_COUNTER_INDEX_OFFSET;
+}
+
 /*
  * Callers need to ensure there can be no nesting of this function, otherwise
  * the seqlock logic goes bad. We can not serialize this because the arch
@@ -1788,11 +2088,17 @@ void perf_counter_update_userpage(struct perf_counter *counter)
         preempt_disable();
         ++userpg->lock;
         barrier();
-        userpg->index = counter->hw.idx;
+        userpg->index = perf_counter_index(counter);
         userpg->offset = atomic64_read(&counter->count);
         if (counter->state == PERF_COUNTER_STATE_ACTIVE)
                 userpg->offset -= atomic64_read(&counter->hw.prev_count);
 
+        userpg->time_enabled = counter->total_time_enabled +
+                        atomic64_read(&counter->child_total_time_enabled);
+
+        userpg->time_running = counter->total_time_running +
+                        atomic64_read(&counter->child_total_time_running);
+
         barrier();
         ++userpg->lock;
         preempt_enable();
@@ -1806,6 +2112,12 @@ static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
         struct perf_mmap_data *data;
         int ret = VM_FAULT_SIGBUS;
 
+        if (vmf->flags & FAULT_FLAG_MKWRITE) {
+                if (vmf->pgoff == 0)
+                        ret = 0;
+                return ret;
+        }
+
         rcu_read_lock();
         data = rcu_dereference(counter->data);
         if (!data)
@@ -1819,9 +2131,16 @@ static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
                 if ((unsigned)nr > data->nr_pages)
                         goto unlock;
 
+                if (vmf->flags & FAULT_FLAG_WRITE)
+                        goto unlock;
+
                 vmf->page = virt_to_page(data->data_pages[nr]);
         }
+
         get_page(vmf->page);
+        vmf->page->mapping = vma->vm_file->f_mapping;
+        vmf->page->index   = vmf->pgoff;
+
         ret = 0;
 unlock:
         rcu_read_unlock();
@@ -1874,6 +2193,14 @@ fail:
         return -ENOMEM;
 }
 
+static void perf_mmap_free_page(unsigned long addr)
+{
+        struct page *page = virt_to_page((void *)addr);
+
+        page->mapping = NULL;
+        __free_page(page);
+}
+
 static void __perf_mmap_data_free(struct rcu_head *rcu_head)
 {
         struct perf_mmap_data *data;
@@ -1881,9 +2208,10 @@ static void __perf_mmap_data_free(struct rcu_head *rcu_head)
 
         data = container_of(rcu_head, struct perf_mmap_data, rcu_head);
 
-        free_page((unsigned long)data->user_page);
+        perf_mmap_free_page((unsigned long)data->user_page);
         for (i = 0; i < data->nr_pages; i++)
-                free_page((unsigned long)data->data_pages[i]);
+                perf_mmap_free_page((unsigned long)data->data_pages[i]);
+
         kfree(data);
 }
 
@@ -1920,9 +2248,10 @@ static void perf_mmap_close(struct vm_area_struct *vma)
 }
 
 static struct vm_operations_struct perf_mmap_vmops = {
-        .open  = perf_mmap_open,
-        .close = perf_mmap_close,
-        .fault = perf_mmap_fault,
+        .open           = perf_mmap_open,
+        .close          = perf_mmap_close,
+        .fault          = perf_mmap_fault,
+        .page_mkwrite   = perf_mmap_fault,
 };
 
 static int perf_mmap(struct file *file, struct vm_area_struct *vma)
@@ -1936,7 +2265,7 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma)
         long user_extra, extra;
         int ret = 0;
 
-        if (!(vma->vm_flags & VM_SHARED) || (vma->vm_flags & VM_WRITE))
+        if (!(vma->vm_flags & VM_SHARED))
                 return -EINVAL;
 
         vma_size = vma->vm_end - vma->vm_start;
@@ -1957,6 +2286,11 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma)
 
         WARN_ON_ONCE(counter->ctx->parent_ctx);
         mutex_lock(&counter->mmap_mutex);
+        if (counter->output) {
+                ret = -EINVAL;
+                goto unlock;
+        }
+
         if (atomic_inc_not_zero(&counter->mmap_count)) {
                 if (nr_pages != counter->data->nr_pages)
                         ret = -EINVAL;
@@ -1995,10 +2329,12 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma)
         atomic_long_add(user_extra, &user->locked_vm);
         vma->vm_mm->locked_vm += extra;
         counter->data->nr_locked = extra;
+        if (vma->vm_flags & VM_WRITE)
+                counter->data->writable = 1;
+
 unlock:
         mutex_unlock(&counter->mmap_mutex);
 
-        vma->vm_flags &= ~VM_MAYWRITE;
         vma->vm_flags |= VM_RESERVED;
         vma->vm_ops = &perf_mmap_vmops;
 
@@ -2064,7 +2400,7 @@ static void perf_pending_counter(struct perf_pending_entry *entry)
 
         if (counter->pending_disable) {
                 counter->pending_disable = 0;
-                perf_counter_disable(counter);
+                __perf_counter_disable(counter);
         }
 
         if (counter->pending_wakeup) {
@@ -2175,11 +2511,38 @@ struct perf_output_handle {
         unsigned long           head;
         unsigned long           offset;
         int                     nmi;
-        int                     overflow;
+        int                     sample;
         int                     locked;
         unsigned long           flags;
 };
 
+static bool perf_output_space(struct perf_mmap_data *data,
+                              unsigned int offset, unsigned int head)
+{
+        unsigned long tail;
+        unsigned long mask;
+
+        if (!data->writable)
+                return true;
+
+        mask = (data->nr_pages << PAGE_SHIFT) - 1;
+        /*
+         * Userspace could choose to issue a mb() before updating the tail
+         * pointer. So that all reads will be completed before the write is
+         * issued.
+         */
+        tail = ACCESS_ONCE(data->user_page->data_tail);
+        smp_rmb();
+
+        offset = (offset - tail) & mask;
+        head   = (head   - tail) & mask;
+
+        if ((int)(head - offset) < 0)
+                return false;
+
+        return true;
+}
+
 static void perf_output_wakeup(struct perf_output_handle *handle)
 {
         atomic_set(&handle->data->poll, POLL_IN);
@@ -2270,37 +2633,93 @@ out:
         local_irq_restore(handle->flags);
 }
 
+static void perf_output_copy(struct perf_output_handle *handle,
+                             const void *buf, unsigned int len)
+{
+        unsigned int pages_mask;
+        unsigned int offset;
+        unsigned int size;
+        void **pages;
+
+        offset          = handle->offset;
+        pages_mask      = handle->data->nr_pages - 1;
+        pages           = handle->data->data_pages;
+
+        do {
+                unsigned int page_offset;
+                int nr;
+
+                nr          = (offset >> PAGE_SHIFT) & pages_mask;
+                page_offset = offset & (PAGE_SIZE - 1);
+                size        = min_t(unsigned int, PAGE_SIZE - page_offset, len);
+
+                memcpy(pages[nr] + page_offset, buf, size);
+
+                len         -= size;
+                buf         += size;
+                offset      += size;
+        } while (len);
+
+        handle->offset = offset;
+
+        /*
+         * Check we didn't copy past our reservation window, taking the
+         * possible unsigned int wrap into account.
+         */
+        WARN_ON_ONCE(((long)(handle->head - handle->offset)) < 0);
+}
+
+#define perf_output_put(handle, x) \
+        perf_output_copy((handle), &(x), sizeof(x))
+
 static int perf_output_begin(struct perf_output_handle *handle,
                              struct perf_counter *counter, unsigned int size,
-                             int nmi, int overflow)
+                             int nmi, int sample)
 {
+        struct perf_counter *output_counter;
         struct perf_mmap_data *data;
         unsigned int offset, head;
+        int have_lost;
+        struct {
+                struct perf_event_header header;
+                u64                      id;
+                u64                      lost;
+        } lost_event;
 
+        rcu_read_lock();
         /*
          * For inherited counters we send all the output towards the parent.
          */
         if (counter->parent)
                 counter = counter->parent;
 
-        rcu_read_lock();
+        output_counter = rcu_dereference(counter->output);
+        if (output_counter)
+                counter = output_counter;
+
         data = rcu_dereference(counter->data);
         if (!data)
                 goto out;
 
-        handle->data     = data;
-        handle->counter  = counter;
-        handle->nmi      = nmi;
-        handle->overflow = overflow;
+        handle->data    = data;
+        handle->counter = counter;
+        handle->nmi     = nmi;
+        handle->sample  = sample;
 
         if (!data->nr_pages)
                 goto fail;
 
+        have_lost = atomic_read(&data->lost);
+        if (have_lost)
+                size += sizeof(lost_event);
+
         perf_output_lock(handle);
 
         do {
                 offset = head = atomic_long_read(&data->head);
                 head += size;
+                if (unlikely(!perf_output_space(data, offset, head)))
+                        goto fail;
         } while (atomic_long_cmpxchg(&data->head, offset, head) != offset);
 
         handle->offset  = offset;
@@ -2309,55 +2728,27 @@ static int perf_output_begin(struct perf_output_handle *handle,
         if ((offset >> PAGE_SHIFT) != (head >> PAGE_SHIFT))
                 atomic_set(&data->wakeup, 1);
 
+        if (have_lost) {
+                lost_event.header.type = PERF_EVENT_LOST;
+                lost_event.header.misc = 0;
+                lost_event.header.size = sizeof(lost_event);
+                lost_event.id          = counter->id;
+                lost_event.lost        = atomic_xchg(&data->lost, 0);
+
+                perf_output_put(handle, lost_event);
+        }
+
         return 0;
 
 fail:
-        perf_output_wakeup(handle);
+        atomic_inc(&data->lost);
+        perf_output_unlock(handle);
 out:
         rcu_read_unlock();
 
         return -ENOSPC;
 }
 
-static void perf_output_copy(struct perf_output_handle *handle,
-                             const void *buf, unsigned int len)
-{
-        unsigned int pages_mask;
-        unsigned int offset;
-        unsigned int size;
-        void **pages;
-
-        offset          = handle->offset;
-        pages_mask      = handle->data->nr_pages - 1;
-        pages           = handle->data->data_pages;
-
-        do {
-                unsigned int page_offset;
-                int nr;
-
-                nr          = (offset >> PAGE_SHIFT) & pages_mask;
-                page_offset = offset & (PAGE_SIZE - 1);
-                size        = min_t(unsigned int, PAGE_SIZE - page_offset, len);
-
-                memcpy(pages[nr] + page_offset, buf, size);
-
-                len         -= size;
-                buf         += size;
-                offset      += size;
-        } while (len);
-
-        handle->offset = offset;
-
-        /*
-         * Check we didn't copy past our reservation window, taking the
-         * possible unsigned int wrap into account.
-         */
-        WARN_ON_ONCE(((long)(handle->head - handle->offset)) < 0);
-}
-
-#define perf_output_put(handle, x) \
-        perf_output_copy((handle), &(x), sizeof(x))
-
 static void perf_output_end(struct perf_output_handle *handle)
 {
         struct perf_counter *counter = handle->counter;
@@ -2365,7 +2756,7 @@ static void perf_output_end(struct perf_output_handle *handle)
 
         int wakeup_events = counter->attr.wakeup_events;
 
-        if (handle->overflow && wakeup_events) {
+        if (handle->sample && wakeup_events) {
                 int events = atomic_inc_return(&data->events);
                 if (events >= wakeup_events) {
                         atomic_sub(wakeup_events, &data->events);
@@ -2399,7 +2790,80 @@ static u32 perf_counter_tid(struct perf_counter *counter, struct task_struct *p)
         return task_pid_nr_ns(p, counter->ns);
 }
 
-static void perf_counter_output(struct perf_counter *counter, int nmi,
+static void perf_output_read_one(struct perf_output_handle *handle,
+                                 struct perf_counter *counter)
+{
+        u64 read_format = counter->attr.read_format;
+        u64 values[4];
+        int n = 0;
+
+        values[n++] = atomic64_read(&counter->count);
+        if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
+                values[n++] = counter->total_time_enabled +
+                        atomic64_read(&counter->child_total_time_enabled);
+        }
+        if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
+                values[n++] = counter->total_time_running +
+                        atomic64_read(&counter->child_total_time_running);
+        }
+        if (read_format & PERF_FORMAT_ID)
+                values[n++] = primary_counter_id(counter);
+
+        perf_output_copy(handle, values, n * sizeof(u64));
+}
+
+/*
+ * XXX PERF_FORMAT_GROUP vs inherited counters seems difficult.
+ */
+static void perf_output_read_group(struct perf_output_handle *handle,
+                            struct perf_counter *counter)
+{
+        struct perf_counter *leader = counter->group_leader, *sub;
+        u64 read_format = counter->attr.read_format;
+        u64 values[5];
+        int n = 0;
+
+        values[n++] = 1 + leader->nr_siblings;
+
+        if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
+                values[n++] = leader->total_time_enabled;
+
+        if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
+                values[n++] = leader->total_time_running;
+
+        if (leader != counter)
+                leader->pmu->read(leader);
+
+        values[n++] = atomic64_read(&leader->count);
+        if (read_format & PERF_FORMAT_ID)
+                values[n++] = primary_counter_id(leader);
+
+        perf_output_copy(handle, values, n * sizeof(u64));
+
+        list_for_each_entry(sub, &leader->sibling_list, list_entry) {
+                n = 0;
+
+                if (sub != counter)
+                        sub->pmu->read(sub);
+
+                values[n++] = atomic64_read(&sub->count);
+                if (read_format & PERF_FORMAT_ID)
+                        values[n++] = primary_counter_id(sub);
+
+                perf_output_copy(handle, values, n * sizeof(u64));
+        }
+}
+
+static void perf_output_read(struct perf_output_handle *handle,
+                             struct perf_counter *counter)
+{
+        if (counter->attr.read_format & PERF_FORMAT_GROUP)
+                perf_output_read_group(handle, counter);
+        else
+                perf_output_read_one(handle, counter);
+}
+
+void perf_counter_output(struct perf_counter *counter, int nmi,
                                 struct perf_sample_data *data)
 {
         int ret;
@@ -2410,10 +2874,6 @@ static void perf_counter_output(struct perf_counter *counter, int nmi,
         struct {
                 u32 pid, tid;
         } tid_entry;
-        struct {
-                u64 id;
-                u64 counter;
-        } group_entry;
         struct perf_callchain_entry *callchain = NULL;
         int callchain_size = 0;
         u64 time;
@@ -2421,15 +2881,14 @@ static void perf_counter_output(struct perf_counter *counter, int nmi,
                 u32 cpu, reserved;
         } cpu_entry;
 
-        header.type = 0;
+        header.type = PERF_EVENT_SAMPLE;
         header.size = sizeof(header);
 
-        header.misc = PERF_EVENT_MISC_OVERFLOW;
+        header.misc = 0;
         header.misc |= perf_misc_flags(data->regs);
 
         if (sample_type & PERF_SAMPLE_IP) {
                 ip = perf_instruction_pointer(data->regs);
-                header.type |= PERF_SAMPLE_IP;
                 header.size += sizeof(ip);
         }
 
@@ -2438,7 +2897,6 @@ static void perf_counter_output(struct perf_counter *counter, int nmi,
                 tid_entry.pid = perf_counter_pid(counter, current);
                 tid_entry.tid = perf_counter_tid(counter, current);
 
-                header.type |= PERF_SAMPLE_TID;
                 header.size += sizeof(tid_entry);
         }
 
@@ -2448,47 +2906,51 @@ static void perf_counter_output(struct perf_counter *counter, int nmi,
                  */
                 time = sched_clock();
 
-                header.type |= PERF_SAMPLE_TIME;
                 header.size += sizeof(u64);
         }
 
-        if (sample_type & PERF_SAMPLE_ADDR) {
-                header.type |= PERF_SAMPLE_ADDR;
+        if (sample_type & PERF_SAMPLE_ADDR)
                 header.size += sizeof(u64);
-        }
 
-        if (sample_type & PERF_SAMPLE_ID) {
-                header.type |= PERF_SAMPLE_ID;
+        if (sample_type & PERF_SAMPLE_ID)
+                header.size += sizeof(u64);
+
+        if (sample_type & PERF_SAMPLE_STREAM_ID)
                 header.size += sizeof(u64);
-        }
 
         if (sample_type & PERF_SAMPLE_CPU) {
-                header.type |= PERF_SAMPLE_CPU;
                 header.size += sizeof(cpu_entry);
 
                 cpu_entry.cpu = raw_smp_processor_id();
+                cpu_entry.reserved = 0;
         }
 
-        if (sample_type & PERF_SAMPLE_PERIOD) {
-                header.type |= PERF_SAMPLE_PERIOD;
+        if (sample_type & PERF_SAMPLE_PERIOD)
                 header.size += sizeof(u64);
-        }
 
-        if (sample_type & PERF_SAMPLE_GROUP) {
-                header.type |= PERF_SAMPLE_GROUP;
-                header.size += sizeof(u64) +
-                        counter->nr_siblings * sizeof(group_entry);
-        }
+        if (sample_type & PERF_SAMPLE_READ)
+                header.size += perf_counter_read_size(counter);
 
         if (sample_type & PERF_SAMPLE_CALLCHAIN) {
                 callchain = perf_callchain(data->regs);
 
                 if (callchain) {
                         callchain_size = (1 + callchain->nr) * sizeof(u64);
-
-                        header.type |= PERF_SAMPLE_CALLCHAIN;
                         header.size += callchain_size;
-                }
+                } else
+                        header.size += sizeof(u64);
+        }
+
+        if (sample_type & PERF_SAMPLE_RAW) {
+                int size = sizeof(u32);
+
+                if (data->raw)
+                        size += data->raw->size;
+                else
+                        size += sizeof(u32);
+
+                WARN_ON_ONCE(size & (sizeof(u64)-1));
+                header.size += size;
         }
 
         ret = perf_output_begin(&handle, counter, header.size, nmi, 1);
@@ -2509,7 +2971,13 @@ static void perf_counter_output(struct perf_counter *counter, int nmi,
         if (sample_type & PERF_SAMPLE_ADDR)
                 perf_output_put(&handle, data->addr);
 
-        if (sample_type & PERF_SAMPLE_ID)
+        if (sample_type & PERF_SAMPLE_ID) {
+                u64 id = primary_counter_id(counter);
+
+                perf_output_put(&handle, id);
+        }
+
+        if (sample_type & PERF_SAMPLE_STREAM_ID)
                 perf_output_put(&handle, counter->id);
 
         if (sample_type & PERF_SAMPLE_CPU)
@@ -2518,76 +2986,125 @@ static void perf_counter_output(struct perf_counter *counter, int nmi,
         if (sample_type & PERF_SAMPLE_PERIOD)
                 perf_output_put(&handle, data->period);
 
-        /*
-         * XXX PERF_SAMPLE_GROUP vs inherited counters seems difficult.
-         */
-        if (sample_type & PERF_SAMPLE_GROUP) {
-                struct perf_counter *leader, *sub;
-                u64 nr = counter->nr_siblings;
+        if (sample_type & PERF_SAMPLE_READ)
+                perf_output_read(&handle, counter);
 
-                perf_output_put(&handle, nr);
-
-                leader = counter->group_leader;
-                list_for_each_entry(sub, &leader->sibling_list, list_entry) {
-                        if (sub != counter)
-                                sub->pmu->read(sub);
-
-                        group_entry.id = sub->id;
-                        group_entry.counter = atomic64_read(&sub->count);
+        if (sample_type & PERF_SAMPLE_CALLCHAIN) {
+                if (callchain)
+                        perf_output_copy(&handle, callchain, callchain_size);
+                else {
+                        u64 nr = 0;
+                        perf_output_put(&handle, nr);
+                }
+        }
 
-                        perf_output_put(&handle, group_entry);
+        if (sample_type & PERF_SAMPLE_RAW) {
+                if (data->raw) {
+                        perf_output_put(&handle, data->raw->size);
+                        perf_output_copy(&handle, data->raw->data, data->raw->size);
+                } else {
+                        struct {
+                                u32     size;
+                                u32     data;
+                        } raw = {
+                                .size = sizeof(u32),
+                                .data = 0,
+                        };
+                        perf_output_put(&handle, raw);
                 }
         }
 
-        if (callchain)
-                perf_output_copy(&handle, callchain, callchain_size);
+        perf_output_end(&handle);
+}
+
+/*
+ * read event
+ */
+
+struct perf_read_event {
+        struct perf_event_header        header;
+
+        u32                             pid;
+        u32                             tid;
+};
+
+static void
+perf_counter_read_event(struct perf_counter *counter,
+                        struct task_struct *task)
+{
+        struct perf_output_handle handle;
+        struct perf_read_event event = {
+                .header = {
+                        .type = PERF_EVENT_READ,
+                        .misc = 0,
+                        .size = sizeof(event) + perf_counter_read_size(counter),
+                },
+                .pid = perf_counter_pid(counter, task),
+                .tid = perf_counter_tid(counter, task),
+        };
+        int ret;
+
+        ret = perf_output_begin(&handle, counter, event.header.size, 0, 0);
+        if (ret)
+                return;
+
+        perf_output_put(&handle, event);
+        perf_output_read(&handle, counter);
 
         perf_output_end(&handle);
 }
 
 /*
- * fork tracking
+ * task tracking -- fork/exit
+ *
+ * enabled by: attr.comm | attr.mmap | attr.task
  */
 
-struct perf_fork_event {
-        struct task_struct      *task;
+struct perf_task_event {
+        struct task_struct              *task;
+        struct perf_counter_context     *task_ctx;
 
         struct {
                 struct perf_event_header        header;
 
                 u32                             pid;
                 u32                             ppid;
+                u32                             tid;
+                u32                             ptid;
         } event;
 };
 
-static void perf_counter_fork_output(struct perf_counter *counter,
-                                     struct perf_fork_event *fork_event)
+static void perf_counter_task_output(struct perf_counter *counter,
+                                     struct perf_task_event *task_event)
 {
         struct perf_output_handle handle;
-        int size = fork_event->event.header.size;
-        struct task_struct *task = fork_event->task;
+        int size = task_event->event.header.size;
+        struct task_struct *task = task_event->task;
         int ret = perf_output_begin(&handle, counter, size, 0, 0);
 
         if (ret)
                 return;
 
-        fork_event->event.pid = perf_counter_pid(counter, task);
-        fork_event->event.ppid = perf_counter_pid(counter, task->real_parent);
+        task_event->event.pid = perf_counter_pid(counter, task);
+        task_event->event.ppid = perf_counter_pid(counter, current);
 
-        perf_output_put(&handle, fork_event->event);
+        task_event->event.tid = perf_counter_tid(counter, task);
+        task_event->event.ptid = perf_counter_tid(counter, current);
+
+        perf_output_put(&handle, task_event->event);
         perf_output_end(&handle);
 }
 
-static int perf_counter_fork_match(struct perf_counter *counter)
+static int perf_counter_task_match(struct perf_counter *counter)
 {
-        if (counter->attr.comm || counter->attr.mmap)
+        if (counter->attr.comm || counter->attr.mmap || counter->attr.task)
                 return 1;
 
         return 0;
 }
 
-static void perf_counter_fork_ctx(struct perf_counter_context *ctx,
-                                  struct perf_fork_event *fork_event)
+static void perf_counter_task_ctx(struct perf_counter_context *ctx,
+                                  struct perf_task_event *task_event)
 {
         struct perf_counter *counter;
 
@@ -2596,51 +3113,62 @@ static void perf_counter_fork_ctx(struct perf_counter_context *ctx,
 
         rcu_read_lock();
         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
-                if (perf_counter_fork_match(counter))
-                        perf_counter_fork_output(counter, fork_event);
+                if (perf_counter_task_match(counter))
+                        perf_counter_task_output(counter, task_event);
         }
         rcu_read_unlock();
 }
 
-static void perf_counter_fork_event(struct perf_fork_event *fork_event)
+static void perf_counter_task_event(struct perf_task_event *task_event)
 {
         struct perf_cpu_context *cpuctx;
-        struct perf_counter_context *ctx;
+        struct perf_counter_context *ctx = task_event->task_ctx;
 
         cpuctx = &get_cpu_var(perf_cpu_context);
-        perf_counter_fork_ctx(&cpuctx->ctx, fork_event);
+        perf_counter_task_ctx(&cpuctx->ctx, task_event);
         put_cpu_var(perf_cpu_context);
 
         rcu_read_lock();
-        /*
-         * doesn't really matter which of the child contexts the
-         * events ends up in.
-         */
-        ctx = rcu_dereference(current->perf_counter_ctxp);
+        if (!ctx)
+                ctx = rcu_dereference(task_event->task->perf_counter_ctxp);
         if (ctx)
-                perf_counter_fork_ctx(ctx, fork_event);
+                perf_counter_task_ctx(ctx, task_event);
         rcu_read_unlock();
 }
 
-void perf_counter_fork(struct task_struct *task)
+static void perf_counter_task(struct task_struct *task,
+                              struct perf_counter_context *task_ctx,
+                              int new)
 {
-        struct perf_fork_event fork_event;
+        struct perf_task_event task_event;
 
         if (!atomic_read(&nr_comm_counters) &&
-            !atomic_read(&nr_mmap_counters))
+            !atomic_read(&nr_mmap_counters) &&
+            !atomic_read(&nr_task_counters))
                 return;
 
-        fork_event = (struct perf_fork_event){
-                .task   = task,
-                .event  = {
+        task_event = (struct perf_task_event){
+                .task     = task,
+                .task_ctx = task_ctx,
+                .event    = {
                         .header = {
-                                .type = PERF_EVENT_FORK,
-                                .size = sizeof(fork_event.event),
+                                .type = new ? PERF_EVENT_FORK : PERF_EVENT_EXIT,
+                                .misc = 0,
+                                .size = sizeof(task_event.event),
                         },
+                        /* .pid  */
+                        /* .ppid */
+                        /* .tid  */
+                        /* .ptid */
                 },
         };
 
-        perf_counter_fork_event(&fork_event);
+        perf_counter_task_event(&task_event);
+}
+
+void perf_counter_fork(struct task_struct *task)
+{
+        perf_counter_task(task, NULL, 1);
 }
 
 /*
@@ -2708,8 +3236,10 @@ static void perf_counter_comm_event(struct perf_comm_event *comm_event)
         struct perf_cpu_context *cpuctx;
         struct perf_counter_context *ctx;
         unsigned int size;
-        char *comm = comm_event->task->comm;
+        char comm[TASK_COMM_LEN];
 
+        memset(comm, 0, sizeof(comm));
+        strncpy(comm, comm_event->task->comm, sizeof(comm));
         size = ALIGN(strlen(comm)+1, sizeof(u64));
 
         comm_event->comm = comm;
@@ -2736,13 +3266,24 @@ void perf_counter_comm(struct task_struct *task)
 {
         struct perf_comm_event comm_event;
 
+        if (task->perf_counter_ctxp)
+                perf_counter_enable_on_exec(task);
+
         if (!atomic_read(&nr_comm_counters))
                 return;
 
         comm_event = (struct perf_comm_event){
                 .task   = task,
+                /* .comm      */
+                /* .comm_size */
                 .event  = {
-                        .header = { .type = PERF_EVENT_COMM, },
+                        .header = {
+                                .type = PERF_EVENT_COMM,
+                                .misc = 0,
+                                /* .size */
+                        },
+                        /* .pid */
+                        /* .tid */
                 },
         };
 
@@ -2825,8 +3366,15 @@ static void perf_counter_mmap_event(struct perf_mmap_event *mmap_event)
         char *buf = NULL;
         const char *name;
 
+        memset(tmp, 0, sizeof(tmp));
+
         if (file) {
-                buf = kzalloc(PATH_MAX, GFP_KERNEL);
+                /*
+                 * d_path works from the end of the buffer backwards, so we
+                 * need to add enough zero bytes after the string to handle
+                 * the 64bit alignment we do later.
+                 */
+                buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
                 if (!buf) {
                         name = strncpy(tmp, "//enomem", sizeof(tmp));
                         goto got_name;
@@ -2837,9 +3385,11 @@ static void perf_counter_mmap_event(struct perf_mmap_event *mmap_event)
                         goto got_name;
                 }
         } else {
-                name = arch_vma_name(mmap_event->vma);
-                if (name)
+                if (arch_vma_name(mmap_event->vma)) {
+                        name = strncpy(tmp, arch_vma_name(mmap_event->vma),
+                                       sizeof(tmp));
                         goto got_name;
+                }
 
                 if (!vma->vm_mm) {
                         name = strncpy(tmp, "[vdso]", sizeof(tmp));
@@ -2884,8 +3434,16 @@ void __perf_counter_mmap(struct vm_area_struct *vma)
 
         mmap_event = (struct perf_mmap_event){
                 .vma    = vma,
+                /* .file_name */
+                /* .file_size */
                 .event  = {
-                        .header = { .type = PERF_EVENT_MMAP, },
+                        .header = {
+                                .type = PERF_EVENT_MMAP,
+                                .misc = 0,
+                                /* .size */
+                        },
+                        /* .pid */
+                        /* .tid */
                         .start  = vma->vm_start,
                         .len    = vma->vm_end - vma->vm_start,
                         .pgoff  = vma->vm_pgoff,
@@ -2896,49 +3454,6 @@ void __perf_counter_mmap(struct vm_area_struct *vma)
 }
 
 /*
- * Log sample_period changes so that analyzing tools can re-normalize the
- * event flow.
- */
-
-struct freq_event {
-        struct perf_event_header        header;
-        u64                             time;
-        u64                             id;
-        u64                             period;
-};
-
-static void perf_log_period(struct perf_counter *counter, u64 period)
-{
-        struct perf_output_handle handle;
-        struct freq_event event;
-        int ret;
-
-        if (counter->hw.sample_period == period)
-                return;
-
-        if (counter->attr.sample_type & PERF_SAMPLE_PERIOD)
-                return;
-
-        event = (struct freq_event) {
-                .header = {
-                        .type = PERF_EVENT_PERIOD,
-                        .misc = 0,
-                        .size = sizeof(event),
-                },
-                .time = sched_clock(),
-                .id = counter->id,
-                .period = period,
-        };
-
-        ret = perf_output_begin(&handle, counter, sizeof(event), 1, 0);
-        if (ret)
-                return;
-
-        perf_output_put(&handle, event);
-        perf_output_end(&handle);
-}
-
-/*
  * IRQ throttle logging
  */
 
@@ -2951,16 +3466,21 @@ static void perf_log_throttle(struct perf_counter *counter, int enable)
                 struct perf_event_header        header;
                 u64                             time;
                 u64                             id;
+                u64                             stream_id;
         } throttle_event = {
                 .header = {
-                        .type = PERF_EVENT_THROTTLE + 1,
+                        .type = PERF_EVENT_THROTTLE,
                         .misc = 0,
                         .size = sizeof(throttle_event),
                 },
-                .time   = sched_clock(),
-                .id     = counter->id,
+                .time           = sched_clock(),
+                .id             = primary_counter_id(counter),
+                .stream_id      = counter->id,
         };
 
+        if (enable)
+                throttle_event.header.type = PERF_EVENT_UNTHROTTLE;
+
         ret = perf_output_begin(&handle, counter, sizeof(throttle_event), 1, 0);
         if (ret)
                 return;
@@ -2970,7 +3490,7 @@ static void perf_log_throttle(struct perf_counter *counter, int enable)
 }
 
 /*
- * Generic counter overflow handling.
+ * Generic counter overflow handling, sampling.
  */
 
 int perf_counter_overflow(struct perf_counter *counter, int nmi,
@@ -3037,130 +3557,111 @@ int perf_counter_overflow(struct perf_counter *counter, int nmi,
  * Generic software counter infrastructure
  */
 
-static void perf_swcounter_update(struct perf_counter *counter)
+/*
+ * We directly increment counter->count and keep a second value in
+ * counter->hw.period_left to count intervals. This period counter
+ * is kept in the range [-sample_period, 0] so that we can use the
+ * sign as trigger.
+ */
+
+static u64 perf_swcounter_set_period(struct perf_counter *counter)
 {
         struct hw_perf_counter *hwc = &counter->hw;
-        u64 prev, now;
-        s64 delta;
+        u64 period = hwc->last_period;
+        u64 nr, offset;
+        s64 old, val;
+
+        hwc->last_period = hwc->sample_period;
 
 again:
-        prev = atomic64_read(&hwc->prev_count);
-        now = atomic64_read(&hwc->count);
-        if (atomic64_cmpxchg(&hwc->prev_count, prev, now) != prev)
-                goto again;
+        old = val = atomic64_read(&hwc->period_left);
+        if (val < 0)
+                return 0;
 
-        delta = now - prev;
+        nr = div64_u64(period + val, period);
+        offset = nr * period;
+        val -= offset;
+        if (atomic64_cmpxchg(&hwc->period_left, old, val) != old)
+                goto again;
 
-        atomic64_add(delta, &counter->count);
-        atomic64_sub(delta, &hwc->period_left);
+        return nr;
 }
 
-static void perf_swcounter_set_period(struct perf_counter *counter)
+static void perf_swcounter_overflow(struct perf_counter *counter,
+                                    int nmi, struct perf_sample_data *data)
 {
         struct hw_perf_counter *hwc = &counter->hw;
-        s64 left = atomic64_read(&hwc->period_left);
-        s64 period = hwc->sample_period;
+        u64 overflow;
 
-        if (unlikely(left <= -period)) {
-                left = period;
-                atomic64_set(&hwc->period_left, left);
-                hwc->last_period = period;
-        }
+        data->period = counter->hw.last_period;
+        overflow = perf_swcounter_set_period(counter);
 
-        if (unlikely(left <= 0)) {
-                left += period;
-                atomic64_add(period, &hwc->period_left);
-                hwc->last_period = period;
-        }
+        if (hwc->interrupts == MAX_INTERRUPTS)
+                return;
 
-        atomic64_set(&hwc->prev_count, -left);
-        atomic64_set(&hwc->count, -left);
+        for (; overflow; overflow--) {
+                if (perf_counter_overflow(counter, nmi, data)) {
+                        /*
+                         * We inhibit the overflow from happening when
+                         * hwc->interrupts == MAX_INTERRUPTS.
+                         */
+                        break;
+                }
+        }
 }
 
-static enum hrtimer_restart perf_swcounter_hrtimer(struct hrtimer *hrtimer)
+static void perf_swcounter_unthrottle(struct perf_counter *counter)
 {
-        enum hrtimer_restart ret = HRTIMER_RESTART;
-        struct perf_sample_data data;
-        struct perf_counter *counter;
-        u64 period;
-
-        counter = container_of(hrtimer, struct perf_counter, hw.hrtimer);
-        counter->pmu->read(counter);
-
-        data.addr = 0;
-        data.regs = get_irq_regs();
         /*
-         * In case we exclude kernel IPs or are somehow not in interrupt
-         * context, provide the next best thing, the user IP.
+         * Nothing to do, we already reset hwc->interrupts.
          */
-        if ((counter->attr.exclude_kernel || !data.regs) &&
-                        !counter->attr.exclude_user)
-                data.regs = task_pt_regs(current);
-
-        if (data.regs) {
-                if (perf_counter_overflow(counter, 0, &data))
-                        ret = HRTIMER_NORESTART;
-        }
-
-        period = max_t(u64, 10000, counter->hw.sample_period);
-        hrtimer_forward_now(hrtimer, ns_to_ktime(period));
-
-        return ret;
 }
 
-static void perf_swcounter_overflow(struct perf_counter *counter,
-                                    int nmi, struct pt_regs *regs, u64 addr)
+static void perf_swcounter_add(struct perf_counter *counter, u64 nr,
+                               int nmi, struct perf_sample_data *data)
 {
-        struct perf_sample_data data = {
-                .regs   = regs,
-                .addr   = addr,
-                .period = counter->hw.last_period,
-        };
+        struct hw_perf_counter *hwc = &counter->hw;
+
+        atomic64_add(nr, &counter->count);
 
-        perf_swcounter_update(counter);
-        perf_swcounter_set_period(counter);
-        if (perf_counter_overflow(counter, nmi, &data))
-                /* soft-disable the counter */
-                ;
+        if (!hwc->sample_period)
+                return;
+
+        if (!data->regs)
+                return;
 
+        if (!atomic64_add_negative(nr, &hwc->period_left))
+                perf_swcounter_overflow(counter, nmi, data);
 }
 
 static int perf_swcounter_is_counting(struct perf_counter *counter)
 {
-        struct perf_counter_context *ctx;
-        unsigned long flags;
-        int count;
-
+        /*
+         * The counter is active, we're good!
+         */
         if (counter->state == PERF_COUNTER_STATE_ACTIVE)
                 return 1;
 
+        /*
+         * The counter is off/error, not counting.
+         */
         if (counter->state != PERF_COUNTER_STATE_INACTIVE)
                 return 0;
 
         /*
-         * If the counter is inactive, it could be just because
-         * its task is scheduled out, or because it's in a group
-         * which could not go on the PMU.  We want to count in
-         * the first case but not the second.  If the context is
-         * currently active then an inactive software counter must
-         * be the second case.  If it's not currently active then
-         * we need to know whether the counter was active when the
-         * context was last active, which we can determine by
-         * comparing counter->tstamp_stopped with ctx->time.
-         *
-         * We are within an RCU read-side critical section,
-         * which protects the existence of *ctx.
+         * The counter is inactive, if the context is active
+         * we're part of a group that didn't make it on the 'pmu',
+         * not counting.
          */
-        ctx = counter->ctx;
-        spin_lock_irqsave(&ctx->lock, flags);
-        count = 1;
-        /* Re-check state now we have the lock */
-        if (counter->state < PERF_COUNTER_STATE_INACTIVE ||
-            counter->ctx->is_active ||
-            counter->tstamp_stopped < ctx->time)
-                count = 0;
-        spin_unlock_irqrestore(&ctx->lock, flags);
-        return count;
+        if (counter->ctx->is_active)
+                return 0;
+
+        /*
+         * We're inactive and the context is too, this means the
+         * task is scheduled out, we're counting events that happen
+         * to us, like migration events.
+         */
+        return 1;
 }
 
 static int perf_swcounter_match(struct perf_counter *counter,
@@ -3186,19 +3687,10 @@ static int perf_swcounter_match(struct perf_counter *counter,
         return 1;
 }
 
-static void perf_swcounter_add(struct perf_counter *counter, u64 nr,
-                               int nmi, struct pt_regs *regs, u64 addr)
-{
-        int neg = atomic64_add_negative(nr, &counter->hw.count);
-
-        if (counter->hw.sample_period && !neg && regs)
-                perf_swcounter_overflow(counter, nmi, regs, addr);
-}
-
 static void perf_swcounter_ctx_event(struct perf_counter_context *ctx,
-                                     enum perf_type_id type, u32 event,
-                                     u64 nr, int nmi, struct pt_regs *regs,
-                                     u64 addr)
+                                     enum perf_type_id type,
+                                     u32 event, u64 nr, int nmi,
+                                     struct perf_sample_data *data)
 {
         struct perf_counter *counter;
 
@@ -3207,8 +3699,8 @@ static void perf_swcounter_ctx_event(struct perf_counter_context *ctx,
 
         rcu_read_lock();
         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
-                if (perf_swcounter_match(counter, type, event, regs))
-                        perf_swcounter_add(counter, nr, nmi, regs, addr);
+                if (perf_swcounter_match(counter, type, event, data->regs))
+                        perf_swcounter_add(counter, nr, nmi, data);
         }
         rcu_read_unlock();
 }
@@ -3227,9 +3719,9 @@ static int *perf_swcounter_recursion_context(struct perf_cpu_context *cpuctx)
         return &cpuctx->recursion[0];
 }
 
-static void __perf_swcounter_event(enum perf_type_id type, u32 event,
-                                   u64 nr, int nmi, struct pt_regs *regs,
-                                   u64 addr)
+static void do_perf_swcounter_event(enum perf_type_id type, u32 event,
+                                    u64 nr, int nmi,
+                                    struct perf_sample_data *data)
 {
         struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
         int *recursion = perf_swcounter_recursion_context(cpuctx);
@@ -3242,7 +3734,7 @@ static void __perf_swcounter_event(enum perf_type_id type, u32 event,
         barrier();
 
         perf_swcounter_ctx_event(&cpuctx->ctx, type, event,
-                                 nr, nmi, regs, addr);
+                                 nr, nmi, data);
         rcu_read_lock();
         /*
          * doesn't really matter which of the child contexts the
@@ -3250,7 +3742,7 @@ static void __perf_swcounter_event(enum perf_type_id type, u32 event,
          */
         ctx = rcu_dereference(current->perf_counter_ctxp);
         if (ctx)
-                perf_swcounter_ctx_event(ctx, type, event, nr, nmi, regs, addr);
+                perf_swcounter_ctx_event(ctx, type, event, nr, nmi, data);
         rcu_read_unlock();
 
         barrier();
@@ -3260,35 +3752,79 @@ out:
         put_cpu_var(perf_cpu_context);
 }
 
-void
-perf_swcounter_event(u32 event, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
+void __perf_swcounter_event(u32 event, u64 nr, int nmi,
+                            struct pt_regs *regs, u64 addr)
 {
-        __perf_swcounter_event(PERF_TYPE_SOFTWARE, event, nr, nmi, regs, addr);
+        struct perf_sample_data data = {
+                .regs = regs,
+                .addr = addr,
+        };
+
+        do_perf_swcounter_event(PERF_TYPE_SOFTWARE, event, nr, nmi, &data);
 }
 
 static void perf_swcounter_read(struct perf_counter *counter)
 {
-        perf_swcounter_update(counter);
 }
 
 static int perf_swcounter_enable(struct perf_counter *counter)
 {
-        perf_swcounter_set_period(counter);
+        struct hw_perf_counter *hwc = &counter->hw;
+
+        if (hwc->sample_period) {
+                hwc->last_period = hwc->sample_period;
+                perf_swcounter_set_period(counter);
+        }
         return 0;
 }
 
 static void perf_swcounter_disable(struct perf_counter *counter)
 {
-        perf_swcounter_update(counter);
 }
 
 static const struct pmu perf_ops_generic = {
         .enable         = perf_swcounter_enable,
         .disable        = perf_swcounter_disable,
         .read           = perf_swcounter_read,
+        .unthrottle     = perf_swcounter_unthrottle,
 };
 
 /*
+ * hrtimer based swcounter callback
+ */
+
+static enum hrtimer_restart perf_swcounter_hrtimer(struct hrtimer *hrtimer)
+{
+        enum hrtimer_restart ret = HRTIMER_RESTART;
+        struct perf_sample_data data;
+        struct perf_counter *counter;
+        u64 period;
+
+        counter = container_of(hrtimer, struct perf_counter, hw.hrtimer);
+        counter->pmu->read(counter);
+
+        data.addr = 0;
+        data.regs = get_irq_regs();
+        /*
+         * In case we exclude kernel IPs or are somehow not in interrupt
+         * context, provide the next best thing, the user IP.
+         */
+        if ((counter->attr.exclude_kernel || !data.regs) &&
+                        !counter->attr.exclude_user)
+                data.regs = task_pt_regs(current);
+
+        if (data.regs) {
+                if (perf_counter_overflow(counter, 0, &data))
+                        ret = HRTIMER_NORESTART;
+        }
+
+        period = max_t(u64, 10000, counter->hw.sample_period);
+        hrtimer_forward_now(hrtimer, ns_to_ktime(period));
+
+        return ret;
+}
+
+/*
  * Software counter: cpu wall time clock
  */
 
@@ -3404,36 +3940,25 @@ static const struct pmu perf_ops_task_clock = {
         .read           = task_clock_perf_counter_read,
 };
 
-/*
- * Software counter: cpu migrations
- */
-void perf_counter_task_migration(struct task_struct *task, int cpu)
-{
-        struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
-        struct perf_counter_context *ctx;
-
-        perf_swcounter_ctx_event(&cpuctx->ctx, PERF_TYPE_SOFTWARE,
-                                 PERF_COUNT_SW_CPU_MIGRATIONS,
-                                 1, 1, NULL, 0);
-
-        ctx = perf_pin_task_context(task);
-        if (ctx) {
-                perf_swcounter_ctx_event(ctx, PERF_TYPE_SOFTWARE,
-                                         PERF_COUNT_SW_CPU_MIGRATIONS,
-                                         1, 1, NULL, 0);
-                perf_unpin_context(ctx);
-        }
-}
-
 #ifdef CONFIG_EVENT_PROFILE
-void perf_tpcounter_event(int event_id)
+void perf_tpcounter_event(int event_id, u64 addr, u64 count, void *record,
+                          int entry_size)
 {
-        struct pt_regs *regs = get_irq_regs();
+        struct perf_raw_record raw = {
+                .size = entry_size,
+                .data = record,
+        };
 
-        if (!regs)
-                regs = task_pt_regs(current);
+        struct perf_sample_data data = {
+                .regs = get_irq_regs(),
+                .addr = addr,
+                .raw = &raw,
+        };
 
-        __perf_swcounter_event(PERF_TYPE_TRACEPOINT, event_id, 1, 1, regs, 0);
+        if (!data.regs)
+                data.regs = task_pt_regs(current);
+
+        do_perf_swcounter_event(PERF_TYPE_TRACEPOINT, event_id, count, 1, &data);
 }
 EXPORT_SYMBOL_GPL(perf_tpcounter_event);
 
@@ -3442,16 +3967,21 @@ extern void ftrace_profile_disable(int);
 
 static void tp_perf_counter_destroy(struct perf_counter *counter)
 {
-        ftrace_profile_disable(perf_event_id(&counter->attr));
+        ftrace_profile_disable(counter->attr.config);
 }
 
 static const struct pmu *tp_perf_counter_init(struct perf_counter *counter)
 {
-        int event_id = perf_event_id(&counter->attr);
-        int ret;
+        /*
+         * Raw tracepoint data is a severe data leak, only allow root to
+         * have these.
+         */
+        if ((counter->attr.sample_type & PERF_SAMPLE_RAW) &&
+                        perf_paranoid_tracepoint_raw() &&
+                        !capable(CAP_SYS_ADMIN))
+                return ERR_PTR(-EPERM);
 
-        ret = ftrace_profile_enable(event_id);
-        if (ret)
+        if (ftrace_profile_enable(counter->attr.config))
                 return NULL;
 
         counter->destroy = tp_perf_counter_destroy;
@@ -3465,9 +3995,21 @@ static const struct pmu *tp_perf_counter_init(struct perf_counter *counter)
 }
 #endif
 
+atomic_t perf_swcounter_enabled[PERF_COUNT_SW_MAX];
+
+static void sw_perf_counter_destroy(struct perf_counter *counter)
+{
+        u64 event = counter->attr.config;
+
+        WARN_ON(counter->parent);
+
+        atomic_dec(&perf_swcounter_enabled[event]);
+}
+
 static const struct pmu *sw_perf_counter_init(struct perf_counter *counter)
 {
         const struct pmu *pmu = NULL;
+        u64 event = counter->attr.config;
 
         /*
          * Software counters (currently) can't in general distinguish
@@ -3476,7 +4018,7 @@ static const struct pmu *sw_perf_counter_init(struct perf_counter *counter)
          * to be kernel events, and page faults are never hypervisor
          * events.
          */
-        switch (counter->attr.config) {
+        switch (event) {
         case PERF_COUNT_SW_CPU_CLOCK:
                 pmu = &perf_ops_cpu_clock;
 
@@ -3497,6 +4039,10 @@ static const struct pmu *sw_perf_counter_init(struct perf_counter *counter)
         case PERF_COUNT_SW_PAGE_FAULTS_MAJ:
         case PERF_COUNT_SW_CONTEXT_SWITCHES:
         case PERF_COUNT_SW_CPU_MIGRATIONS:
+                if (!counter->parent) {
+                        atomic_inc(&perf_swcounter_enabled[event]);
+                        counter->destroy = sw_perf_counter_destroy;
+                }
                 pmu = &perf_ops_generic;
                 break;
         }
@@ -3512,6 +4058,7 @@ perf_counter_alloc(struct perf_counter_attr *attr,
                    int cpu,
                    struct perf_counter_context *ctx,
                    struct perf_counter *group_leader,
+                   struct perf_counter *parent_counter,
                    gfp_t gfpflags)
 {
         const struct pmu *pmu;
@@ -3547,6 +4094,8 @@ perf_counter_alloc(struct perf_counter_attr *attr,
         counter->ctx            = ctx;
         counter->oncpu          = -1;
 
+        counter->parent         = parent_counter;
+
         counter->ns             = get_pid_ns(current->nsproxy->pid_ns);
         counter->id             = atomic64_inc_return(&perf_counter_id);
 
@@ -3561,13 +4110,14 @@ perf_counter_alloc(struct perf_counter_attr *attr,
         hwc->sample_period = attr->sample_period;
         if (attr->freq && attr->sample_freq)
                 hwc->sample_period = 1;
+        hwc->last_period = hwc->sample_period;
 
         atomic64_set(&hwc->period_left, hwc->sample_period);
 
         /*
-         * we currently do not support PERF_SAMPLE_GROUP on inherited counters
+         * we currently do not support PERF_FORMAT_GROUP on inherited counters
          */
-        if (attr->inherit && (attr->sample_type & PERF_SAMPLE_GROUP))
+        if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
                 goto done;
 
         switch (attr->type) {
@@ -3604,11 +4154,15 @@ done:
 
         counter->pmu = pmu;
 
-        atomic_inc(&nr_counters);
-        if (counter->attr.mmap)
-                atomic_inc(&nr_mmap_counters);
-        if (counter->attr.comm)
-                atomic_inc(&nr_comm_counters);
+        if (!counter->parent) {
+                atomic_inc(&nr_counters);
+                if (counter->attr.mmap)
+                        atomic_inc(&nr_mmap_counters);
+                if (counter->attr.comm)
+                        atomic_inc(&nr_comm_counters);
+                if (counter->attr.task)
+                        atomic_inc(&nr_task_counters);
+        }
 
         return counter;
 }
@@ -3661,6 +4215,7 @@ static int perf_copy_attr(struct perf_counter_attr __user *uattr,
                         if (val)
                                 goto err_size;
                 }
+                size = sizeof(*attr);
         }
 
         ret = copy_from_user(attr, uattr, size);
@@ -3692,6 +4247,57 @@ err_size:
         goto out;
 }
 
+int perf_counter_set_output(struct perf_counter *counter, int output_fd)
+{
+        struct perf_counter *output_counter = NULL;
+        struct file *output_file = NULL;
+        struct perf_counter *old_output;
+        int fput_needed = 0;
+        int ret = -EINVAL;
+
+        if (!output_fd)
+                goto set;
+
+        output_file = fget_light(output_fd, &fput_needed);
+        if (!output_file)
+                return -EBADF;
+
+        if (output_file->f_op != &perf_fops)
+                goto out;
+
+        output_counter = output_file->private_data;
+
+        /* Don't chain output fds */
+        if (output_counter->output)
+                goto out;
+
+        /* Don't set an output fd when we already have an output channel */
+        if (counter->data)
+                goto out;
+
+        atomic_long_inc(&output_file->f_count);
+
+set:
+        mutex_lock(&counter->mmap_mutex);
+        old_output = counter->output;
+        rcu_assign_pointer(counter->output, output_counter);
+        mutex_unlock(&counter->mmap_mutex);
+
+        if (old_output) {
+                /*
+                 * we need to make sure no existing perf_output_*()
+                 * is still referencing this counter.
+                 */
+                synchronize_rcu();
+                fput(old_output->filp);
+        }
+
+        ret = 0;
+out:
+        fput_light(output_file, fput_needed);
+        return ret;
+}
+
 /**
  * sys_perf_counter_open - open a performance counter, associate it to a task/cpu
  *
@@ -3711,15 +4317,15 @@ SYSCALL_DEFINE5(perf_counter_open,
         struct file *group_file = NULL;
         int fput_needed = 0;
         int fput_needed2 = 0;
-        int ret;
+        int err;
 
         /* for future expandability... */
-        if (flags)
+        if (flags & ~(PERF_FLAG_FD_NO_GROUP | PERF_FLAG_FD_OUTPUT))
                 return -EINVAL;
 
-        ret = perf_copy_attr(attr_uptr, &attr);
-        if (ret)
-                return ret;
+        err = perf_copy_attr(attr_uptr, &attr);
+        if (err)
+                return err;
 
         if (!attr.exclude_kernel) {
                 if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
@@ -3742,8 +4348,8 @@ SYSCALL_DEFINE5(perf_counter_open,
          * Look up the group leader (we will attach this counter to it):
          */
         group_leader = NULL;
-        if (group_fd != -1) {
-                ret = -EINVAL;
+        if (group_fd != -1 && !(flags & PERF_FLAG_FD_NO_GROUP)) {
+                err = -EINVAL;
                 group_file = fget_light(group_fd, &fput_needed);
                 if (!group_file)
                         goto err_put_context;
@@ -3771,19 +4377,25 @@ SYSCALL_DEFINE5(perf_counter_open,
         }
 
         counter = perf_counter_alloc(&attr, cpu, ctx, group_leader,
-                                     GFP_KERNEL);
-        ret = PTR_ERR(counter);
+                                     NULL, GFP_KERNEL);
+        err = PTR_ERR(counter);
         if (IS_ERR(counter))
                 goto err_put_context;
 
-        ret = anon_inode_getfd("[perf_counter]", &perf_fops, counter, 0);
-        if (ret < 0)
+        err = anon_inode_getfd("[perf_counter]", &perf_fops, counter, 0);
+        if (err < 0)
                 goto err_free_put_context;
 
-        counter_file = fget_light(ret, &fput_needed2);
+        counter_file = fget_light(err, &fput_needed2);
         if (!counter_file)
                 goto err_free_put_context;
 
+        if (flags & PERF_FLAG_FD_OUTPUT) {
+                err = perf_counter_set_output(counter, group_fd);
+                if (err)
+                        goto err_fput_free_put_context;
+        }
+
         counter->filp = counter_file;
         WARN_ON_ONCE(ctx->parent_ctx);
         mutex_lock(&ctx->mutex);
@@ -3797,20 +4409,20 @@ SYSCALL_DEFINE5(perf_counter_open,
         list_add_tail(&counter->owner_entry, &current->perf_counter_list);
         mutex_unlock(&current->perf_counter_mutex);
 
+err_fput_free_put_context:
         fput_light(counter_file, fput_needed2);
 
-out_fput:
-        fput_light(group_file, fput_needed);
-
-        return ret;
-
 err_free_put_context:
-        kfree(counter);
+        if (err < 0)
+                kfree(counter);
 
 err_put_context:
-        put_ctx(ctx);
+        if (err < 0)
+                put_ctx(ctx);
 
-        goto out_fput;
+        fput_light(group_file, fput_needed);
+
+        return err;
 }
 
 /*
@@ -3837,7 +4449,8 @@ inherit_counter(struct perf_counter *parent_counter,
 
         child_counter = perf_counter_alloc(&parent_counter->attr,
                                            parent_counter->cpu, child_ctx,
-                                           group_leader, GFP_KERNEL);
+                                           group_leader, parent_counter,
+                                           GFP_KERNEL);
         if (IS_ERR(child_counter))
                 return child_counter;
         get_ctx(child_ctx);
@@ -3860,12 +4473,6 @@ inherit_counter(struct perf_counter *parent_counter,
          */
         add_counter_to_ctx(child_counter, child_ctx);
 
-        child_counter->parent = parent_counter;
-        /*
-         * inherit into child's child as well:
-         */
-        child_counter->attr.inherit = 1;
-
         /*
          * Get a reference to the parent filp - we will fput it
          * when the child counter exits. This is safe to do because
@@ -3909,10 +4516,14 @@ static int inherit_group(struct perf_counter *parent_counter,
 }
 
 static void sync_child_counter(struct perf_counter *child_counter,
-                               struct perf_counter *parent_counter)
+                               struct task_struct *child)
 {
+        struct perf_counter *parent_counter = child_counter->parent;
         u64 child_val;
 
+        if (child_counter->attr.inherit_stat)
+                perf_counter_read_event(child_counter, child);
+
         child_val = atomic64_read(&child_counter->count);
 
         /*
@@ -3941,7 +4552,8 @@ static void sync_child_counter(struct perf_counter *child_counter,
 
 static void
 __perf_counter_exit_task(struct perf_counter *child_counter,
-                         struct perf_counter_context *child_ctx)
+                         struct perf_counter_context *child_ctx,
+                         struct task_struct *child)
 {
         struct perf_counter *parent_counter;
 
@@ -3955,7 +4567,7 @@ __perf_counter_exit_task(struct perf_counter *child_counter,
          * counters need to be zapped - but otherwise linger.
          */
         if (parent_counter) {
-                sync_child_counter(child_counter, parent_counter);
+                sync_child_counter(child_counter, child);
                 free_counter(child_counter);
         }
 }
@@ -3969,8 +4581,10 @@ void perf_counter_exit_task(struct task_struct *child)
         struct perf_counter_context *child_ctx;
         unsigned long flags;
 
-        if (likely(!child->perf_counter_ctxp))
+        if (likely(!child->perf_counter_ctxp)) {
+                perf_counter_task(child, NULL, 0);
                 return;
+        }
 
         local_irq_save(flags);
         /*
@@ -3989,17 +4603,20 @@ void perf_counter_exit_task(struct task_struct *child)
          */
         spin_lock(&child_ctx->lock);
         child->perf_counter_ctxp = NULL;
-        if (child_ctx->parent_ctx) {
-                /*
-                 * This context is a clone; unclone it so it can't get
-                 * swapped to another process while we're removing all
-                 * the counters from it.
-                 */
-                put_ctx(child_ctx->parent_ctx);
-                child_ctx->parent_ctx = NULL;
-        }
-        spin_unlock(&child_ctx->lock);
-        local_irq_restore(flags);
+        /*
+         * If this context is a clone; unclone it so it can't get
+         * swapped to another process while we're removing all
+         * the counters from it.
+         */
+        unclone_ctx(child_ctx);
+        spin_unlock_irqrestore(&child_ctx->lock, flags);
+
+        /*
+         * Report the task dead after unscheduling the counters so that we
+         * won't get any samples after PERF_EVENT_EXIT. We can however still
+         * get a few PERF_EVENT_READ events.
+         */
+        perf_counter_task(child, child_ctx, 0);
 
         /*
          * We can recurse on the same lock type through:
@@ -4017,7 +4634,7 @@ void perf_counter_exit_task(struct task_struct *child)
 again:
         list_for_each_entry_safe(child_counter, tmp, &child_ctx->counter_list,
                                  list_entry)
-                __perf_counter_exit_task(child_counter, child_ctx);
+                __perf_counter_exit_task(child_counter, child_ctx, child);
 
         /*
          * If the last counter was a group counter, it will have appended all
@@ -4220,6 +4837,11 @@ perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
                 perf_counter_init_cpu(cpu);
                 break;
 
+        case CPU_ONLINE:
+        case CPU_ONLINE_FROZEN:
+                hw_perf_counter_setup_online(cpu);
+                break;
+
         case CPU_DOWN_PREPARE:
         case CPU_DOWN_PREPARE_FROZEN:
                 perf_counter_exit_cpu(cpu);
@@ -4244,6 +4866,8 @@ void __init perf_counter_init(void)
 {
         perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE,
                         (void *)(long)smp_processor_id());
+        perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_ONLINE,
+                        (void *)(long)smp_processor_id());
         register_cpu_notifier(&perf_cpu_nb);
 }
 
diff --git a/kernel/pid.c b/kernel/pid.c
index b2e5f78fd281..31310b5d3f50 100644
--- a/kernel/pid.c
+++ b/kernel/pid.c
@@ -378,26 +378,15 @@ EXPORT_SYMBOL(pid_task);
 /*
  * Must be called under rcu_read_lock() or with tasklist_lock read-held.
  */
-struct task_struct *find_task_by_pid_type_ns(int type, int nr,
-                struct pid_namespace *ns)
+struct task_struct *find_task_by_pid_ns(pid_t nr, struct pid_namespace *ns)
 {
-        return pid_task(find_pid_ns(nr, ns), type);
+        return pid_task(find_pid_ns(nr, ns), PIDTYPE_PID);
 }
 
-EXPORT_SYMBOL(find_task_by_pid_type_ns);
-
 struct task_struct *find_task_by_vpid(pid_t vnr)
 {
-        return find_task_by_pid_type_ns(PIDTYPE_PID, vnr,
-                        current->nsproxy->pid_ns);
-}
-EXPORT_SYMBOL(find_task_by_vpid);
-
-struct task_struct *find_task_by_pid_ns(pid_t nr, struct pid_namespace *ns)
-{
-        return find_task_by_pid_type_ns(PIDTYPE_PID, nr, ns);
+        return find_task_by_pid_ns(vnr, current->nsproxy->pid_ns);
 }
-EXPORT_SYMBOL(find_task_by_pid_ns);
 
 struct pid *get_task_pid(struct task_struct *task, enum pid_type type)
 {
diff --git a/kernel/pid_namespace.c b/kernel/pid_namespace.c
index 2d1001b4858d..821722ae58a7 100644
--- a/kernel/pid_namespace.c
+++ b/kernel/pid_namespace.c
@@ -67,9 +67,10 @@ err_alloc:
         return NULL;
 }
 
-static struct pid_namespace *create_pid_namespace(unsigned int level)
+static struct pid_namespace *create_pid_namespace(struct pid_namespace *parent_pid_ns)
 {
         struct pid_namespace *ns;
+        unsigned int level = parent_pid_ns->level + 1;
         int i;
 
         ns = kmem_cache_zalloc(pid_ns_cachep, GFP_KERNEL);
@@ -86,6 +87,7 @@ static struct pid_namespace *create_pid_namespace(unsigned int level)
 
         kref_init(&ns->kref);
         ns->level = level;
+        ns->parent = get_pid_ns(parent_pid_ns);
 
         set_bit(0, ns->pidmap[0].page);
         atomic_set(&ns->pidmap[0].nr_free, BITS_PER_PAGE - 1);
@@ -114,25 +116,11 @@ static void destroy_pid_namespace(struct pid_namespace *ns)
 
 struct pid_namespace *copy_pid_ns(unsigned long flags, struct pid_namespace *old_ns)
 {
-        struct pid_namespace *new_ns;
-
-        BUG_ON(!old_ns);
-        new_ns = get_pid_ns(old_ns);
         if (!(flags & CLONE_NEWPID))
-                goto out;
-
-        new_ns = ERR_PTR(-EINVAL);
+                return get_pid_ns(old_ns);
         if (flags & CLONE_THREAD)
-                goto out_put;
-
-        new_ns = create_pid_namespace(old_ns->level + 1);
-        if (!IS_ERR(new_ns))
-                new_ns->parent = get_pid_ns(old_ns);
-
-out_put:
-        put_pid_ns(old_ns);
-out:
-        return new_ns;
+                return ERR_PTR(-EINVAL);
+        return create_pid_namespace(old_ns);
 }
 
 void free_pid_ns(struct kref *kref)
diff --git a/kernel/posix-cpu-timers.c b/kernel/posix-cpu-timers.c
index bece7c0b67b2..e33a21cb9407 100644
--- a/kernel/posix-cpu-timers.c
+++ b/kernel/posix-cpu-timers.c
@@ -521,11 +521,12 @@ void posix_cpu_timers_exit(struct task_struct *tsk)
 }
 void posix_cpu_timers_exit_group(struct task_struct *tsk)
 {
-        struct task_cputime cputime;
+        struct signal_struct *const sig = tsk->signal;
 
-        thread_group_cputimer(tsk, &cputime);
         cleanup_timers(tsk->signal->cpu_timers,
-                       cputime.utime, cputime.stime, cputime.sum_exec_runtime);
+                       cputime_add(tsk->utime, sig->utime),
+                       cputime_add(tsk->stime, sig->stime),
+                       tsk->se.sum_exec_runtime + sig->sum_sched_runtime);
 }
 
 static void clear_dead_task(struct k_itimer *timer, union cpu_time_count now)
diff --git a/kernel/posix-timers.c b/kernel/posix-timers.c
index 052ec4d195c7..d089d052c4a9 100644
--- a/kernel/posix-timers.c
+++ b/kernel/posix-timers.c
@@ -202,6 +202,12 @@ static int no_timer_create(struct k_itimer *new_timer)
         return -EOPNOTSUPP;
 }
 
+static int no_nsleep(const clockid_t which_clock, int flags,
+                     struct timespec *tsave, struct timespec __user *rmtp)
+{
+        return -EOPNOTSUPP;
+}
+
 /*
  * Return nonzero if we know a priori this clockid_t value is bogus.
  */
@@ -254,6 +260,7 @@ static __init int init_posix_timers(void)
                 .clock_get = posix_get_monotonic_raw,
                 .clock_set = do_posix_clock_nosettime,
                 .timer_create = no_timer_create,
+                .nsleep = no_nsleep,
         };
 
         register_posix_clock(CLOCK_REALTIME, &clock_realtime);
diff --git a/kernel/power/Kconfig b/kernel/power/Kconfig
index 72067cbdb37f..91e09d3b2eb2 100644
--- a/kernel/power/Kconfig
+++ b/kernel/power/Kconfig
@@ -208,3 +208,17 @@ config APM_EMULATION
           random kernel OOPSes or reboots that don't seem to be related to
           anything, try disabling/enabling this option (or disabling/enabling
           APM in your BIOS).
+
+config PM_RUNTIME
+        bool "Run-time PM core functionality"
+        depends on PM
+        ---help---
+          Enable functionality allowing I/O devices to be put into energy-saving
+          (low power) states at run time (or autosuspended) after a specified
+          period of inactivity and woken up in response to a hardware-generated
+          wake-up event or a driver's request.
+
+          Hardware support is generally required for this functionality to work
+          and the bus type drivers of the buses the devices are on are
+          responsible for the actual handling of the autosuspend requests and
+          wake-up events.
diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c
index 81d2e7464893..04b3a83d686f 100644
--- a/kernel/power/hibernate.c
+++ b/kernel/power/hibernate.c
@@ -298,8 +298,8 @@ int hibernation_snapshot(int platform_mode)
         if (error)
                 return error;
 
-        /* Free memory before shutting down devices. */
-        error = swsusp_shrink_memory();
+        /* Preallocate image memory before shutting down devices. */
+        error = hibernate_preallocate_memory();
         if (error)
                 goto Close;
 
@@ -315,6 +315,10 @@ int hibernation_snapshot(int platform_mode)
         /* Control returns here after successful restore */
 
  Resume_devices:
+        /* We may need to release the preallocated image pages here. */
+        if (error || !in_suspend)
+                swsusp_free();
+
         dpm_resume_end(in_suspend ?
                 (error ? PMSG_RECOVER : PMSG_THAW) : PMSG_RESTORE);
         resume_console();
@@ -460,11 +464,11 @@ int hibernation_platform_enter(void)
 
         error = hibernation_ops->prepare();
         if (error)
-                goto Platofrm_finish;
+                goto Platform_finish;
 
         error = disable_nonboot_cpus();
         if (error)
-                goto Platofrm_finish;
+                goto Platform_finish;
 
         local_irq_disable();
         sysdev_suspend(PMSG_HIBERNATE);
@@ -476,7 +480,7 @@ int hibernation_platform_enter(void)
          * We don't need to reenable the nonboot CPUs or resume consoles, since
          * the system is going to be halted anyway.
          */
- Platofrm_finish:
+ Platform_finish:
         hibernation_ops->finish();
 
         dpm_suspend_noirq(PMSG_RESTORE);
@@ -578,7 +582,10 @@ int hibernate(void)
                 goto Thaw;
 
         error = hibernation_snapshot(hibernation_mode == HIBERNATION_PLATFORM);
-        if (in_suspend && !error) {
+        if (error)
+                goto Thaw;
+
+        if (in_suspend) {
                 unsigned int flags = 0;
 
                 if (hibernation_mode == HIBERNATION_PLATFORM)
@@ -590,8 +597,8 @@ int hibernate(void)
                         power_down();
         } else {
                 pr_debug("PM: Image restored successfully.\n");
-                swsusp_free();
         }
+
  Thaw:
         thaw_processes();
  Finish:
diff --git a/kernel/power/main.c b/kernel/power/main.c
index f710e36930cc..347d2cc88cd0 100644
--- a/kernel/power/main.c
+++ b/kernel/power/main.c
@@ -11,6 +11,7 @@
 #include <linux/kobject.h>
 #include <linux/string.h>
 #include <linux/resume-trace.h>
+#include <linux/workqueue.h>
 
 #include "power.h"
 
@@ -217,8 +218,24 @@ static struct attribute_group attr_group = {
         .attrs = g,
 };
 
+#ifdef CONFIG_PM_RUNTIME
+struct workqueue_struct *pm_wq;
+
+static int __init pm_start_workqueue(void)
+{
+        pm_wq = create_freezeable_workqueue("pm");
+
+        return pm_wq ? 0 : -ENOMEM;
+}
+#else
+static inline int pm_start_workqueue(void) { return 0; }
+#endif
+
 static int __init pm_init(void)
 {
+        int error = pm_start_workqueue();
+        if (error)
+                return error;
         power_kobj = kobject_create_and_add("power", NULL);
         if (!power_kobj)
                 return -ENOMEM;
diff --git a/kernel/power/power.h b/kernel/power/power.h
index 26d5a26f82e3..46c5a26630a3 100644
--- a/kernel/power/power.h
+++ b/kernel/power/power.h
@@ -74,7 +74,7 @@ extern asmlinkage int swsusp_arch_resume(void);
 
 extern int create_basic_memory_bitmaps(void);
 extern void free_basic_memory_bitmaps(void);
-extern int swsusp_shrink_memory(void);
+extern int hibernate_preallocate_memory(void);
 
 /**
  *      Auxiliary structure used for reading the snapshot image data and
diff --git a/kernel/power/process.c b/kernel/power/process.c
index ca634019497a..da2072d73811 100644
--- a/kernel/power/process.c
+++ b/kernel/power/process.c
@@ -117,9 +117,12 @@ int freeze_processes(void)
         if (error)
                 goto Exit;
         printk("done.");
+
+        oom_killer_disable();
  Exit:
         BUG_ON(in_atomic());
         printk("\n");
+
         return error;
 }
 
@@ -145,6 +148,8 @@ static void thaw_tasks(bool nosig_only)
 
 void thaw_processes(void)
 {
+        oom_killer_enable();
+
         printk("Restarting tasks ... ");
         thaw_tasks(true);
         thaw_tasks(false);
diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c
index 523a451b45d3..97955b0e44f4 100644
--- a/kernel/power/snapshot.c
+++ b/kernel/power/snapshot.c
@@ -233,7 +233,7 @@ static void *chain_alloc(struct chain_allocator *ca, unsigned int size)
 
 #define BM_END_OF_MAP   (~0UL)
 
-#define BM_BITS_PER_BLOCK       (PAGE_SIZE << 3)
+#define BM_BITS_PER_BLOCK       (PAGE_SIZE * BITS_PER_BYTE)
 
 struct bm_block {
         struct list_head hook;  /* hook into a list of bitmap blocks */
@@ -275,7 +275,7 @@ static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free);
 
 /**
  *      create_bm_block_list - create a list of block bitmap objects
- *      @nr_blocks - number of blocks to allocate
+ *      @pages - number of pages to track
  *      @list - list to put the allocated blocks into
  *      @ca - chain allocator to be used for allocating memory
  */
@@ -853,7 +853,7 @@ static unsigned int count_highmem_pages(void)
         struct zone *zone;
         unsigned int n = 0;
 
-        for_each_zone(zone) {
+        for_each_populated_zone(zone) {
                 unsigned long pfn, max_zone_pfn;
 
                 if (!is_highmem(zone))
@@ -916,7 +916,7 @@ static unsigned int count_data_pages(void)
         unsigned long pfn, max_zone_pfn;
         unsigned int n = 0;
 
-        for_each_zone(zone) {
+        for_each_populated_zone(zone) {
                 if (is_highmem(zone))
                         continue;
 
@@ -1010,7 +1010,7 @@ copy_data_pages(struct memory_bitmap *copy_bm, struct memory_bitmap *orig_bm)
         struct zone *zone;
         unsigned long pfn;
 
-        for_each_zone(zone) {
+        for_each_populated_zone(zone) {
                 unsigned long max_zone_pfn;
 
                 mark_free_pages(zone);
@@ -1033,6 +1033,25 @@ copy_data_pages(struct memory_bitmap *copy_bm, struct memory_bitmap *orig_bm)
 static unsigned int nr_copy_pages;
 /* Number of pages needed for saving the original pfns of the image pages */
 static unsigned int nr_meta_pages;
+/*
+ * Numbers of normal and highmem page frames allocated for hibernation image
+ * before suspending devices.
+ */
+unsigned int alloc_normal, alloc_highmem;
+/*
+ * Memory bitmap used for marking saveable pages (during hibernation) or
+ * hibernation image pages (during restore)
+ */
+static struct memory_bitmap orig_bm;
+/*
+ * Memory bitmap used during hibernation for marking allocated page frames that
+ * will contain copies of saveable pages.  During restore it is initially used
+ * for marking hibernation image pages, but then the set bits from it are
+ * duplicated in @orig_bm and it is released.  On highmem systems it is next
+ * used for marking "safe" highmem pages, but it has to be reinitialized for
+ * this purpose.
+ */
+static struct memory_bitmap copy_bm;
 
 /**
  *      swsusp_free - free pages allocated for the suspend.
@@ -1046,7 +1065,7 @@ void swsusp_free(void)
         struct zone *zone;
         unsigned long pfn, max_zone_pfn;
 
-        for_each_zone(zone) {
+        for_each_populated_zone(zone) {
                 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
                 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
                         if (pfn_valid(pfn)) {
@@ -1064,74 +1083,286 @@ void swsusp_free(void)
         nr_meta_pages = 0;
         restore_pblist = NULL;
         buffer = NULL;
+        alloc_normal = 0;
+        alloc_highmem = 0;
 }
 
+/* Helper functions used for the shrinking of memory. */
+
+#define GFP_IMAGE       (GFP_KERNEL | __GFP_NOWARN)
+
 /**
- *      swsusp_shrink_memory -  Try to free as much memory as needed
- *
- *      ... but do not OOM-kill anyone
+ * preallocate_image_pages - Allocate a number of pages for hibernation image
+ * @nr_pages: Number of page frames to allocate.
+ * @mask: GFP flags to use for the allocation.
  *
- *      Notice: all userland should be stopped before it is called, or
- *      livelock is possible.
+ * Return value: Number of page frames actually allocated
+ */
+static unsigned long preallocate_image_pages(unsigned long nr_pages, gfp_t mask)
+{
+        unsigned long nr_alloc = 0;
+
+        while (nr_pages > 0) {
+                struct page *page;
+
+                page = alloc_image_page(mask);
+                if (!page)
+                        break;
+                memory_bm_set_bit(&copy_bm, page_to_pfn(page));
+                if (PageHighMem(page))
+                        alloc_highmem++;
+                else
+                        alloc_normal++;
+                nr_pages--;
+                nr_alloc++;
+        }
+
+        return nr_alloc;
+}
+
+static unsigned long preallocate_image_memory(unsigned long nr_pages)
+{
+        return preallocate_image_pages(nr_pages, GFP_IMAGE);
+}
+
+#ifdef CONFIG_HIGHMEM
+static unsigned long preallocate_image_highmem(unsigned long nr_pages)
+{
+        return preallocate_image_pages(nr_pages, GFP_IMAGE | __GFP_HIGHMEM);
+}
+
+/**
+ *  __fraction - Compute (an approximation of) x * (multiplier / base)
  */
+static unsigned long __fraction(u64 x, u64 multiplier, u64 base)
+{
+        x *= multiplier;
+        do_div(x, base);
+        return (unsigned long)x;
+}
+
+static unsigned long preallocate_highmem_fraction(unsigned long nr_pages,
+                                                unsigned long highmem,
+                                                unsigned long total)
+{
+        unsigned long alloc = __fraction(nr_pages, highmem, total);
 
-#define SHRINK_BITE     10000
-static inline unsigned long __shrink_memory(long tmp)
+        return preallocate_image_pages(alloc, GFP_IMAGE | __GFP_HIGHMEM);
+}
+#else /* CONFIG_HIGHMEM */
+static inline unsigned long preallocate_image_highmem(unsigned long nr_pages)
 {
-        if (tmp > SHRINK_BITE)
-                tmp = SHRINK_BITE;
-        return shrink_all_memory(tmp);
+        return 0;
 }
 
-int swsusp_shrink_memory(void)
+static inline unsigned long preallocate_highmem_fraction(unsigned long nr_pages,
+                                                unsigned long highmem,
+                                                unsigned long total)
+{
+        return 0;
+}
+#endif /* CONFIG_HIGHMEM */
+
+/**
+ * free_unnecessary_pages - Release preallocated pages not needed for the image
+ */
+static void free_unnecessary_pages(void)
+{
+        unsigned long save_highmem, to_free_normal, to_free_highmem;
+
+        to_free_normal = alloc_normal - count_data_pages();
+        save_highmem = count_highmem_pages();
+        if (alloc_highmem > save_highmem) {
+                to_free_highmem = alloc_highmem - save_highmem;
+        } else {
+                to_free_highmem = 0;
+                to_free_normal -= save_highmem - alloc_highmem;
+        }
+
+        memory_bm_position_reset(&copy_bm);
+
+        while (to_free_normal > 0 && to_free_highmem > 0) {
+                unsigned long pfn = memory_bm_next_pfn(&copy_bm);
+                struct page *page = pfn_to_page(pfn);
+
+                if (PageHighMem(page)) {
+                        if (!to_free_highmem)
+                                continue;
+                        to_free_highmem--;
+                        alloc_highmem--;
+                } else {
+                        if (!to_free_normal)
+                                continue;
+                        to_free_normal--;
+                        alloc_normal--;
+                }
+                memory_bm_clear_bit(&copy_bm, pfn);
+                swsusp_unset_page_forbidden(page);
+                swsusp_unset_page_free(page);
+                __free_page(page);
+        }
+}
+
+/**
+ * minimum_image_size - Estimate the minimum acceptable size of an image
+ * @saveable: Number of saveable pages in the system.
+ *
+ * We want to avoid attempting to free too much memory too hard, so estimate the
+ * minimum acceptable size of a hibernation image to use as the lower limit for
+ * preallocating memory.
+ *
+ * We assume that the minimum image size should be proportional to
+ *
+ * [number of saveable pages] - [number of pages that can be freed in theory]
+ *
+ * where the second term is the sum of (1) reclaimable slab pages, (2) active
+ * and (3) inactive anonymouns pages, (4) active and (5) inactive file pages,
+ * minus mapped file pages.
+ */
+static unsigned long minimum_image_size(unsigned long saveable)
+{
+        unsigned long size;
+
+        size = global_page_state(NR_SLAB_RECLAIMABLE)
+                + global_page_state(NR_ACTIVE_ANON)
+                + global_page_state(NR_INACTIVE_ANON)
+                + global_page_state(NR_ACTIVE_FILE)
+                + global_page_state(NR_INACTIVE_FILE)
+                - global_page_state(NR_FILE_MAPPED);
+
+        return saveable <= size ? 0 : saveable - size;
+}
+
+/**
+ * hibernate_preallocate_memory - Preallocate memory for hibernation image
+ *
+ * To create a hibernation image it is necessary to make a copy of every page
+ * frame in use.  We also need a number of page frames to be free during
+ * hibernation for allocations made while saving the image and for device
+ * drivers, in case they need to allocate memory from their hibernation
+ * callbacks (these two numbers are given by PAGES_FOR_IO and SPARE_PAGES,
+ * respectively, both of which are rough estimates).  To make this happen, we
+ * compute the total number of available page frames and allocate at least
+ *
+ * ([page frames total] + PAGES_FOR_IO + [metadata pages]) / 2 + 2 * SPARE_PAGES
+ *
+ * of them, which corresponds to the maximum size of a hibernation image.
+ *
+ * If image_size is set below the number following from the above formula,
+ * the preallocation of memory is continued until the total number of saveable
+ * pages in the system is below the requested image size or the minimum
+ * acceptable image size returned by minimum_image_size(), whichever is greater.
+ */
+int hibernate_preallocate_memory(void)
 {
-        long tmp;
         struct zone *zone;
-        unsigned long pages = 0;
-        unsigned int i = 0;
-        char *p = "-\\|/";
+        unsigned long saveable, size, max_size, count, highmem, pages = 0;
+        unsigned long alloc, save_highmem, pages_highmem;
         struct timeval start, stop;
+        int error;
 
-        printk(KERN_INFO "PM: Shrinking memory...  ");
+        printk(KERN_INFO "PM: Preallocating image memory... ");
         do_gettimeofday(&start);
-        do {
-                long size, highmem_size;
-
-                highmem_size = count_highmem_pages();
-                size = count_data_pages() + PAGES_FOR_IO + SPARE_PAGES;
-                tmp = size;
-                size += highmem_size;
-                for_each_populated_zone(zone) {
-                        tmp += snapshot_additional_pages(zone);
-                        if (is_highmem(zone)) {
-                                highmem_size -=
-                                        zone_page_state(zone, NR_FREE_PAGES);
-                        } else {
-                                tmp -= zone_page_state(zone, NR_FREE_PAGES);
-                                tmp += zone->lowmem_reserve[ZONE_NORMAL];
-                        }
-                }
 
-                if (highmem_size < 0)
-                        highmem_size = 0;
+        error = memory_bm_create(&orig_bm, GFP_IMAGE, PG_ANY);
+        if (error)
+                goto err_out;
 
-                tmp += highmem_size;
-                if (tmp > 0) {
-                        tmp = __shrink_memory(tmp);
-                        if (!tmp)
-                                return -ENOMEM;
-                        pages += tmp;
-                } else if (size > image_size / PAGE_SIZE) {
-                        tmp = __shrink_memory(size - (image_size / PAGE_SIZE));
-                        pages += tmp;
-                }
-                printk("\b%c", p[i++%4]);
-        } while (tmp > 0);
+        error = memory_bm_create(&copy_bm, GFP_IMAGE, PG_ANY);
+        if (error)
+                goto err_out;
+
+        alloc_normal = 0;
+        alloc_highmem = 0;
+
+        /* Count the number of saveable data pages. */
+        save_highmem = count_highmem_pages();
+        saveable = count_data_pages();
+
+        /*
+         * Compute the total number of page frames we can use (count) and the
+         * number of pages needed for image metadata (size).
+         */
+        count = saveable;
+        saveable += save_highmem;
+        highmem = save_highmem;
+        size = 0;
+        for_each_populated_zone(zone) {
+                size += snapshot_additional_pages(zone);
+                if (is_highmem(zone))
+                        highmem += zone_page_state(zone, NR_FREE_PAGES);
+                else
+                        count += zone_page_state(zone, NR_FREE_PAGES);
+        }
+        count += highmem;
+        count -= totalreserve_pages;
+
+        /* Compute the maximum number of saveable pages to leave in memory. */
+        max_size = (count - (size + PAGES_FOR_IO)) / 2 - 2 * SPARE_PAGES;
+        size = DIV_ROUND_UP(image_size, PAGE_SIZE);
+        if (size > max_size)
+                size = max_size;
+        /*
+         * If the maximum is not less than the current number of saveable pages
+         * in memory, allocate page frames for the image and we're done.
+         */
+        if (size >= saveable) {
+                pages = preallocate_image_highmem(save_highmem);
+                pages += preallocate_image_memory(saveable - pages);
+                goto out;
+        }
+
+        /* Estimate the minimum size of the image. */
+        pages = minimum_image_size(saveable);
+        if (size < pages)
+                size = min_t(unsigned long, pages, max_size);
+
+        /*
+         * Let the memory management subsystem know that we're going to need a
+         * large number of page frames to allocate and make it free some memory.
+         * NOTE: If this is not done, performance will be hurt badly in some
+         * test cases.
+         */
+        shrink_all_memory(saveable - size);
+
+        /*
+         * The number of saveable pages in memory was too high, so apply some
+         * pressure to decrease it.  First, make room for the largest possible
+         * image and fail if that doesn't work.  Next, try to decrease the size
+         * of the image as much as indicated by 'size' using allocations from
+         * highmem and non-highmem zones separately.
+         */
+        pages_highmem = preallocate_image_highmem(highmem / 2);
+        alloc = (count - max_size) - pages_highmem;
+        pages = preallocate_image_memory(alloc);
+        if (pages < alloc)
+                goto err_out;
+        size = max_size - size;
+        alloc = size;
+        size = preallocate_highmem_fraction(size, highmem, count);
+        pages_highmem += size;
+        alloc -= size;
+        pages += preallocate_image_memory(alloc);
+        pages += pages_highmem;
+
+        /*
+         * We only need as many page frames for the image as there are saveable
+         * pages in memory, but we have allocated more.  Release the excessive
+         * ones now.
+         */
+        free_unnecessary_pages();
+
+ out:
         do_gettimeofday(&stop);
-        printk("\bdone (%lu pages freed)\n", pages);
-        swsusp_show_speed(&start, &stop, pages, "Freed");
+        printk(KERN_CONT "done (allocated %lu pages)\n", pages);
+        swsusp_show_speed(&start, &stop, pages, "Allocated");
 
         return 0;
+
+ err_out:
+        printk(KERN_CONT "\n");
+        swsusp_free();
+        return -ENOMEM;
 }
 
 #ifdef CONFIG_HIGHMEM
@@ -1142,7 +1373,7 @@ int swsusp_shrink_memory(void)
 
 static unsigned int count_pages_for_highmem(unsigned int nr_highmem)
 {
-        unsigned int free_highmem = count_free_highmem_pages();
+        unsigned int free_highmem = count_free_highmem_pages() + alloc_highmem;
 
         if (free_highmem >= nr_highmem)
                 nr_highmem = 0;
@@ -1164,19 +1395,17 @@ count_pages_for_highmem(unsigned int nr_highmem) { return 0; }
 static int enough_free_mem(unsigned int nr_pages, unsigned int nr_highmem)
 {
         struct zone *zone;
-        unsigned int free = 0, meta = 0;
+        unsigned int free = alloc_normal;
 
-        for_each_zone(zone) {
-                meta += snapshot_additional_pages(zone);
+        for_each_populated_zone(zone)
                 if (!is_highmem(zone))
                         free += zone_page_state(zone, NR_FREE_PAGES);
-        }
 
         nr_pages += count_pages_for_highmem(nr_highmem);
-        pr_debug("PM: Normal pages needed: %u + %u + %u, available pages: %u\n",
-                nr_pages, PAGES_FOR_IO, meta, free);
+        pr_debug("PM: Normal pages needed: %u + %u, available pages: %u\n",
+                nr_pages, PAGES_FOR_IO, free);
 
-        return free > nr_pages + PAGES_FOR_IO + meta;
+        return free > nr_pages + PAGES_FOR_IO;
 }
 
 #ifdef CONFIG_HIGHMEM
@@ -1198,7 +1427,7 @@ static inline int get_highmem_buffer(int safe_needed)
  */
 
 static inline unsigned int
-alloc_highmem_image_pages(struct memory_bitmap *bm, unsigned int nr_highmem)
+alloc_highmem_pages(struct memory_bitmap *bm, unsigned int nr_highmem)
 {
         unsigned int to_alloc = count_free_highmem_pages();
 
@@ -1218,7 +1447,7 @@ alloc_highmem_image_pages(struct memory_bitmap *bm, unsigned int nr_highmem)
 static inline int get_highmem_buffer(int safe_needed) { return 0; }
 
 static inline unsigned int
-alloc_highmem_image_pages(struct memory_bitmap *bm, unsigned int n) { return 0; }
+alloc_highmem_pages(struct memory_bitmap *bm, unsigned int n) { return 0; }
 #endif /* CONFIG_HIGHMEM */
 
 /**
@@ -1237,51 +1466,36 @@ static int
 swsusp_alloc(struct memory_bitmap *orig_bm, struct memory_bitmap *copy_bm,
                 unsigned int nr_pages, unsigned int nr_highmem)
 {
-        int error;
-
-        error = memory_bm_create(orig_bm, GFP_ATOMIC | __GFP_COLD, PG_ANY);
-        if (error)
-                goto Free;
-
-        error = memory_bm_create(copy_bm, GFP_ATOMIC | __GFP_COLD, PG_ANY);
-        if (error)
-                goto Free;
+        int error = 0;
 
         if (nr_highmem > 0) {
                 error = get_highmem_buffer(PG_ANY);
                 if (error)
-                        goto Free;
-
-                nr_pages += alloc_highmem_image_pages(copy_bm, nr_highmem);
+                        goto err_out;
+                if (nr_highmem > alloc_highmem) {
+                        nr_highmem -= alloc_highmem;
+                        nr_pages += alloc_highmem_pages(copy_bm, nr_highmem);
+                }
         }
-        while (nr_pages-- > 0) {
-                struct page *page = alloc_image_page(GFP_ATOMIC | __GFP_COLD);
-
-                if (!page)
-                        goto Free;
+        if (nr_pages > alloc_normal) {
+                nr_pages -= alloc_normal;
+                while (nr_pages-- > 0) {
+                        struct page *page;
 
-                memory_bm_set_bit(copy_bm, page_to_pfn(page));
+                        page = alloc_image_page(GFP_ATOMIC | __GFP_COLD);
+                        if (!page)
+                                goto err_out;
+                        memory_bm_set_bit(copy_bm, page_to_pfn(page));
+                }
         }
+
         return 0;
 
- Free:
+ err_out:
         swsusp_free();
-        return -ENOMEM;
+        return error;
 }
 
-/* Memory bitmap used for marking saveable pages (during suspend) or the
- * suspend image pages (during resume)
- */
-static struct memory_bitmap orig_bm;
-/* Memory bitmap used on suspend for marking allocated pages that will contain
- * the copies of saveable pages.  During resume it is initially used for
- * marking the suspend image pages, but then its set bits are duplicated in
- * @orig_bm and it is released.  Next, on systems with high memory, it may be
- * used for marking "safe" highmem pages, but it has to be reinitialized for
- * this purpose.
- */
-static struct memory_bitmap copy_bm;
-
 asmlinkage int swsusp_save(void)
 {
         unsigned int nr_pages, nr_highmem;
@@ -1474,7 +1688,7 @@ static int mark_unsafe_pages(struct memory_bitmap *bm)
         unsigned long pfn, max_zone_pfn;
 
         /* Clear page flags */
-        for_each_zone(zone) {
+        for_each_populated_zone(zone) {
                 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
                 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
                         if (pfn_valid(pfn))
diff --git a/kernel/power/user.c b/kernel/power/user.c
index ed97375daae9..bf0014d6a5f0 100644
--- a/kernel/power/user.c
+++ b/kernel/power/user.c
@@ -23,7 +23,6 @@
 #include <linux/console.h>
 #include <linux/cpu.h>
 #include <linux/freezer.h>
-#include <linux/smp_lock.h>
 #include <scsi/scsi_scan.h>
 
 #include <asm/uaccess.h>
diff --git a/kernel/printk.c b/kernel/printk.c
index b4d97b54c1ec..602033acd6c7 100644
--- a/kernel/printk.c
+++ b/kernel/printk.c
@@ -37,6 +37,12 @@
 #include <asm/uaccess.h>
 
 /*
+ * for_each_console() allows you to iterate on each console
+ */
+#define for_each_console(con) \
+        for (con = console_drivers; con != NULL; con = con->next)
+
+/*
  * Architectures can override it:
  */
 void asmlinkage __attribute__((weak)) early_printk(const char *fmt, ...)
@@ -61,6 +67,8 @@ int console_printk[4] = {
         DEFAULT_CONSOLE_LOGLEVEL,       /* default_console_loglevel */
 };
 
+static int saved_console_loglevel = -1;
+
 /*
  * Low level drivers may need that to know if they can schedule in
  * their unblank() callback or not. So let's export it.
@@ -372,10 +380,15 @@ int do_syslog(int type, char __user *buf, int len)
                 logged_chars = 0;
                 break;
         case 6:         /* Disable logging to console */
+                if (saved_console_loglevel == -1)
+                        saved_console_loglevel = console_loglevel;
                 console_loglevel = minimum_console_loglevel;
                 break;
         case 7:         /* Enable logging to console */
-                console_loglevel = default_console_loglevel;
+                if (saved_console_loglevel != -1) {
+                        console_loglevel = saved_console_loglevel;
+                        saved_console_loglevel = -1;
+                }
                 break;
         case 8:         /* Set level of messages printed to console */
                 error = -EINVAL;
@@ -384,6 +397,8 @@ int do_syslog(int type, char __user *buf, int len)
                 if (len < minimum_console_loglevel)
                         len = minimum_console_loglevel;
                 console_loglevel = len;
+                /* Implicitly re-enable logging to console */
+                saved_console_loglevel = -1;
                 error = 0;
                 break;
         case 9:         /* Number of chars in the log buffer */
@@ -412,7 +427,7 @@ static void __call_console_drivers(unsigned start, unsigned end)
 {
         struct console *con;
 
-        for (con = console_drivers; con; con = con->next) {
+        for_each_console(con) {
                 if ((con->flags & CON_ENABLED) && con->write &&
                                 (cpu_online(smp_processor_id()) ||
                                 (con->flags & CON_ANYTIME)))
@@ -544,7 +559,7 @@ static int have_callable_console(void)
 {
         struct console *con;
 
-        for (con = console_drivers; con; con = con->next)
+        for_each_console(con)
                 if (con->flags & CON_ANYTIME)
                         return 1;
 
@@ -1060,12 +1075,6 @@ void __sched console_conditional_schedule(void)
 }
 EXPORT_SYMBOL(console_conditional_schedule);
 
-void console_print(const char *s)
-{
-        printk(KERN_EMERG "%s", s);
-}
-EXPORT_SYMBOL(console_print);
-
 void console_unblank(void)
 {
         struct console *c;
@@ -1082,7 +1091,7 @@ void console_unblank(void)
 
         console_locked = 1;
         console_may_schedule = 0;
-        for (c = console_drivers; c != NULL; c = c->next)
+        for_each_console(c)
                 if ((c->flags & CON_ENABLED) && c->unblank)
                         c->unblank();
         release_console_sem();
@@ -1097,7 +1106,7 @@ struct tty_driver *console_device(int *index)
         struct tty_driver *driver = NULL;
 
         acquire_console_sem();
-        for (c = console_drivers; c != NULL; c = c->next) {
+        for_each_console(c) {
                 if (!c->device)
                         continue;
                 driver = c->device(c, index);
@@ -1134,25 +1143,49 @@ EXPORT_SYMBOL(console_start);
  * to register the console printing procedure with printk() and to
  * print any messages that were printed by the kernel before the
  * console driver was initialized.
+ *
+ * This can happen pretty early during the boot process (because of
+ * early_printk) - sometimes before setup_arch() completes - be careful
+ * of what kernel features are used - they may not be initialised yet.
+ *
+ * There are two types of consoles - bootconsoles (early_printk) and
+ * "real" consoles (everything which is not a bootconsole) which are
+ * handled differently.
+ *  - Any number of bootconsoles can be registered at any time.
+ *  - As soon as a "real" console is registered, all bootconsoles
+ *    will be unregistered automatically.
+ *  - Once a "real" console is registered, any attempt to register a
+ *    bootconsoles will be rejected
  */
-void register_console(struct console *console)
+void register_console(struct console *newcon)
 {
         int i;
         unsigned long flags;
-        struct console *bootconsole = NULL;
+        struct console *bcon = NULL;
 
-        if (console_drivers) {
-                if (console->flags & CON_BOOT)
-                        return;
-                if (console_drivers->flags & CON_BOOT)
-                        bootconsole = console_drivers;
+        /*
+         * before we register a new CON_BOOT console, make sure we don't
+         * already have a valid console
+         */
+        if (console_drivers && newcon->flags & CON_BOOT) {
+                /* find the last or real console */
+                for_each_console(bcon) {
+                        if (!(bcon->flags & CON_BOOT)) {
+                                printk(KERN_INFO "Too late to register bootconsole %s%d\n",
+                                        newcon->name, newcon->index);
+                                return;
+                        }
+                }
         }
 
-        if (preferred_console < 0 || bootconsole || !console_drivers)
+        if (console_drivers && console_drivers->flags & CON_BOOT)
+                bcon = console_drivers;
+
+        if (preferred_console < 0 || bcon || !console_drivers)
                 preferred_console = selected_console;
 
-        if (console->early_setup)
-                console->early_setup();
+        if (newcon->early_setup)
+                newcon->early_setup();
 
         /*
          *      See if we want to use this console driver. If we
@@ -1160,13 +1193,13 @@ void register_console(struct console *console)
          *      that registers here.
          */
         if (preferred_console < 0) {
-                if (console->index < 0)
-                        console->index = 0;
-                if (console->setup == NULL ||
-                    console->setup(console, NULL) == 0) {
-                        console->flags |= CON_ENABLED;
-                        if (console->device) {
-                                console->flags |= CON_CONSDEV;
+                if (newcon->index < 0)
+                        newcon->index = 0;
+                if (newcon->setup == NULL ||
+                    newcon->setup(newcon, NULL) == 0) {
+                        newcon->flags |= CON_ENABLED;
+                        if (newcon->device) {
+                                newcon->flags |= CON_CONSDEV;
                                 preferred_console = 0;
                         }
                 }
@@ -1178,64 +1211,62 @@ void register_console(struct console *console)
          */
         for (i = 0; i < MAX_CMDLINECONSOLES && console_cmdline[i].name[0];
                         i++) {
-                if (strcmp(console_cmdline[i].name, console->name) != 0)
+                if (strcmp(console_cmdline[i].name, newcon->name) != 0)
                         continue;
-                if (console->index >= 0 &&
-                    console->index != console_cmdline[i].index)
+                if (newcon->index >= 0 &&
+                    newcon->index != console_cmdline[i].index)
                         continue;
-                if (console->index < 0)
-                        console->index = console_cmdline[i].index;
+                if (newcon->index < 0)
+                        newcon->index = console_cmdline[i].index;
 #ifdef CONFIG_A11Y_BRAILLE_CONSOLE
                 if (console_cmdline[i].brl_options) {
-                        console->flags |= CON_BRL;
-                        braille_register_console(console,
+                        newcon->flags |= CON_BRL;
+                        braille_register_console(newcon,
                                         console_cmdline[i].index,
                                         console_cmdline[i].options,
                                         console_cmdline[i].brl_options);
                         return;
                 }
 #endif
-                if (console->setup &&
-                    console->setup(console, console_cmdline[i].options) != 0)
+                if (newcon->setup &&
+                    newcon->setup(newcon, console_cmdline[i].options) != 0)
                         break;
-                console->flags |= CON_ENABLED;
-                console->index = console_cmdline[i].index;
+                newcon->flags |= CON_ENABLED;
+                newcon->index = console_cmdline[i].index;
                 if (i == selected_console) {
-                        console->flags |= CON_CONSDEV;
+                        newcon->flags |= CON_CONSDEV;
                         preferred_console = selected_console;
                 }
                 break;
         }
 
-        if (!(console->flags & CON_ENABLED))
+        if (!(newcon->flags & CON_ENABLED))
                 return;
 
-        if (bootconsole && (console->flags & CON_CONSDEV)) {
-                printk(KERN_INFO "console handover: boot [%s%d] -> real [%s%d]\n",
-                       bootconsole->name, bootconsole->index,
-                       console->name, console->index);
-                unregister_console(bootconsole);
-                console->flags &= ~CON_PRINTBUFFER;
-        } else {
-                printk(KERN_INFO "console [%s%d] enabled\n",
-                       console->name, console->index);
-        }
+        /*
+         * If we have a bootconsole, and are switching to a real console,
+         * don't print everything out again, since when the boot console, and
+         * the real console are the same physical device, it's annoying to
+         * see the beginning boot messages twice
+         */
+        if (bcon && ((newcon->flags & (CON_CONSDEV | CON_BOOT)) == CON_CONSDEV))
+                newcon->flags &= ~CON_PRINTBUFFER;
 
         /*
          *      Put this console in the list - keep the
          *      preferred driver at the head of the list.
          */
         acquire_console_sem();
-        if ((console->flags & CON_CONSDEV) || console_drivers == NULL) {
-                console->next = console_drivers;
-                console_drivers = console;
-                if (console->next)
-                        console->next->flags &= ~CON_CONSDEV;
+        if ((newcon->flags & CON_CONSDEV) || console_drivers == NULL) {
+                newcon->next = console_drivers;
+                console_drivers = newcon;
+                if (newcon->next)
+                        newcon->next->flags &= ~CON_CONSDEV;
         } else {
-                console->next = console_drivers->next;
-                console_drivers->next = console;
+                newcon->next = console_drivers->next;
+                console_drivers->next = newcon;
         }
-        if (console->flags & CON_PRINTBUFFER) {
+        if (newcon->flags & CON_PRINTBUFFER) {
                 /*
                  * release_console_sem() will print out the buffered messages
                  * for us.
@@ -1245,6 +1276,28 @@ void register_console(struct console *console)
                 spin_unlock_irqrestore(&logbuf_lock, flags);
         }
         release_console_sem();
+
+        /*
+         * By unregistering the bootconsoles after we enable the real console
+         * we get the "console xxx enabled" message on all the consoles -
+         * boot consoles, real consoles, etc - this is to ensure that end
+         * users know there might be something in the kernel's log buffer that
+         * went to the bootconsole (that they do not see on the real console)
+         */
+        if (bcon && ((newcon->flags & (CON_CONSDEV | CON_BOOT)) == CON_CONSDEV)) {
+                /* we need to iterate through twice, to make sure we print
+                 * everything out, before we unregister the console(s)
+                 */
+                printk(KERN_INFO "console [%s%d] enabled, bootconsole disabled\n",
+                        newcon->name, newcon->index);
+                for_each_console(bcon)
+                        if (bcon->flags & CON_BOOT)
+                                unregister_console(bcon);
+        } else {
+                printk(KERN_INFO "%sconsole [%s%d] enabled\n",
+                        (newcon->flags & CON_BOOT) ? "boot" : "" ,
+                        newcon->name, newcon->index);
+        }
 }
 EXPORT_SYMBOL(register_console);
 
@@ -1287,11 +1340,13 @@ EXPORT_SYMBOL(unregister_console);
 
 static int __init disable_boot_consoles(void)
 {
-        if (console_drivers != NULL) {
-                if (console_drivers->flags & CON_BOOT) {
+        struct console *con;
+
+        for_each_console(con) {
+                if (con->flags & CON_BOOT) {
                         printk(KERN_INFO "turn off boot console %s%d\n",
-                                console_drivers->name, console_drivers->index);
-                        return unregister_console(console_drivers);
+                                con->name, con->index);
+                        unregister_console(con);
                 }
         }
         return 0;
diff --git a/kernel/profile.c b/kernel/profile.c
index 28cf26ad2d24..419250ebec4d 100644
--- a/kernel/profile.c
+++ b/kernel/profile.c
@@ -117,11 +117,12 @@ int __ref profile_init(void)
 
         cpumask_copy(prof_cpu_mask, cpu_possible_mask);
 
-        prof_buffer = kzalloc(buffer_bytes, GFP_KERNEL);
+        prof_buffer = kzalloc(buffer_bytes, GFP_KERNEL|__GFP_NOWARN);
         if (prof_buffer)
                 return 0;
 
-        prof_buffer = alloc_pages_exact(buffer_bytes, GFP_KERNEL|__GFP_ZERO);
+        prof_buffer = alloc_pages_exact(buffer_bytes,
+                                        GFP_KERNEL|__GFP_ZERO|__GFP_NOWARN);
         if (prof_buffer)
                 return 0;
 
@@ -365,7 +366,7 @@ static int __cpuinit profile_cpu_callback(struct notifier_block *info,
                 node = cpu_to_node(cpu);
                 per_cpu(cpu_profile_flip, cpu) = 0;
                 if (!per_cpu(cpu_profile_hits, cpu)[1]) {
-                        page = alloc_pages_node(node,
+                        page = alloc_pages_exact_node(node,
                                         GFP_KERNEL | __GFP_ZERO,
                                         0);
                         if (!page)
@@ -373,7 +374,7 @@ static int __cpuinit profile_cpu_callback(struct notifier_block *info,
                         per_cpu(cpu_profile_hits, cpu)[1] = page_address(page);
                 }
                 if (!per_cpu(cpu_profile_hits, cpu)[0]) {
-                        page = alloc_pages_node(node,
+                        page = alloc_pages_exact_node(node,
                                         GFP_KERNEL | __GFP_ZERO,
                                         0);
                         if (!page)
@@ -564,14 +565,14 @@ static int create_hash_tables(void)
                 int node = cpu_to_node(cpu);
                 struct page *page;
 
-                page = alloc_pages_node(node,
+                page = alloc_pages_exact_node(node,
                                 GFP_KERNEL | __GFP_ZERO | GFP_THISNODE,
                                 0);
                 if (!page)
                         goto out_cleanup;
                 per_cpu(cpu_profile_hits, cpu)[1]
                                 = (struct profile_hit *)page_address(page);
-                page = alloc_pages_node(node,
+                page = alloc_pages_exact_node(node,
                                 GFP_KERNEL | __GFP_ZERO | GFP_THISNODE,
                                 0);
                 if (!page)
diff --git a/kernel/ptrace.c b/kernel/ptrace.c
index f6d8b8cb5e34..307c285af59e 100644
--- a/kernel/ptrace.c
+++ b/kernel/ptrace.c
@@ -152,7 +152,7 @@ int __ptrace_may_access(struct task_struct *task, unsigned int mode)
         if (!dumpable && !capable(CAP_SYS_PTRACE))
                 return -EPERM;
 
-        return security_ptrace_may_access(task, mode);
+        return security_ptrace_access_check(task, mode);
 }
 
 bool ptrace_may_access(struct task_struct *task, unsigned int mode)
@@ -167,67 +167,82 @@ bool ptrace_may_access(struct task_struct *task, unsigned int mode)
 int ptrace_attach(struct task_struct *task)
 {
         int retval;
-        unsigned long flags;
 
         audit_ptrace(task);
 
         retval = -EPERM;
+        if (unlikely(task->flags & PF_KTHREAD))
+                goto out;
         if (same_thread_group(task, current))
                 goto out;
 
-        /* Protect the target's credential calculations against our
+        /*
+         * Protect exec's credential calculations against our interference;
          * interference; SUID, SGID and LSM creds get determined differently
          * under ptrace.
          */
-        retval = mutex_lock_interruptible(&task->cred_guard_mutex);
-        if (retval  < 0)
+        retval = -ERESTARTNOINTR;
+        if (mutex_lock_interruptible(&task->cred_guard_mutex))
                 goto out;
 
-        retval = -EPERM;
-repeat:
-        /*
-         * Nasty, nasty.
-         *
-         * We want to hold both the task-lock and the
-         * tasklist_lock for writing at the same time.
-         * But that's against the rules (tasklist_lock
-         * is taken for reading by interrupts on other
-         * cpu's that may have task_lock).
-         */
         task_lock(task);
-        if (!write_trylock_irqsave(&tasklist_lock, flags)) {
-                task_unlock(task);
-                do {
-                        cpu_relax();
-                } while (!write_can_lock(&tasklist_lock));
-                goto repeat;
-        }
-
-        if (!task->mm)
-                goto bad;
-        /* the same process cannot be attached many times */
-        if (task->ptrace & PT_PTRACED)
-                goto bad;
         retval = __ptrace_may_access(task, PTRACE_MODE_ATTACH);
+        task_unlock(task);
         if (retval)
-                goto bad;
+                goto unlock_creds;
 
-        /* Go */
-        task->ptrace |= PT_PTRACED;
+        write_lock_irq(&tasklist_lock);
+        retval = -EPERM;
+        if (unlikely(task->exit_state))
+                goto unlock_tasklist;
+        if (task->ptrace)
+                goto unlock_tasklist;
+
+        task->ptrace = PT_PTRACED;
         if (capable(CAP_SYS_PTRACE))
                 task->ptrace |= PT_PTRACE_CAP;
 
         __ptrace_link(task, current);
-
         send_sig_info(SIGSTOP, SEND_SIG_FORCED, task);
-bad:
-        write_unlock_irqrestore(&tasklist_lock, flags);
-        task_unlock(task);
+
+        retval = 0;
+unlock_tasklist:
+        write_unlock_irq(&tasklist_lock);
+unlock_creds:
         mutex_unlock(&task->cred_guard_mutex);
 out:
         return retval;
 }
 
+/**
+ * ptrace_traceme  --  helper for PTRACE_TRACEME
+ *
+ * Performs checks and sets PT_PTRACED.
+ * Should be used by all ptrace implementations for PTRACE_TRACEME.
+ */
+int ptrace_traceme(void)
+{
+        int ret = -EPERM;
+
+        write_lock_irq(&tasklist_lock);
+        /* Are we already being traced? */
+        if (!current->ptrace) {
+                ret = security_ptrace_traceme(current->parent);
+                /*
+                 * Check PF_EXITING to ensure ->real_parent has not passed
+                 * exit_ptrace(). Otherwise we don't report the error but
+                 * pretend ->real_parent untraces us right after return.
+                 */
+                if (!ret && !(current->real_parent->flags & PF_EXITING)) {
+                        current->ptrace = PT_PTRACED;
+                        __ptrace_link(current, current->real_parent);
+                }
+        }
+        write_unlock_irq(&tasklist_lock);
+
+        return ret;
+}
+
 /*
  * Called with irqs disabled, returns true if childs should reap themselves.
  */
@@ -409,37 +424,33 @@ static int ptrace_setoptions(struct task_struct *child, long data)
 
 static int ptrace_getsiginfo(struct task_struct *child, siginfo_t *info)
 {
+        unsigned long flags;
         int error = -ESRCH;
 
-        read_lock(&tasklist_lock);
-        if (likely(child->sighand != NULL)) {
+        if (lock_task_sighand(child, &flags)) {
                 error = -EINVAL;
-                spin_lock_irq(&child->sighand->siglock);
                 if (likely(child->last_siginfo != NULL)) {
                         *info = *child->last_siginfo;
                         error = 0;
                 }
-                spin_unlock_irq(&child->sighand->siglock);
+                unlock_task_sighand(child, &flags);
         }
-        read_unlock(&tasklist_lock);
         return error;
 }
 
 static int ptrace_setsiginfo(struct task_struct *child, const siginfo_t *info)
 {
+        unsigned long flags;
         int error = -ESRCH;
 
-        read_lock(&tasklist_lock);
-        if (likely(child->sighand != NULL)) {
+        if (lock_task_sighand(child, &flags)) {
                 error = -EINVAL;
-                spin_lock_irq(&child->sighand->siglock);
                 if (likely(child->last_siginfo != NULL)) {
                         *child->last_siginfo = *info;
                         error = 0;
                 }
-                spin_unlock_irq(&child->sighand->siglock);
+                unlock_task_sighand(child, &flags);
         }
-        read_unlock(&tasklist_lock);
         return error;
 }
 
@@ -566,72 +577,16 @@ int ptrace_request(struct task_struct *child, long request,
         return ret;
 }
 
-/**
- * ptrace_traceme  --  helper for PTRACE_TRACEME
- *
- * Performs checks and sets PT_PTRACED.
- * Should be used by all ptrace implementations for PTRACE_TRACEME.
- */
-int ptrace_traceme(void)
-{
-        int ret = -EPERM;
-
-        /*
-         * Are we already being traced?
-         */
-repeat:
-        task_lock(current);
-        if (!(current->ptrace & PT_PTRACED)) {
-                /*
-                 * See ptrace_attach() comments about the locking here.
-                 */
-                unsigned long flags;
-                if (!write_trylock_irqsave(&tasklist_lock, flags)) {
-                        task_unlock(current);
-                        do {
-                                cpu_relax();
-                        } while (!write_can_lock(&tasklist_lock));
-                        goto repeat;
-                }
-
-                ret = security_ptrace_traceme(current->parent);
-
-                /*
-                 * Check PF_EXITING to ensure ->real_parent has not passed
-                 * exit_ptrace(). Otherwise we don't report the error but
-                 * pretend ->real_parent untraces us right after return.
-                 */
-                if (!ret && !(current->real_parent->flags & PF_EXITING)) {
-                        current->ptrace |= PT_PTRACED;
-                        __ptrace_link(current, current->real_parent);
-                }
-
-                write_unlock_irqrestore(&tasklist_lock, flags);
-        }
-        task_unlock(current);
-        return ret;
-}
-
-/**
- * ptrace_get_task_struct  --  grab a task struct reference for ptrace
- * @pid:       process id to grab a task_struct reference of
- *
- * This function is a helper for ptrace implementations.  It checks
- * permissions and then grabs a task struct for use of the actual
- * ptrace implementation.
- *
- * Returns the task_struct for @pid or an ERR_PTR() on failure.
- */
-struct task_struct *ptrace_get_task_struct(pid_t pid)
+static struct task_struct *ptrace_get_task_struct(pid_t pid)
 {
         struct task_struct *child;
 
-        read_lock(&tasklist_lock);
+        rcu_read_lock();
         child = find_task_by_vpid(pid);
         if (child)
                 get_task_struct(child);
+        rcu_read_unlock();
 
-        read_unlock(&tasklist_lock);
         if (!child)
                 return ERR_PTR(-ESRCH);
         return child;
diff --git a/kernel/rcuclassic.c b/kernel/rcuclassic.c
deleted file mode 100644
index 0f2b0b311304..000000000000
--- a/kernel/rcuclassic.c
+++ /dev/null
@@ -1,807 +0,0 @@
-/*
- * Read-Copy Update mechanism for mutual exclusion
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
- *
- * Copyright IBM Corporation, 2001
- *
- * Authors: Dipankar Sarma <dipankar@in.ibm.com>
- *          Manfred Spraul <manfred@colorfullife.com>
- *
- * Based on the original work by Paul McKenney <paulmck@us.ibm.com>
- * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
- * Papers:
- * http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf
- * http://lse.sourceforge.net/locking/rclock_OLS.2001.05.01c.sc.pdf (OLS2001)
- *
- * For detailed explanation of Read-Copy Update mechanism see -
- *              Documentation/RCU
- *
- */
-#include <linux/types.h>
-#include <linux/kernel.h>
-#include <linux/init.h>
-#include <linux/spinlock.h>
-#include <linux/smp.h>
-#include <linux/rcupdate.h>
-#include <linux/interrupt.h>
-#include <linux/sched.h>
-#include <asm/atomic.h>
-#include <linux/bitops.h>
-#include <linux/module.h>
-#include <linux/completion.h>
-#include <linux/moduleparam.h>
-#include <linux/percpu.h>
-#include <linux/notifier.h>
-#include <linux/cpu.h>
-#include <linux/mutex.h>
-#include <linux/time.h>
-
-#ifdef CONFIG_DEBUG_LOCK_ALLOC
-static struct lock_class_key rcu_lock_key;
-struct lockdep_map rcu_lock_map =
-        STATIC_LOCKDEP_MAP_INIT("rcu_read_lock", &rcu_lock_key);
-EXPORT_SYMBOL_GPL(rcu_lock_map);
-#endif
-
-
-/* Definition for rcupdate control block. */
-static struct rcu_ctrlblk rcu_ctrlblk = {
-        .cur = -300,
-        .completed = -300,
-        .pending = -300,
-        .lock = __SPIN_LOCK_UNLOCKED(&rcu_ctrlblk.lock),
-        .cpumask = CPU_BITS_NONE,
-};
-
-static struct rcu_ctrlblk rcu_bh_ctrlblk = {
-        .cur = -300,
-        .completed = -300,
-        .pending = -300,
-        .lock = __SPIN_LOCK_UNLOCKED(&rcu_bh_ctrlblk.lock),
-        .cpumask = CPU_BITS_NONE,
-};
-
-static DEFINE_PER_CPU(struct rcu_data, rcu_data);
-static DEFINE_PER_CPU(struct rcu_data, rcu_bh_data);
-
-/*
- * Increment the quiescent state counter.
- * The counter is a bit degenerated: We do not need to know
- * how many quiescent states passed, just if there was at least
- * one since the start of the grace period. Thus just a flag.
- */
-void rcu_qsctr_inc(int cpu)
-{
-        struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
-        rdp->passed_quiesc = 1;
-}
-
-void rcu_bh_qsctr_inc(int cpu)
-{
-        struct rcu_data *rdp = &per_cpu(rcu_bh_data, cpu);
-        rdp->passed_quiesc = 1;
-}
-
-static int blimit = 10;
-static int qhimark = 10000;
-static int qlowmark = 100;
-
-#ifdef CONFIG_SMP
-static void force_quiescent_state(struct rcu_data *rdp,
-                        struct rcu_ctrlblk *rcp)
-{
-        int cpu;
-        unsigned long flags;
-
-        set_need_resched();
-        spin_lock_irqsave(&rcp->lock, flags);
-        if (unlikely(!rcp->signaled)) {
-                rcp->signaled = 1;
-                /*
-                 * Don't send IPI to itself. With irqs disabled,
-                 * rdp->cpu is the current cpu.
-                 *
-                 * cpu_online_mask is updated by the _cpu_down()
-                 * using __stop_machine(). Since we're in irqs disabled
-                 * section, __stop_machine() is not exectuting, hence
-                 * the cpu_online_mask is stable.
-                 *
-                 * However,  a cpu might have been offlined _just_ before
-                 * we disabled irqs while entering here.
-                 * And rcu subsystem might not yet have handled the CPU_DEAD
-                 * notification, leading to the offlined cpu's bit
-                 * being set in the rcp->cpumask.
-                 *
-                 * Hence cpumask = (rcp->cpumask & cpu_online_mask) to prevent
-                 * sending smp_reschedule() to an offlined CPU.
-                 */
-                for_each_cpu_and(cpu,
-                                  to_cpumask(rcp->cpumask), cpu_online_mask) {
-                        if (cpu != rdp->cpu)
-                                smp_send_reschedule(cpu);
-                }
-        }
-        spin_unlock_irqrestore(&rcp->lock, flags);
-}
-#else
-static inline void force_quiescent_state(struct rcu_data *rdp,
-                        struct rcu_ctrlblk *rcp)
-{
-        set_need_resched();
-}
-#endif
-
-static void __call_rcu(struct rcu_head *head, struct rcu_ctrlblk *rcp,
-                struct rcu_data *rdp)
-{
-        long batch;
-
-        head->next = NULL;
-        smp_mb(); /* Read of rcu->cur must happen after any change by caller. */
-
-        /*
-         * Determine the batch number of this callback.
-         *
-         * Using ACCESS_ONCE to avoid the following error when gcc eliminates
-         * local variable "batch" and emits codes like this:
-         *      1) rdp->batch = rcp->cur + 1 # gets old value
-         *      ......
-         *      2)rcu_batch_after(rcp->cur + 1, rdp->batch) # gets new value
-         * then [*nxttail[0], *nxttail[1]) may contain callbacks
-         * that batch# = rdp->batch, see the comment of struct rcu_data.
-         */
-        batch = ACCESS_ONCE(rcp->cur) + 1;
-
-        if (rdp->nxtlist && rcu_batch_after(batch, rdp->batch)) {
-                /* process callbacks */
-                rdp->nxttail[0] = rdp->nxttail[1];
-                rdp->nxttail[1] = rdp->nxttail[2];
-                if (rcu_batch_after(batch - 1, rdp->batch))
-                        rdp->nxttail[0] = rdp->nxttail[2];
-        }
-
-        rdp->batch = batch;
-        *rdp->nxttail[2] = head;
-        rdp->nxttail[2] = &head->next;
-
-        if (unlikely(++rdp->qlen > qhimark)) {
-                rdp->blimit = INT_MAX;
-                force_quiescent_state(rdp, &rcu_ctrlblk);
-        }
-}
-
-#ifdef CONFIG_RCU_CPU_STALL_DETECTOR
-
-static void record_gp_stall_check_time(struct rcu_ctrlblk *rcp)
-{
-        rcp->gp_start = jiffies;
-        rcp->jiffies_stall = jiffies + RCU_SECONDS_TILL_STALL_CHECK;
-}
-
-static void print_other_cpu_stall(struct rcu_ctrlblk *rcp)
-{
-        int cpu;
-        long delta;
-        unsigned long flags;
-
-        /* Only let one CPU complain about others per time interval. */
-
-        spin_lock_irqsave(&rcp->lock, flags);
-        delta = jiffies - rcp->jiffies_stall;
-        if (delta < 2 || rcp->cur != rcp->completed) {
-                spin_unlock_irqrestore(&rcp->lock, flags);
-                return;
-        }
-        rcp->jiffies_stall = jiffies + RCU_SECONDS_TILL_STALL_RECHECK;
-        spin_unlock_irqrestore(&rcp->lock, flags);
-
-        /* OK, time to rat on our buddy... */
-
-        printk(KERN_ERR "INFO: RCU detected CPU stalls:");
-        for_each_possible_cpu(cpu) {
-                if (cpumask_test_cpu(cpu, to_cpumask(rcp->cpumask)))
-                        printk(" %d", cpu);
-        }
-        printk(" (detected by %d, t=%ld jiffies)\n",
-               smp_processor_id(), (long)(jiffies - rcp->gp_start));
-}
-
-static void print_cpu_stall(struct rcu_ctrlblk *rcp)
-{
-        unsigned long flags;
-
-        printk(KERN_ERR "INFO: RCU detected CPU %d stall (t=%lu/%lu jiffies)\n",
-                        smp_processor_id(), jiffies,
-                        jiffies - rcp->gp_start);
-        dump_stack();
-        spin_lock_irqsave(&rcp->lock, flags);
-        if ((long)(jiffies - rcp->jiffies_stall) >= 0)
-                rcp->jiffies_stall =
-                        jiffies + RCU_SECONDS_TILL_STALL_RECHECK;
-        spin_unlock_irqrestore(&rcp->lock, flags);
-        set_need_resched();  /* kick ourselves to get things going. */
-}
-
-static void check_cpu_stall(struct rcu_ctrlblk *rcp)
-{
-        long delta;
-
-        delta = jiffies - rcp->jiffies_stall;
-        if (cpumask_test_cpu(smp_processor_id(), to_cpumask(rcp->cpumask)) &&
-                delta >= 0) {
-
-                /* We haven't checked in, so go dump stack. */
-                print_cpu_stall(rcp);
-
-        } else if (rcp->cur != rcp->completed && delta >= 2) {
-
-                /* They had two seconds to dump stack, so complain. */
-                print_other_cpu_stall(rcp);
-        }
-}
-
-#else /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
-
-static void record_gp_stall_check_time(struct rcu_ctrlblk *rcp)
-{
-}
-
-static inline void check_cpu_stall(struct rcu_ctrlblk *rcp)
-{
-}
-
-#endif /* #else #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
-
-/**
- * call_rcu - Queue an RCU callback for invocation after a grace period.
- * @head: structure to be used for queueing the RCU updates.
- * @func: actual update function to be invoked after the grace period
- *
- * The update function will be invoked some time after a full grace
- * period elapses, in other words after all currently executing RCU
- * read-side critical sections have completed.  RCU read-side critical
- * sections are delimited by rcu_read_lock() and rcu_read_unlock(),
- * and may be nested.
- */
-void call_rcu(struct rcu_head *head,
-                                void (*func)(struct rcu_head *rcu))
-{
-        unsigned long flags;
-
-        head->func = func;
-        local_irq_save(flags);
-        __call_rcu(head, &rcu_ctrlblk, &__get_cpu_var(rcu_data));
-        local_irq_restore(flags);
-}
-EXPORT_SYMBOL_GPL(call_rcu);
-
-/**
- * call_rcu_bh - Queue an RCU for invocation after a quicker grace period.
- * @head: structure to be used for queueing the RCU updates.
- * @func: actual update function to be invoked after the grace period
- *
- * The update function will be invoked some time after a full grace
- * period elapses, in other words after all currently executing RCU
- * read-side critical sections have completed. call_rcu_bh() assumes
- * that the read-side critical sections end on completion of a softirq
- * handler. This means that read-side critical sections in process
- * context must not be interrupted by softirqs. This interface is to be
- * used when most of the read-side critical sections are in softirq context.
- * RCU read-side critical sections are delimited by rcu_read_lock() and
- * rcu_read_unlock(), * if in interrupt context or rcu_read_lock_bh()
- * and rcu_read_unlock_bh(), if in process context. These may be nested.
- */
-void call_rcu_bh(struct rcu_head *head,
-                                void (*func)(struct rcu_head *rcu))
-{
-        unsigned long flags;
-
-        head->func = func;
-        local_irq_save(flags);
-        __call_rcu(head, &rcu_bh_ctrlblk, &__get_cpu_var(rcu_bh_data));
-        local_irq_restore(flags);
-}
-EXPORT_SYMBOL_GPL(call_rcu_bh);
-
-/*
- * Return the number of RCU batches processed thus far.  Useful
- * for debug and statistics.
- */
-long rcu_batches_completed(void)
-{
-        return rcu_ctrlblk.completed;
-}
-EXPORT_SYMBOL_GPL(rcu_batches_completed);
-
-/*
- * Return the number of RCU batches processed thus far.  Useful
- * for debug and statistics.
- */
-long rcu_batches_completed_bh(void)
-{
-        return rcu_bh_ctrlblk.completed;
-}
-EXPORT_SYMBOL_GPL(rcu_batches_completed_bh);
-
-/* Raises the softirq for processing rcu_callbacks. */
-static inline void raise_rcu_softirq(void)
-{
-        raise_softirq(RCU_SOFTIRQ);
-}
-
-/*
- * Invoke the completed RCU callbacks. They are expected to be in
- * a per-cpu list.
- */
-static void rcu_do_batch(struct rcu_data *rdp)
-{
-        unsigned long flags;
-        struct rcu_head *next, *list;
-        int count = 0;
-
-        list = rdp->donelist;
-        while (list) {
-                next = list->next;
-                prefetch(next);
-                list->func(list);
-                list = next;
-                if (++count >= rdp->blimit)
-                        break;
-        }
-        rdp->donelist = list;
-
-        local_irq_save(flags);
-        rdp->qlen -= count;
-        local_irq_restore(flags);
-        if (rdp->blimit == INT_MAX && rdp->qlen <= qlowmark)
-                rdp->blimit = blimit;
-
-        if (!rdp->donelist)
-                rdp->donetail = &rdp->donelist;
-        else
-                raise_rcu_softirq();
-}
-
-/*
- * Grace period handling:
- * The grace period handling consists out of two steps:
- * - A new grace period is started.
- *   This is done by rcu_start_batch. The start is not broadcasted to
- *   all cpus, they must pick this up by comparing rcp->cur with
- *   rdp->quiescbatch. All cpus are recorded  in the
- *   rcu_ctrlblk.cpumask bitmap.
- * - All cpus must go through a quiescent state.
- *   Since the start of the grace period is not broadcasted, at least two
- *   calls to rcu_check_quiescent_state are required:
- *   The first call just notices that a new grace period is running. The
- *   following calls check if there was a quiescent state since the beginning
- *   of the grace period. If so, it updates rcu_ctrlblk.cpumask. If
- *   the bitmap is empty, then the grace period is completed.
- *   rcu_check_quiescent_state calls rcu_start_batch(0) to start the next grace
- *   period (if necessary).
- */
-
-/*
- * Register a new batch of callbacks, and start it up if there is currently no
- * active batch and the batch to be registered has not already occurred.
- * Caller must hold rcu_ctrlblk.lock.
- */
-static void rcu_start_batch(struct rcu_ctrlblk *rcp)
-{
-        if (rcp->cur != rcp->pending &&
-                        rcp->completed == rcp->cur) {
-                rcp->cur++;
-                record_gp_stall_check_time(rcp);
-
-                /*
-                 * Accessing nohz_cpu_mask before incrementing rcp->cur needs a
-                 * Barrier  Otherwise it can cause tickless idle CPUs to be
-                 * included in rcp->cpumask, which will extend graceperiods
-                 * unnecessarily.
-                 */
-                smp_mb();
-                cpumask_andnot(to_cpumask(rcp->cpumask),
-                               cpu_online_mask, nohz_cpu_mask);
-
-                rcp->signaled = 0;
-        }
-}
-
-/*
- * cpu went through a quiescent state since the beginning of the grace period.
- * Clear it from the cpu mask and complete the grace period if it was the last
- * cpu. Start another grace period if someone has further entries pending
- */
-static void cpu_quiet(int cpu, struct rcu_ctrlblk *rcp)
-{
-        cpumask_clear_cpu(cpu, to_cpumask(rcp->cpumask));
-        if (cpumask_empty(to_cpumask(rcp->cpumask))) {
-                /* batch completed ! */
-                rcp->completed = rcp->cur;
-                rcu_start_batch(rcp);
-        }
-}
-
-/*
- * Check if the cpu has gone through a quiescent state (say context
- * switch). If so and if it already hasn't done so in this RCU
- * quiescent cycle, then indicate that it has done so.
- */
-static void rcu_check_quiescent_state(struct rcu_ctrlblk *rcp,
-                                        struct rcu_data *rdp)
-{
-        unsigned long flags;
-
-        if (rdp->quiescbatch != rcp->cur) {
-                /* start new grace period: */
-                rdp->qs_pending = 1;
-                rdp->passed_quiesc = 0;
-                rdp->quiescbatch = rcp->cur;
-                return;
-        }
-
-        /* Grace period already completed for this cpu?
-         * qs_pending is checked instead of the actual bitmap to avoid
-         * cacheline trashing.
-         */
-        if (!rdp->qs_pending)
-                return;
-
-        /*
-         * Was there a quiescent state since the beginning of the grace
-         * period? If no, then exit and wait for the next call.
-         */
-        if (!rdp->passed_quiesc)
-                return;
-        rdp->qs_pending = 0;
-
-        spin_lock_irqsave(&rcp->lock, flags);
-        /*
-         * rdp->quiescbatch/rcp->cur and the cpu bitmap can come out of sync
-         * during cpu startup. Ignore the quiescent state.
-         */
-        if (likely(rdp->quiescbatch == rcp->cur))
-                cpu_quiet(rdp->cpu, rcp);
-
-        spin_unlock_irqrestore(&rcp->lock, flags);
-}
-
-
-#ifdef CONFIG_HOTPLUG_CPU
-
-/* warning! helper for rcu_offline_cpu. do not use elsewhere without reviewing
- * locking requirements, the list it's pulling from has to belong to a cpu
- * which is dead and hence not processing interrupts.
- */
-static void rcu_move_batch(struct rcu_data *this_rdp, struct rcu_head *list,
-                                struct rcu_head **tail, long batch)
-{
-        unsigned long flags;
-
-        if (list) {
-                local_irq_save(flags);
-                this_rdp->batch = batch;
-                *this_rdp->nxttail[2] = list;
-                this_rdp->nxttail[2] = tail;
-                local_irq_restore(flags);
-        }
-}
-
-static void __rcu_offline_cpu(struct rcu_data *this_rdp,
-                                struct rcu_ctrlblk *rcp, struct rcu_data *rdp)
-{
-        unsigned long flags;
-
-        /*
-         * if the cpu going offline owns the grace period
-         * we can block indefinitely waiting for it, so flush
-         * it here
-         */
-        spin_lock_irqsave(&rcp->lock, flags);
-        if (rcp->cur != rcp->completed)
-                cpu_quiet(rdp->cpu, rcp);
-        rcu_move_batch(this_rdp, rdp->donelist, rdp->donetail, rcp->cur + 1);
-        rcu_move_batch(this_rdp, rdp->nxtlist, rdp->nxttail[2], rcp->cur + 1);
-        spin_unlock(&rcp->lock);
-
-        this_rdp->qlen += rdp->qlen;
-        local_irq_restore(flags);
-}
-
-static void rcu_offline_cpu(int cpu)
-{
-        struct rcu_data *this_rdp = &get_cpu_var(rcu_data);
-        struct rcu_data *this_bh_rdp = &get_cpu_var(rcu_bh_data);
-
-        __rcu_offline_cpu(this_rdp, &rcu_ctrlblk,
-                                        &per_cpu(rcu_data, cpu));
-        __rcu_offline_cpu(this_bh_rdp, &rcu_bh_ctrlblk,
-                                        &per_cpu(rcu_bh_data, cpu));
-        put_cpu_var(rcu_data);
-        put_cpu_var(rcu_bh_data);
-}
-
-#else
-
-static void rcu_offline_cpu(int cpu)
-{
-}
-
-#endif
-
-/*
- * This does the RCU processing work from softirq context.
- */
-static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp,
-                                        struct rcu_data *rdp)
-{
-        unsigned long flags;
-        long completed_snap;
-
-        if (rdp->nxtlist) {
-                local_irq_save(flags);
-                completed_snap = ACCESS_ONCE(rcp->completed);
-
-                /*
-                 * move the other grace-period-completed entries to
-                 * [rdp->nxtlist, *rdp->nxttail[0]) temporarily
-                 */
-                if (!rcu_batch_before(completed_snap, rdp->batch))
-                        rdp->nxttail[0] = rdp->nxttail[1] = rdp->nxttail[2];
-                else if (!rcu_batch_before(completed_snap, rdp->batch - 1))
-                        rdp->nxttail[0] = rdp->nxttail[1];
-
-                /*
-                 * the grace period for entries in
-                 * [rdp->nxtlist, *rdp->nxttail[0]) has completed and
-                 * move these entries to donelist
-                 */
-                if (rdp->nxttail[0] != &rdp->nxtlist) {
-                        *rdp->donetail = rdp->nxtlist;
-                        rdp->donetail = rdp->nxttail[0];
-                        rdp->nxtlist = *rdp->nxttail[0];
-                        *rdp->donetail = NULL;
-
-                        if (rdp->nxttail[1] == rdp->nxttail[0])
-                                rdp->nxttail[1] = &rdp->nxtlist;
-                        if (rdp->nxttail[2] == rdp->nxttail[0])
-                                rdp->nxttail[2] = &rdp->nxtlist;
-                        rdp->nxttail[0] = &rdp->nxtlist;
-                }
-
-                local_irq_restore(flags);
-
-                if (rcu_batch_after(rdp->batch, rcp->pending)) {
-                        unsigned long flags2;
-
-                        /* and start it/schedule start if it's a new batch */
-                        spin_lock_irqsave(&rcp->lock, flags2);
-                        if (rcu_batch_after(rdp->batch, rcp->pending)) {
-                                rcp->pending = rdp->batch;
-                                rcu_start_batch(rcp);
-                        }
-                        spin_unlock_irqrestore(&rcp->lock, flags2);
-                }
-        }
-
-        rcu_check_quiescent_state(rcp, rdp);
-        if (rdp->donelist)
-                rcu_do_batch(rdp);
-}
-
-static void rcu_process_callbacks(struct softirq_action *unused)
-{
-        /*
-         * Memory references from any prior RCU read-side critical sections
-         * executed by the interrupted code must be see before any RCU
-         * grace-period manupulations below.
-         */
-
-        smp_mb(); /* See above block comment. */
-
-        __rcu_process_callbacks(&rcu_ctrlblk, &__get_cpu_var(rcu_data));
-        __rcu_process_callbacks(&rcu_bh_ctrlblk, &__get_cpu_var(rcu_bh_data));
-
-        /*
-         * Memory references from any later RCU read-side critical sections
-         * executed by the interrupted code must be see after any RCU
-         * grace-period manupulations above.
-         */
-
-        smp_mb(); /* See above block comment. */
-}
-
-static int __rcu_pending(struct rcu_ctrlblk *rcp, struct rcu_data *rdp)
-{
-        /* Check for CPU stalls, if enabled. */
-        check_cpu_stall(rcp);
-
-        if (rdp->nxtlist) {
-                long completed_snap = ACCESS_ONCE(rcp->completed);
-
-                /*
-                 * This cpu has pending rcu entries and the grace period
-                 * for them has completed.
-                 */
-                if (!rcu_batch_before(completed_snap, rdp->batch))
-                        return 1;
-                if (!rcu_batch_before(completed_snap, rdp->batch - 1) &&
-                                rdp->nxttail[0] != rdp->nxttail[1])
-                        return 1;
-                if (rdp->nxttail[0] != &rdp->nxtlist)
-                        return 1;
-
-                /*
-                 * This cpu has pending rcu entries and the new batch
-                 * for then hasn't been started nor scheduled start
-                 */
-                if (rcu_batch_after(rdp->batch, rcp->pending))
-                        return 1;
-        }
-
-        /* This cpu has finished callbacks to invoke */
-        if (rdp->donelist)
-                return 1;
-
-        /* The rcu core waits for a quiescent state from the cpu */
-        if (rdp->quiescbatch != rcp->cur || rdp->qs_pending)
-                return 1;
-
-        /* nothing to do */
-        return 0;
-}
-
-/*
- * Check to see if there is any immediate RCU-related work to be done
- * by the current CPU, returning 1 if so.  This function is part of the
- * RCU implementation; it is -not- an exported member of the RCU API.
- */
-int rcu_pending(int cpu)
-{
-        return __rcu_pending(&rcu_ctrlblk, &per_cpu(rcu_data, cpu)) ||
-                __rcu_pending(&rcu_bh_ctrlblk, &per_cpu(rcu_bh_data, cpu));
-}
-
-/*
- * Check to see if any future RCU-related work will need to be done
- * by the current CPU, even if none need be done immediately, returning
- * 1 if so.  This function is part of the RCU implementation; it is -not-
- * an exported member of the RCU API.
- */
-int rcu_needs_cpu(int cpu)
-{
-        struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
-        struct rcu_data *rdp_bh = &per_cpu(rcu_bh_data, cpu);
-
-        return !!rdp->nxtlist || !!rdp_bh->nxtlist || rcu_pending(cpu);
-}
-
-/*
- * Top-level function driving RCU grace-period detection, normally
- * invoked from the scheduler-clock interrupt.  This function simply
- * increments counters that are read only from softirq by this same
- * CPU, so there are no memory barriers required.
- */
-void rcu_check_callbacks(int cpu, int user)
-{
-        if (user ||
-            (idle_cpu(cpu) && rcu_scheduler_active &&
-             !in_softirq() && hardirq_count() <= (1 << HARDIRQ_SHIFT))) {
-
-                /*
-                 * Get here if this CPU took its interrupt from user
-                 * mode or from the idle loop, and if this is not a
-                 * nested interrupt.  In this case, the CPU is in
-                 * a quiescent state, so count it.
-                 *
-                 * Also do a memory barrier.  This is needed to handle
-                 * the case where writes from a preempt-disable section
-                 * of code get reordered into schedule() by this CPU's
-                 * write buffer.  The memory barrier makes sure that
-                 * the rcu_qsctr_inc() and rcu_bh_qsctr_inc() are see
-                 * by other CPUs to happen after any such write.
-                 */
-
-                smp_mb();  /* See above block comment. */
-                rcu_qsctr_inc(cpu);
-                rcu_bh_qsctr_inc(cpu);
-
-        } else if (!in_softirq()) {
-
-                /*
-                 * Get here if this CPU did not take its interrupt from
-                 * softirq, in other words, if it is not interrupting
-                 * a rcu_bh read-side critical section.  This is an _bh
-                 * critical section, so count it.  The memory barrier
-                 * is needed for the same reason as is the above one.
-                 */
-
-                smp_mb();  /* See above block comment. */
-                rcu_bh_qsctr_inc(cpu);
-        }
-        raise_rcu_softirq();
-}
-
-static void __cpuinit rcu_init_percpu_data(int cpu, struct rcu_ctrlblk *rcp,
-                                                struct rcu_data *rdp)
-{
-        unsigned long flags;
-
-        spin_lock_irqsave(&rcp->lock, flags);
-        memset(rdp, 0, sizeof(*rdp));
-        rdp->nxttail[0] = rdp->nxttail[1] = rdp->nxttail[2] = &rdp->nxtlist;
-        rdp->donetail = &rdp->donelist;
-        rdp->quiescbatch = rcp->completed;
-        rdp->qs_pending = 0;
-        rdp->cpu = cpu;
-        rdp->blimit = blimit;
-        spin_unlock_irqrestore(&rcp->lock, flags);
-}
-
-static void __cpuinit rcu_online_cpu(int cpu)
-{
-        struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
-        struct rcu_data *bh_rdp = &per_cpu(rcu_bh_data, cpu);
-
-        rcu_init_percpu_data(cpu, &rcu_ctrlblk, rdp);
-        rcu_init_percpu_data(cpu, &rcu_bh_ctrlblk, bh_rdp);
-        open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
-}
-
-static int __cpuinit rcu_cpu_notify(struct notifier_block *self,
-                                unsigned long action, void *hcpu)
-{
-        long cpu = (long)hcpu;
-
-        switch (action) {
-        case CPU_UP_PREPARE:
-        case CPU_UP_PREPARE_FROZEN:
-                rcu_online_cpu(cpu);
-                break;
-        case CPU_DEAD:
-        case CPU_DEAD_FROZEN:
-                rcu_offline_cpu(cpu);
-                break;
-        default:
-                break;
-        }
-        return NOTIFY_OK;
-}
-
-static struct notifier_block __cpuinitdata rcu_nb = {
-        .notifier_call  = rcu_cpu_notify,
-};
-
-/*
- * Initializes rcu mechanism.  Assumed to be called early.
- * That is before local timer(SMP) or jiffie timer (uniproc) is setup.
- * Note that rcu_qsctr and friends are implicitly
- * initialized due to the choice of ``0'' for RCU_CTR_INVALID.
- */
-void __init __rcu_init(void)
-{
-#ifdef CONFIG_RCU_CPU_STALL_DETECTOR
-        printk(KERN_INFO "RCU-based detection of stalled CPUs is enabled.\n");
-#endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
-        rcu_cpu_notify(&rcu_nb, CPU_UP_PREPARE,
-                        (void *)(long)smp_processor_id());
-        /* Register notifier for non-boot CPUs */
-        register_cpu_notifier(&rcu_nb);
-}
-
-module_param(blimit, int, 0);
-module_param(qhimark, int, 0);
-module_param(qlowmark, int, 0);
diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c
index a967c9feb90a..bd5d5c8e5140 100644
--- a/kernel/rcupdate.c
+++ b/kernel/rcupdate.c
@@ -98,6 +98,30 @@ void synchronize_rcu(void)
 }
 EXPORT_SYMBOL_GPL(synchronize_rcu);
 
+/**
+ * synchronize_rcu_bh - wait until an rcu_bh grace period has elapsed.
+ *
+ * Control will return to the caller some time after a full rcu_bh grace
+ * period has elapsed, in other words after all currently executing rcu_bh
+ * read-side critical sections have completed.  RCU read-side critical
+ * sections are delimited by rcu_read_lock_bh() and rcu_read_unlock_bh(),
+ * and may be nested.
+ */
+void synchronize_rcu_bh(void)
+{
+        struct rcu_synchronize rcu;
+
+        if (rcu_blocking_is_gp())
+                return;
+
+        init_completion(&rcu.completion);
+        /* Will wake me after RCU finished. */
+        call_rcu_bh(&rcu.head, wakeme_after_rcu);
+        /* Wait for it. */
+        wait_for_completion(&rcu.completion);
+}
+EXPORT_SYMBOL_GPL(synchronize_rcu_bh);
+
 static void rcu_barrier_callback(struct rcu_head *notused)
 {
         if (atomic_dec_and_test(&rcu_barrier_cpu_count))
@@ -129,6 +153,7 @@ static void rcu_barrier_func(void *type)
 static inline void wait_migrated_callbacks(void)
 {
         wait_event(rcu_migrate_wq, !atomic_read(&rcu_migrate_type_count));
+        smp_mb(); /* In case we didn't sleep. */
 }
 
 /*
@@ -192,9 +217,13 @@ static void rcu_migrate_callback(struct rcu_head *notused)
                 wake_up(&rcu_migrate_wq);
 }
 
+extern int rcu_cpu_notify(struct notifier_block *self,
+                          unsigned long action, void *hcpu);
+
 static int __cpuinit rcu_barrier_cpu_hotplug(struct notifier_block *self,
                 unsigned long action, void *hcpu)
 {
+        rcu_cpu_notify(self, action, hcpu);
         if (action == CPU_DYING) {
                 /*
                  * preempt_disable() in on_each_cpu() prevents stop_machine(),
@@ -209,7 +238,8 @@ static int __cpuinit rcu_barrier_cpu_hotplug(struct notifier_block *self,
                 call_rcu_bh(rcu_migrate_head, rcu_migrate_callback);
                 call_rcu_sched(rcu_migrate_head + 1, rcu_migrate_callback);
                 call_rcu(rcu_migrate_head + 2, rcu_migrate_callback);
-        } else if (action == CPU_POST_DEAD) {
+        } else if (action == CPU_DOWN_PREPARE) {
+                /* Don't need to wait until next removal operation. */
                 /* rcu_migrate_head is protected by cpu_add_remove_lock */
                 wait_migrated_callbacks();
         }
@@ -219,8 +249,18 @@ static int __cpuinit rcu_barrier_cpu_hotplug(struct notifier_block *self,
 
 void __init rcu_init(void)
 {
+        int i;
+
         __rcu_init();
-        hotcpu_notifier(rcu_barrier_cpu_hotplug, 0);
+        cpu_notifier(rcu_barrier_cpu_hotplug, 0);
+
+        /*
+         * We don't need protection against CPU-hotplug here because
+         * this is called early in boot, before either interrupts
+         * or the scheduler are operational.
+         */
+        for_each_online_cpu(i)
+                rcu_barrier_cpu_hotplug(NULL, CPU_UP_PREPARE, (void *)(long)i);
 }
 
 void rcu_scheduler_starting(void)
diff --git a/kernel/rcupreempt.c b/kernel/rcupreempt.c
deleted file mode 100644
index beb0e659adcc..000000000000
--- a/kernel/rcupreempt.c
+++ /dev/null
@@ -1,1539 +0,0 @@
-/*
- * Read-Copy Update mechanism for mutual exclusion, realtime implementation
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
- *
- * Copyright IBM Corporation, 2006
- *
- * Authors: Paul E. McKenney <paulmck@us.ibm.com>
- *              With thanks to Esben Nielsen, Bill Huey, and Ingo Molnar
- *              for pushing me away from locks and towards counters, and
- *              to Suparna Bhattacharya for pushing me completely away
- *              from atomic instructions on the read side.
- *
- *  - Added handling of Dynamic Ticks
- *      Copyright 2007 - Paul E. Mckenney <paulmck@us.ibm.com>
- *                     - Steven Rostedt <srostedt@redhat.com>
- *
- * Papers:  http://www.rdrop.com/users/paulmck/RCU
- *
- * Design Document: http://lwn.net/Articles/253651/
- *
- * For detailed explanation of Read-Copy Update mechanism see -
- *              Documentation/RCU/ *.txt
- *
- */
-#include <linux/types.h>
-#include <linux/kernel.h>
-#include <linux/init.h>
-#include <linux/spinlock.h>
-#include <linux/smp.h>
-#include <linux/rcupdate.h>
-#include <linux/interrupt.h>
-#include <linux/sched.h>
-#include <asm/atomic.h>
-#include <linux/bitops.h>
-#include <linux/module.h>
-#include <linux/kthread.h>
-#include <linux/completion.h>
-#include <linux/moduleparam.h>
-#include <linux/percpu.h>
-#include <linux/notifier.h>
-#include <linux/cpu.h>
-#include <linux/random.h>
-#include <linux/delay.h>
-#include <linux/cpumask.h>
-#include <linux/rcupreempt_trace.h>
-#include <asm/byteorder.h>
-
-/*
- * PREEMPT_RCU data structures.
- */
-
-/*
- * GP_STAGES specifies the number of times the state machine has
- * to go through the all the rcu_try_flip_states (see below)
- * in a single Grace Period.
- *
- * GP in GP_STAGES stands for Grace Period ;)
- */
-#define GP_STAGES    2
-struct rcu_data {
-        spinlock_t      lock;           /* Protect rcu_data fields. */
-        long            completed;      /* Number of last completed batch. */
-        int             waitlistcount;
-        struct rcu_head *nextlist;
-        struct rcu_head **nexttail;
-        struct rcu_head *waitlist[GP_STAGES];
-        struct rcu_head **waittail[GP_STAGES];
-        struct rcu_head *donelist;      /* from waitlist & waitschedlist */
-        struct rcu_head **donetail;
-        long rcu_flipctr[2];
-        struct rcu_head *nextschedlist;
-        struct rcu_head **nextschedtail;
-        struct rcu_head *waitschedlist;
-        struct rcu_head **waitschedtail;
-        int rcu_sched_sleeping;
-#ifdef CONFIG_RCU_TRACE
-        struct rcupreempt_trace trace;
-#endif /* #ifdef CONFIG_RCU_TRACE */
-};
-
-/*
- * States for rcu_try_flip() and friends.
- */
-
-enum rcu_try_flip_states {
-
-        /*
-         * Stay here if nothing is happening. Flip the counter if somthing
-         * starts happening. Denoted by "I"
-         */
-        rcu_try_flip_idle_state,
-
-        /*
-         * Wait here for all CPUs to notice that the counter has flipped. This
-         * prevents the old set of counters from ever being incremented once
-         * we leave this state, which in turn is necessary because we cannot
-         * test any individual counter for zero -- we can only check the sum.
-         * Denoted by "A".
-         */
-        rcu_try_flip_waitack_state,
-
-        /*
-         * Wait here for the sum of the old per-CPU counters to reach zero.
-         * Denoted by "Z".
-         */
-        rcu_try_flip_waitzero_state,
-
-        /*
-         * Wait here for each of the other CPUs to execute a memory barrier.
-         * This is necessary to ensure that these other CPUs really have
-         * completed executing their RCU read-side critical sections, despite
-         * their CPUs wildly reordering memory. Denoted by "M".
-         */
-        rcu_try_flip_waitmb_state,
-};
-
-/*
- * States for rcu_ctrlblk.rcu_sched_sleep.
- */
-
-enum rcu_sched_sleep_states {
-        rcu_sched_not_sleeping, /* Not sleeping, callbacks need GP.  */
-        rcu_sched_sleep_prep,   /* Thinking of sleeping, rechecking. */
-        rcu_sched_sleeping,     /* Sleeping, awaken if GP needed. */
-};
-
-struct rcu_ctrlblk {
-        spinlock_t      fliplock;       /* Protect state-machine transitions. */
-        long            completed;      /* Number of last completed batch. */
-        enum rcu_try_flip_states rcu_try_flip_state; /* The current state of
-                                                        the rcu state machine */
-        spinlock_t      schedlock;      /* Protect rcu_sched sleep state. */
-        enum rcu_sched_sleep_states sched_sleep; /* rcu_sched state. */
-        wait_queue_head_t sched_wq;     /* Place for rcu_sched to sleep. */
-};
-
-struct rcu_dyntick_sched {
-        int dynticks;
-        int dynticks_snap;
-        int sched_qs;
-        int sched_qs_snap;
-        int sched_dynticks_snap;
-};
-
-static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_dyntick_sched, rcu_dyntick_sched) = {
-        .dynticks = 1,
-};
-
-void rcu_qsctr_inc(int cpu)
-{
-        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
-
-        rdssp->sched_qs++;
-}
-
-#ifdef CONFIG_NO_HZ
-
-void rcu_enter_nohz(void)
-{
-        static DEFINE_RATELIMIT_STATE(rs, 10 * HZ, 1);
-
-        smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */
-        __get_cpu_var(rcu_dyntick_sched).dynticks++;
-        WARN_ON_RATELIMIT(__get_cpu_var(rcu_dyntick_sched).dynticks & 0x1, &rs);
-}
-
-void rcu_exit_nohz(void)
-{
-        static DEFINE_RATELIMIT_STATE(rs, 10 * HZ, 1);
-
-        __get_cpu_var(rcu_dyntick_sched).dynticks++;
-        smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */
-        WARN_ON_RATELIMIT(!(__get_cpu_var(rcu_dyntick_sched).dynticks & 0x1),
-                                &rs);
-}
-
-#endif /* CONFIG_NO_HZ */
-
-
-static DEFINE_PER_CPU(struct rcu_data, rcu_data);
-
-static struct rcu_ctrlblk rcu_ctrlblk = {
-        .fliplock = __SPIN_LOCK_UNLOCKED(rcu_ctrlblk.fliplock),
-        .completed = 0,
-        .rcu_try_flip_state = rcu_try_flip_idle_state,
-        .schedlock = __SPIN_LOCK_UNLOCKED(rcu_ctrlblk.schedlock),
-        .sched_sleep = rcu_sched_not_sleeping,
-        .sched_wq = __WAIT_QUEUE_HEAD_INITIALIZER(rcu_ctrlblk.sched_wq),
-};
-
-static struct task_struct *rcu_sched_grace_period_task;
-
-#ifdef CONFIG_RCU_TRACE
-static char *rcu_try_flip_state_names[] =
-        { "idle", "waitack", "waitzero", "waitmb" };
-#endif /* #ifdef CONFIG_RCU_TRACE */
-
-static DECLARE_BITMAP(rcu_cpu_online_map, NR_CPUS) __read_mostly
-        = CPU_BITS_NONE;
-
-/*
- * Enum and per-CPU flag to determine when each CPU has seen
- * the most recent counter flip.
- */
-
-enum rcu_flip_flag_values {
-        rcu_flip_seen,          /* Steady/initial state, last flip seen. */
-                                /* Only GP detector can update. */
-        rcu_flipped             /* Flip just completed, need confirmation. */
-                                /* Only corresponding CPU can update. */
-};
-static DEFINE_PER_CPU_SHARED_ALIGNED(enum rcu_flip_flag_values, rcu_flip_flag)
-                                                                = rcu_flip_seen;
-
-/*
- * Enum and per-CPU flag to determine when each CPU has executed the
- * needed memory barrier to fence in memory references from its last RCU
- * read-side critical section in the just-completed grace period.
- */
-
-enum rcu_mb_flag_values {
-        rcu_mb_done,            /* Steady/initial state, no mb()s required. */
-                                /* Only GP detector can update. */
-        rcu_mb_needed           /* Flip just completed, need an mb(). */
-                                /* Only corresponding CPU can update. */
-};
-static DEFINE_PER_CPU_SHARED_ALIGNED(enum rcu_mb_flag_values, rcu_mb_flag)
-                                                                = rcu_mb_done;
-
-/*
- * RCU_DATA_ME: find the current CPU's rcu_data structure.
- * RCU_DATA_CPU: find the specified CPU's rcu_data structure.
- */
-#define RCU_DATA_ME()           (&__get_cpu_var(rcu_data))
-#define RCU_DATA_CPU(cpu)       (&per_cpu(rcu_data, cpu))
-
-/*
- * Helper macro for tracing when the appropriate rcu_data is not
- * cached in a local variable, but where the CPU number is so cached.
- */
-#define RCU_TRACE_CPU(f, cpu) RCU_TRACE(f, &(RCU_DATA_CPU(cpu)->trace));
-
-/*
- * Helper macro for tracing when the appropriate rcu_data is not
- * cached in a local variable.
- */
-#define RCU_TRACE_ME(f) RCU_TRACE(f, &(RCU_DATA_ME()->trace));
-
-/*
- * Helper macro for tracing when the appropriate rcu_data is pointed
- * to by a local variable.
- */
-#define RCU_TRACE_RDP(f, rdp) RCU_TRACE(f, &((rdp)->trace));
-
-#define RCU_SCHED_BATCH_TIME (HZ / 50)
-
-/*
- * Return the number of RCU batches processed thus far.  Useful
- * for debug and statistics.
- */
-long rcu_batches_completed(void)
-{
-        return rcu_ctrlblk.completed;
-}
-EXPORT_SYMBOL_GPL(rcu_batches_completed);
-
-void __rcu_read_lock(void)
-{
-        int idx;
-        struct task_struct *t = current;
-        int nesting;
-
-        nesting = ACCESS_ONCE(t->rcu_read_lock_nesting);
-        if (nesting != 0) {
-
-                /* An earlier rcu_read_lock() covers us, just count it. */
-
-                t->rcu_read_lock_nesting = nesting + 1;
-
-        } else {
-                unsigned long flags;
-
-                /*
-                 * We disable interrupts for the following reasons:
-                 * - If we get scheduling clock interrupt here, and we
-                 *   end up acking the counter flip, it's like a promise
-                 *   that we will never increment the old counter again.
-                 *   Thus we will break that promise if that
-                 *   scheduling clock interrupt happens between the time
-                 *   we pick the .completed field and the time that we
-                 *   increment our counter.
-                 *
-                 * - We don't want to be preempted out here.
-                 *
-                 * NMIs can still occur, of course, and might themselves
-                 * contain rcu_read_lock().
-                 */
-
-                local_irq_save(flags);
-
-                /*
-                 * Outermost nesting of rcu_read_lock(), so increment
-                 * the current counter for the current CPU.  Use volatile
-                 * casts to prevent the compiler from reordering.
-                 */
-
-                idx = ACCESS_ONCE(rcu_ctrlblk.completed) & 0x1;
-                ACCESS_ONCE(RCU_DATA_ME()->rcu_flipctr[idx])++;
-
-                /*
-                 * Now that the per-CPU counter has been incremented, we
-                 * are protected from races with rcu_read_lock() invoked
-                 * from NMI handlers on this CPU.  We can therefore safely
-                 * increment the nesting counter, relieving further NMIs
-                 * of the need to increment the per-CPU counter.
-                 */
-
-                ACCESS_ONCE(t->rcu_read_lock_nesting) = nesting + 1;
-
-                /*
-                 * Now that we have preventing any NMIs from storing
-                 * to the ->rcu_flipctr_idx, we can safely use it to
-                 * remember which counter to decrement in the matching
-                 * rcu_read_unlock().
-                 */
-
-                ACCESS_ONCE(t->rcu_flipctr_idx) = idx;
-                local_irq_restore(flags);
-        }
-}
-EXPORT_SYMBOL_GPL(__rcu_read_lock);
-
-void __rcu_read_unlock(void)
-{
-        int idx;
-        struct task_struct *t = current;
-        int nesting;
-
-        nesting = ACCESS_ONCE(t->rcu_read_lock_nesting);
-        if (nesting > 1) {
-
-                /*
-                 * We are still protected by the enclosing rcu_read_lock(),
-                 * so simply decrement the counter.
-                 */
-
-                t->rcu_read_lock_nesting = nesting - 1;
-
-        } else {
-                unsigned long flags;
-
-                /*
-                 * Disable local interrupts to prevent the grace-period
-                 * detection state machine from seeing us half-done.
-                 * NMIs can still occur, of course, and might themselves
-                 * contain rcu_read_lock() and rcu_read_unlock().
-                 */
-
-                local_irq_save(flags);
-
-                /*
-                 * Outermost nesting of rcu_read_unlock(), so we must
-                 * decrement the current counter for the current CPU.
-                 * This must be done carefully, because NMIs can
-                 * occur at any point in this code, and any rcu_read_lock()
-                 * and rcu_read_unlock() pairs in the NMI handlers
-                 * must interact non-destructively with this code.
-                 * Lots of volatile casts, and -very- careful ordering.
-                 *
-                 * Changes to this code, including this one, must be
-                 * inspected, validated, and tested extremely carefully!!!
-                 */
-
-                /*
-                 * First, pick up the index.
-                 */
-
-                idx = ACCESS_ONCE(t->rcu_flipctr_idx);
-
-                /*
-                 * Now that we have fetched the counter index, it is
-                 * safe to decrement the per-task RCU nesting counter.
-                 * After this, any interrupts or NMIs will increment and
-                 * decrement the per-CPU counters.
-                 */
-                ACCESS_ONCE(t->rcu_read_lock_nesting) = nesting - 1;
-
-                /*
-                 * It is now safe to decrement this task's nesting count.
-                 * NMIs that occur after this statement will route their
-                 * rcu_read_lock() calls through this "else" clause, and
-                 * will thus start incrementing the per-CPU counter on
-                 * their own.  They will also clobber ->rcu_flipctr_idx,
-                 * but that is OK, since we have already fetched it.
-                 */
-
-                ACCESS_ONCE(RCU_DATA_ME()->rcu_flipctr[idx])--;
-                local_irq_restore(flags);
-        }
-}
-EXPORT_SYMBOL_GPL(__rcu_read_unlock);
-
-/*
- * If a global counter flip has occurred since the last time that we
- * advanced callbacks, advance them.  Hardware interrupts must be
- * disabled when calling this function.
- */
-static void __rcu_advance_callbacks(struct rcu_data *rdp)
-{
-        int cpu;
-        int i;
-        int wlc = 0;
-
-        if (rdp->completed != rcu_ctrlblk.completed) {
-                if (rdp->waitlist[GP_STAGES - 1] != NULL) {
-                        *rdp->donetail = rdp->waitlist[GP_STAGES - 1];
-                        rdp->donetail = rdp->waittail[GP_STAGES - 1];
-                        RCU_TRACE_RDP(rcupreempt_trace_move2done, rdp);
-                }
-                for (i = GP_STAGES - 2; i >= 0; i--) {
-                        if (rdp->waitlist[i] != NULL) {
-                                rdp->waitlist[i + 1] = rdp->waitlist[i];
-                                rdp->waittail[i + 1] = rdp->waittail[i];
-                                wlc++;
-                        } else {
-                                rdp->waitlist[i + 1] = NULL;
-                                rdp->waittail[i + 1] =
-                                        &rdp->waitlist[i + 1];
-                        }
-                }
-                if (rdp->nextlist != NULL) {
-                        rdp->waitlist[0] = rdp->nextlist;
-                        rdp->waittail[0] = rdp->nexttail;
-                        wlc++;
-                        rdp->nextlist = NULL;
-                        rdp->nexttail = &rdp->nextlist;
-                        RCU_TRACE_RDP(rcupreempt_trace_move2wait, rdp);
-                } else {
-                        rdp->waitlist[0] = NULL;
-                        rdp->waittail[0] = &rdp->waitlist[0];
-                }
-                rdp->waitlistcount = wlc;
-                rdp->completed = rcu_ctrlblk.completed;
-        }
-
-        /*
-         * Check to see if this CPU needs to report that it has seen
-         * the most recent counter flip, thereby declaring that all
-         * subsequent rcu_read_lock() invocations will respect this flip.
-         */
-
-        cpu = raw_smp_processor_id();
-        if (per_cpu(rcu_flip_flag, cpu) == rcu_flipped) {
-                smp_mb();  /* Subsequent counter accesses must see new value */
-                per_cpu(rcu_flip_flag, cpu) = rcu_flip_seen;
-                smp_mb();  /* Subsequent RCU read-side critical sections */
-                           /*  seen -after- acknowledgement. */
-        }
-}
-
-#ifdef CONFIG_NO_HZ
-static DEFINE_PER_CPU(int, rcu_update_flag);
-
-/**
- * rcu_irq_enter - Called from Hard irq handlers and NMI/SMI.
- *
- * If the CPU was idle with dynamic ticks active, this updates the
- * rcu_dyntick_sched.dynticks to let the RCU handling know that the
- * CPU is active.
- */
-void rcu_irq_enter(void)
-{
-        int cpu = smp_processor_id();
-        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
-
-        if (per_cpu(rcu_update_flag, cpu))
-                per_cpu(rcu_update_flag, cpu)++;
-
-        /*
-         * Only update if we are coming from a stopped ticks mode
-         * (rcu_dyntick_sched.dynticks is even).
-         */
-        if (!in_interrupt() &&
-            (rdssp->dynticks & 0x1) == 0) {
-                /*
-                 * The following might seem like we could have a race
-                 * with NMI/SMIs. But this really isn't a problem.
-                 * Here we do a read/modify/write, and the race happens
-                 * when an NMI/SMI comes in after the read and before
-                 * the write. But NMI/SMIs will increment this counter
-                 * twice before returning, so the zero bit will not
-                 * be corrupted by the NMI/SMI which is the most important
-                 * part.
-                 *
-                 * The only thing is that we would bring back the counter
-                 * to a postion that it was in during the NMI/SMI.
-                 * But the zero bit would be set, so the rest of the
-                 * counter would again be ignored.
-                 *
-                 * On return from the IRQ, the counter may have the zero
-                 * bit be 0 and the counter the same as the return from
-                 * the NMI/SMI. If the state machine was so unlucky to
-                 * see that, it still doesn't matter, since all
-                 * RCU read-side critical sections on this CPU would
-                 * have already completed.
-                 */
-                rdssp->dynticks++;
-                /*
-                 * The following memory barrier ensures that any
-                 * rcu_read_lock() primitives in the irq handler
-                 * are seen by other CPUs to follow the above
-                 * increment to rcu_dyntick_sched.dynticks. This is
-                 * required in order for other CPUs to correctly
-                 * determine when it is safe to advance the RCU
-                 * grace-period state machine.
-                 */
-                smp_mb(); /* see above block comment. */
-                /*
-                 * Since we can't determine the dynamic tick mode from
-                 * the rcu_dyntick_sched.dynticks after this routine,
-                 * we use a second flag to acknowledge that we came
-                 * from an idle state with ticks stopped.
-                 */
-                per_cpu(rcu_update_flag, cpu)++;
-                /*
-                 * If we take an NMI/SMI now, they will also increment
-                 * the rcu_update_flag, and will not update the
-                 * rcu_dyntick_sched.dynticks on exit. That is for
-                 * this IRQ to do.
-                 */
-        }
-}
-
-/**
- * rcu_irq_exit - Called from exiting Hard irq context.
- *
- * If the CPU was idle with dynamic ticks active, update the
- * rcu_dyntick_sched.dynticks to put let the RCU handling be
- * aware that the CPU is going back to idle with no ticks.
- */
-void rcu_irq_exit(void)
-{
-        int cpu = smp_processor_id();
-        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
-
-        /*
-         * rcu_update_flag is set if we interrupted the CPU
-         * when it was idle with ticks stopped.
-         * Once this occurs, we keep track of interrupt nesting
-         * because a NMI/SMI could also come in, and we still
-         * only want the IRQ that started the increment of the
-         * rcu_dyntick_sched.dynticks to be the one that modifies
-         * it on exit.
-         */
-        if (per_cpu(rcu_update_flag, cpu)) {
-                if (--per_cpu(rcu_update_flag, cpu))
-                        return;
-
-                /* This must match the interrupt nesting */
-                WARN_ON(in_interrupt());
-
-                /*
-                 * If an NMI/SMI happens now we are still
-                 * protected by the rcu_dyntick_sched.dynticks being odd.
-                 */
-
-                /*
-                 * The following memory barrier ensures that any
-                 * rcu_read_unlock() primitives in the irq handler
-                 * are seen by other CPUs to preceed the following
-                 * increment to rcu_dyntick_sched.dynticks. This
-                 * is required in order for other CPUs to determine
-                 * when it is safe to advance the RCU grace-period
-                 * state machine.
-                 */
-                smp_mb(); /* see above block comment. */
-                rdssp->dynticks++;
-                WARN_ON(rdssp->dynticks & 0x1);
-        }
-}
-
-void rcu_nmi_enter(void)
-{
-        rcu_irq_enter();
-}
-
-void rcu_nmi_exit(void)
-{
-        rcu_irq_exit();
-}
-
-static void dyntick_save_progress_counter(int cpu)
-{
-        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
-
-        rdssp->dynticks_snap = rdssp->dynticks;
-}
-
-static inline int
-rcu_try_flip_waitack_needed(int cpu)
-{
-        long curr;
-        long snap;
-        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
-
-        curr = rdssp->dynticks;
-        snap = rdssp->dynticks_snap;
-        smp_mb(); /* force ordering with cpu entering/leaving dynticks. */
-
-        /*
-         * If the CPU remained in dynticks mode for the entire time
-         * and didn't take any interrupts, NMIs, SMIs, or whatever,
-         * then it cannot be in the middle of an rcu_read_lock(), so
-         * the next rcu_read_lock() it executes must use the new value
-         * of the counter.  So we can safely pretend that this CPU
-         * already acknowledged the counter.
-         */
-
-        if ((curr == snap) && ((curr & 0x1) == 0))
-                return 0;
-
-        /*
-         * If the CPU passed through or entered a dynticks idle phase with
-         * no active irq handlers, then, as above, we can safely pretend
-         * that this CPU already acknowledged the counter.
-         */
-
-        if ((curr - snap) > 2 || (curr & 0x1) == 0)
-                return 0;
-
-        /* We need this CPU to explicitly acknowledge the counter flip. */
-
-        return 1;
-}
-
-static inline int
-rcu_try_flip_waitmb_needed(int cpu)
-{
-        long curr;
-        long snap;
-        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
-
-        curr = rdssp->dynticks;
-        snap = rdssp->dynticks_snap;
-        smp_mb(); /* force ordering with cpu entering/leaving dynticks. */
-
-        /*
-         * If the CPU remained in dynticks mode for the entire time
-         * and didn't take any interrupts, NMIs, SMIs, or whatever,
-         * then it cannot have executed an RCU read-side critical section
-         * during that time, so there is no need for it to execute a
-         * memory barrier.
-         */
-
-        if ((curr == snap) && ((curr & 0x1) == 0))
-                return 0;
-
-        /*
-         * If the CPU either entered or exited an outermost interrupt,
-         * SMI, NMI, or whatever handler, then we know that it executed
-         * a memory barrier when doing so.  So we don't need another one.
-         */
-        if (curr != snap)
-                return 0;
-
-        /* We need the CPU to execute a memory barrier. */
-
-        return 1;
-}
-
-static void dyntick_save_progress_counter_sched(int cpu)
-{
-        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
-
-        rdssp->sched_dynticks_snap = rdssp->dynticks;
-}
-
-static int rcu_qsctr_inc_needed_dyntick(int cpu)
-{
-        long curr;
-        long snap;
-        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
-
-        curr = rdssp->dynticks;
-        snap = rdssp->sched_dynticks_snap;
-        smp_mb(); /* force ordering with cpu entering/leaving dynticks. */
-
-        /*
-         * If the CPU remained in dynticks mode for the entire time
-         * and didn't take any interrupts, NMIs, SMIs, or whatever,
-         * then it cannot be in the middle of an rcu_read_lock(), so
-         * the next rcu_read_lock() it executes must use the new value
-         * of the counter.  Therefore, this CPU has been in a quiescent
-         * state the entire time, and we don't need to wait for it.
-         */
-
-        if ((curr == snap) && ((curr & 0x1) == 0))
-                return 0;
-
-        /*
-         * If the CPU passed through or entered a dynticks idle phase with
-         * no active irq handlers, then, as above, this CPU has already
-         * passed through a quiescent state.
-         */
-
-        if ((curr - snap) > 2 || (snap & 0x1) == 0)
-                return 0;
-
-        /* We need this CPU to go through a quiescent state. */
-
-        return 1;
-}
-
-#else /* !CONFIG_NO_HZ */
-
-# define dyntick_save_progress_counter(cpu)             do { } while (0)
-# define rcu_try_flip_waitack_needed(cpu)               (1)
-# define rcu_try_flip_waitmb_needed(cpu)                (1)
-
-# define dyntick_save_progress_counter_sched(cpu)       do { } while (0)
-# define rcu_qsctr_inc_needed_dyntick(cpu)              (1)
-
-#endif /* CONFIG_NO_HZ */
-
-static void save_qsctr_sched(int cpu)
-{
-        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
-
-        rdssp->sched_qs_snap = rdssp->sched_qs;
-}
-
-static inline int rcu_qsctr_inc_needed(int cpu)
-{
-        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
-
-        /*
-         * If there has been a quiescent state, no more need to wait
-         * on this CPU.
-         */
-
-        if (rdssp->sched_qs != rdssp->sched_qs_snap) {
-                smp_mb(); /* force ordering with cpu entering schedule(). */
-                return 0;
-        }
-
-        /* We need this CPU to go through a quiescent state. */
-
-        return 1;
-}
-
-/*
- * Get here when RCU is idle.  Decide whether we need to
- * move out of idle state, and return non-zero if so.
- * "Straightforward" approach for the moment, might later
- * use callback-list lengths, grace-period duration, or
- * some such to determine when to exit idle state.
- * Might also need a pre-idle test that does not acquire
- * the lock, but let's get the simple case working first...
- */
-
-static int
-rcu_try_flip_idle(void)
-{
-        int cpu;
-
-        RCU_TRACE_ME(rcupreempt_trace_try_flip_i1);
-        if (!rcu_pending(smp_processor_id())) {
-                RCU_TRACE_ME(rcupreempt_trace_try_flip_ie1);
-                return 0;
-        }
-
-        /*
-         * Do the flip.
-         */
-
-        RCU_TRACE_ME(rcupreempt_trace_try_flip_g1);
-        rcu_ctrlblk.completed++;  /* stands in for rcu_try_flip_g2 */
-
-        /*
-         * Need a memory barrier so that other CPUs see the new
-         * counter value before they see the subsequent change of all
-         * the rcu_flip_flag instances to rcu_flipped.
-         */
-
-        smp_mb();       /* see above block comment. */
-
-        /* Now ask each CPU for acknowledgement of the flip. */
-
-        for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map)) {
-                per_cpu(rcu_flip_flag, cpu) = rcu_flipped;
-                dyntick_save_progress_counter(cpu);
-        }
-
-        return 1;
-}
-
-/*
- * Wait for CPUs to acknowledge the flip.
- */
-
-static int
-rcu_try_flip_waitack(void)
-{
-        int cpu;
-
-        RCU_TRACE_ME(rcupreempt_trace_try_flip_a1);
-        for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map))
-                if (rcu_try_flip_waitack_needed(cpu) &&
-                    per_cpu(rcu_flip_flag, cpu) != rcu_flip_seen) {
-                        RCU_TRACE_ME(rcupreempt_trace_try_flip_ae1);
-                        return 0;
-                }
-
-        /*
-         * Make sure our checks above don't bleed into subsequent
-         * waiting for the sum of the counters to reach zero.
-         */
-
-        smp_mb();       /* see above block comment. */
-        RCU_TRACE_ME(rcupreempt_trace_try_flip_a2);
-        return 1;
-}
-
-/*
- * Wait for collective ``last'' counter to reach zero,
- * then tell all CPUs to do an end-of-grace-period memory barrier.
- */
-
-static int
-rcu_try_flip_waitzero(void)
-{
-        int cpu;
-        int lastidx = !(rcu_ctrlblk.completed & 0x1);
-        int sum = 0;
-
-        /* Check to see if the sum of the "last" counters is zero. */
-
-        RCU_TRACE_ME(rcupreempt_trace_try_flip_z1);
-        for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map))
-                sum += RCU_DATA_CPU(cpu)->rcu_flipctr[lastidx];
-        if (sum != 0) {
-                RCU_TRACE_ME(rcupreempt_trace_try_flip_ze1);
-                return 0;
-        }
-
-        /*
-         * This ensures that the other CPUs see the call for
-         * memory barriers -after- the sum to zero has been
-         * detected here
-         */
-        smp_mb();  /*  ^^^^^^^^^^^^ */
-
-        /* Call for a memory barrier from each CPU. */
-        for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map)) {
-                per_cpu(rcu_mb_flag, cpu) = rcu_mb_needed;
-                dyntick_save_progress_counter(cpu);
-        }
-
-        RCU_TRACE_ME(rcupreempt_trace_try_flip_z2);
-        return 1;
-}
-
-/*
- * Wait for all CPUs to do their end-of-grace-period memory barrier.
- * Return 0 once all CPUs have done so.
- */
-
-static int
-rcu_try_flip_waitmb(void)
-{
-        int cpu;
-
-        RCU_TRACE_ME(rcupreempt_trace_try_flip_m1);
-        for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map))
-                if (rcu_try_flip_waitmb_needed(cpu) &&
-                    per_cpu(rcu_mb_flag, cpu) != rcu_mb_done) {
-                        RCU_TRACE_ME(rcupreempt_trace_try_flip_me1);
-                        return 0;
-                }
-
-        smp_mb(); /* Ensure that the above checks precede any following flip. */
-        RCU_TRACE_ME(rcupreempt_trace_try_flip_m2);
-        return 1;
-}
-
-/*
- * Attempt a single flip of the counters.  Remember, a single flip does
- * -not- constitute a grace period.  Instead, the interval between
- * at least GP_STAGES consecutive flips is a grace period.
- *
- * If anyone is nuts enough to run this CONFIG_PREEMPT_RCU implementation
- * on a large SMP, they might want to use a hierarchical organization of
- * the per-CPU-counter pairs.
- */
-static void rcu_try_flip(void)
-{
-        unsigned long flags;
-
-        RCU_TRACE_ME(rcupreempt_trace_try_flip_1);
-        if (unlikely(!spin_trylock_irqsave(&rcu_ctrlblk.fliplock, flags))) {
-                RCU_TRACE_ME(rcupreempt_trace_try_flip_e1);
-                return;
-        }
-
-        /*
-         * Take the next transition(s) through the RCU grace-period
-         * flip-counter state machine.
-         */
-
-        switch (rcu_ctrlblk.rcu_try_flip_state) {
-        case rcu_try_flip_idle_state:
-                if (rcu_try_flip_idle())
-                        rcu_ctrlblk.rcu_try_flip_state =
-                                rcu_try_flip_waitack_state;
-                break;
-        case rcu_try_flip_waitack_state:
-                if (rcu_try_flip_waitack())
-                        rcu_ctrlblk.rcu_try_flip_state =
-                                rcu_try_flip_waitzero_state;
-                break;
-        case rcu_try_flip_waitzero_state:
-                if (rcu_try_flip_waitzero())
-                        rcu_ctrlblk.rcu_try_flip_state =
-                                rcu_try_flip_waitmb_state;
-                break;
-        case rcu_try_flip_waitmb_state:
-                if (rcu_try_flip_waitmb())
-                        rcu_ctrlblk.rcu_try_flip_state =
-                                rcu_try_flip_idle_state;
-        }
-        spin_unlock_irqrestore(&rcu_ctrlblk.fliplock, flags);
-}
-
-/*
- * Check to see if this CPU needs to do a memory barrier in order to
- * ensure that any prior RCU read-side critical sections have committed
- * their counter manipulations and critical-section memory references
- * before declaring the grace period to be completed.
- */
-static void rcu_check_mb(int cpu)
-{
-        if (per_cpu(rcu_mb_flag, cpu) == rcu_mb_needed) {
-                smp_mb();  /* Ensure RCU read-side accesses are visible. */
-                per_cpu(rcu_mb_flag, cpu) = rcu_mb_done;
-        }
-}
-
-void rcu_check_callbacks(int cpu, int user)
-{
-        unsigned long flags;
-        struct rcu_data *rdp = RCU_DATA_CPU(cpu);
-
-        /*
-         * If this CPU took its interrupt from user mode or from the
-         * idle loop, and this is not a nested interrupt, then
-         * this CPU has to have exited all prior preept-disable
-         * sections of code.  So increment the counter to note this.
-         *
-         * The memory barrier is needed to handle the case where
-         * writes from a preempt-disable section of code get reordered
-         * into schedule() by this CPU's write buffer.  So the memory
-         * barrier makes sure that the rcu_qsctr_inc() is seen by other
-         * CPUs to happen after any such write.
-         */
-
-        if (user ||
-            (idle_cpu(cpu) && !in_softirq() &&
-             hardirq_count() <= (1 << HARDIRQ_SHIFT))) {
-                smp_mb();       /* Guard against aggressive schedule(). */
-                rcu_qsctr_inc(cpu);
-        }
-
-        rcu_check_mb(cpu);
-        if (rcu_ctrlblk.completed == rdp->completed)
-                rcu_try_flip();
-        spin_lock_irqsave(&rdp->lock, flags);
-        RCU_TRACE_RDP(rcupreempt_trace_check_callbacks, rdp);
-        __rcu_advance_callbacks(rdp);
-        if (rdp->donelist == NULL) {
-                spin_unlock_irqrestore(&rdp->lock, flags);
-        } else {
-                spin_unlock_irqrestore(&rdp->lock, flags);
-                raise_softirq(RCU_SOFTIRQ);
-        }
-}
-
-/*
- * Needed by dynticks, to make sure all RCU processing has finished
- * when we go idle:
- */
-void rcu_advance_callbacks(int cpu, int user)
-{
-        unsigned long flags;
-        struct rcu_data *rdp = RCU_DATA_CPU(cpu);
-
-        if (rcu_ctrlblk.completed == rdp->completed) {
-                rcu_try_flip();
-                if (rcu_ctrlblk.completed == rdp->completed)
-                        return;
-        }
-        spin_lock_irqsave(&rdp->lock, flags);
-        RCU_TRACE_RDP(rcupreempt_trace_check_callbacks, rdp);
-        __rcu_advance_callbacks(rdp);
-        spin_unlock_irqrestore(&rdp->lock, flags);
-}
-
-#ifdef CONFIG_HOTPLUG_CPU
-#define rcu_offline_cpu_enqueue(srclist, srctail, dstlist, dsttail) do { \
-                *dsttail = srclist; \
-                if (srclist != NULL) { \
-                        dsttail = srctail; \
-                        srclist = NULL; \
-                        srctail = &srclist;\
-                } \
-        } while (0)
-
-void rcu_offline_cpu(int cpu)
-{
-        int i;
-        struct rcu_head *list = NULL;
-        unsigned long flags;
-        struct rcu_data *rdp = RCU_DATA_CPU(cpu);
-        struct rcu_head *schedlist = NULL;
-        struct rcu_head **schedtail = &schedlist;
-        struct rcu_head **tail = &list;
-
-        /*
-         * Remove all callbacks from the newly dead CPU, retaining order.
-         * Otherwise rcu_barrier() will fail
-         */
-
-        spin_lock_irqsave(&rdp->lock, flags);
-        rcu_offline_cpu_enqueue(rdp->donelist, rdp->donetail, list, tail);
-        for (i = GP_STAGES - 1; i >= 0; i--)
-                rcu_offline_cpu_enqueue(rdp->waitlist[i], rdp->waittail[i],
-                                                list, tail);
-        rcu_offline_cpu_enqueue(rdp->nextlist, rdp->nexttail, list, tail);
-        rcu_offline_cpu_enqueue(rdp->waitschedlist, rdp->waitschedtail,
-                                schedlist, schedtail);
-        rcu_offline_cpu_enqueue(rdp->nextschedlist, rdp->nextschedtail,
-                                schedlist, schedtail);
-        rdp->rcu_sched_sleeping = 0;
-        spin_unlock_irqrestore(&rdp->lock, flags);
-        rdp->waitlistcount = 0;
-
-        /* Disengage the newly dead CPU from the grace-period computation. */
-
-        spin_lock_irqsave(&rcu_ctrlblk.fliplock, flags);
-        rcu_check_mb(cpu);
-        if (per_cpu(rcu_flip_flag, cpu) == rcu_flipped) {
-                smp_mb();  /* Subsequent counter accesses must see new value */
-                per_cpu(rcu_flip_flag, cpu) = rcu_flip_seen;
-                smp_mb();  /* Subsequent RCU read-side critical sections */
-                           /*  seen -after- acknowledgement. */
-        }
-
-        RCU_DATA_ME()->rcu_flipctr[0] += RCU_DATA_CPU(cpu)->rcu_flipctr[0];
-        RCU_DATA_ME()->rcu_flipctr[1] += RCU_DATA_CPU(cpu)->rcu_flipctr[1];
-
-        RCU_DATA_CPU(cpu)->rcu_flipctr[0] = 0;
-        RCU_DATA_CPU(cpu)->rcu_flipctr[1] = 0;
-
-        cpumask_clear_cpu(cpu, to_cpumask(rcu_cpu_online_map));
-
-        spin_unlock_irqrestore(&rcu_ctrlblk.fliplock, flags);
-
-        /*
-         * Place the removed callbacks on the current CPU's queue.
-         * Make them all start a new grace period: simple approach,
-         * in theory could starve a given set of callbacks, but
-         * you would need to be doing some serious CPU hotplugging
-         * to make this happen.  If this becomes a problem, adding
-         * a synchronize_rcu() to the hotplug path would be a simple
-         * fix.
-         */
-
-        local_irq_save(flags);  /* disable preempt till we know what lock. */
-        rdp = RCU_DATA_ME();
-        spin_lock(&rdp->lock);
-        *rdp->nexttail = list;
-        if (list)
-                rdp->nexttail = tail;
-        *rdp->nextschedtail = schedlist;
-        if (schedlist)
-                rdp->nextschedtail = schedtail;
-        spin_unlock_irqrestore(&rdp->lock, flags);
-}
-
-#else /* #ifdef CONFIG_HOTPLUG_CPU */
-
-void rcu_offline_cpu(int cpu)
-{
-}
-
-#endif /* #else #ifdef CONFIG_HOTPLUG_CPU */
-
-void __cpuinit rcu_online_cpu(int cpu)
-{
-        unsigned long flags;
-        struct rcu_data *rdp;
-
-        spin_lock_irqsave(&rcu_ctrlblk.fliplock, flags);
-        cpumask_set_cpu(cpu, to_cpumask(rcu_cpu_online_map));
-        spin_unlock_irqrestore(&rcu_ctrlblk.fliplock, flags);
-
-        /*
-         * The rcu_sched grace-period processing might have bypassed
-         * this CPU, given that it was not in the rcu_cpu_online_map
-         * when the grace-period scan started.  This means that the
-         * grace-period task might sleep.  So make sure that if this
-         * should happen, the first callback posted to this CPU will
-         * wake up the grace-period task if need be.
-         */
-
-        rdp = RCU_DATA_CPU(cpu);
-        spin_lock_irqsave(&rdp->lock, flags);
-        rdp->rcu_sched_sleeping = 1;
-        spin_unlock_irqrestore(&rdp->lock, flags);
-}
-
-static void rcu_process_callbacks(struct softirq_action *unused)
-{
-        unsigned long flags;
-        struct rcu_head *next, *list;
-        struct rcu_data *rdp;
-
-        local_irq_save(flags);
-        rdp = RCU_DATA_ME();
-        spin_lock(&rdp->lock);
-        list = rdp->donelist;
-        if (list == NULL) {
-                spin_unlock_irqrestore(&rdp->lock, flags);
-                return;
-        }
-        rdp->donelist = NULL;
-        rdp->donetail = &rdp->donelist;
-        RCU_TRACE_RDP(rcupreempt_trace_done_remove, rdp);
-        spin_unlock_irqrestore(&rdp->lock, flags);
-        while (list) {
-                next = list->next;
-                list->func(list);
-                list = next;
-                RCU_TRACE_ME(rcupreempt_trace_invoke);
-        }
-}
-
-void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
-{
-        unsigned long flags;
-        struct rcu_data *rdp;
-
-        head->func = func;
-        head->next = NULL;
-        local_irq_save(flags);
-        rdp = RCU_DATA_ME();
-        spin_lock(&rdp->lock);
-        __rcu_advance_callbacks(rdp);
-        *rdp->nexttail = head;
-        rdp->nexttail = &head->next;
-        RCU_TRACE_RDP(rcupreempt_trace_next_add, rdp);
-        spin_unlock_irqrestore(&rdp->lock, flags);
-}
-EXPORT_SYMBOL_GPL(call_rcu);
-
-void call_rcu_sched(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
-{
-        unsigned long flags;
-        struct rcu_data *rdp;
-        int wake_gp = 0;
-
-        head->func = func;
-        head->next = NULL;
-        local_irq_save(flags);
-        rdp = RCU_DATA_ME();
-        spin_lock(&rdp->lock);
-        *rdp->nextschedtail = head;
-        rdp->nextschedtail = &head->next;
-        if (rdp->rcu_sched_sleeping) {
-
-                /* Grace-period processing might be sleeping... */
-
-                rdp->rcu_sched_sleeping = 0;
-                wake_gp = 1;
-        }
-        spin_unlock_irqrestore(&rdp->lock, flags);
-        if (wake_gp) {
-
-                /* Wake up grace-period processing, unless someone beat us. */
-
-                spin_lock_irqsave(&rcu_ctrlblk.schedlock, flags);
-                if (rcu_ctrlblk.sched_sleep != rcu_sched_sleeping)
-                        wake_gp = 0;
-                rcu_ctrlblk.sched_sleep = rcu_sched_not_sleeping;
-                spin_unlock_irqrestore(&rcu_ctrlblk.schedlock, flags);
-                if (wake_gp)
-                        wake_up_interruptible(&rcu_ctrlblk.sched_wq);
-        }
-}
-EXPORT_SYMBOL_GPL(call_rcu_sched);
-
-/*
- * Wait until all currently running preempt_disable() code segments
- * (including hardware-irq-disable segments) complete.  Note that
- * in -rt this does -not- necessarily result in all currently executing
- * interrupt -handlers- having completed.
- */
-void __synchronize_sched(void)
-{
-        struct rcu_synchronize rcu;
-
-        if (num_online_cpus() == 1)
-                return;  /* blocking is gp if only one CPU! */
-
-        init_completion(&rcu.completion);
-        /* Will wake me after RCU finished. */
-        call_rcu_sched(&rcu.head, wakeme_after_rcu);
-        /* Wait for it. */
-        wait_for_completion(&rcu.completion);
-}
-EXPORT_SYMBOL_GPL(__synchronize_sched);
-
-/*
- * kthread function that manages call_rcu_sched grace periods.
- */
-static int rcu_sched_grace_period(void *arg)
-{
-        int couldsleep;         /* might sleep after current pass. */
-        int couldsleepnext = 0; /* might sleep after next pass. */
-        int cpu;
-        unsigned long flags;
-        struct rcu_data *rdp;
-        int ret;
-
-        /*
-         * Each pass through the following loop handles one
-         * rcu_sched grace period cycle.
-         */
-        do {
-                /* Save each CPU's current state. */
-
-                for_each_online_cpu(cpu) {
-                        dyntick_save_progress_counter_sched(cpu);
-                        save_qsctr_sched(cpu);
-                }
-
-                /*
-                 * Sleep for about an RCU grace-period's worth to
-                 * allow better batching and to consume less CPU.
-                 */
-                schedule_timeout_interruptible(RCU_SCHED_BATCH_TIME);
-
-                /*
-                 * If there was nothing to do last time, prepare to
-                 * sleep at the end of the current grace period cycle.
-                 */
-                couldsleep = couldsleepnext;
-                couldsleepnext = 1;
-                if (couldsleep) {
-                        spin_lock_irqsave(&rcu_ctrlblk.schedlock, flags);
-                        rcu_ctrlblk.sched_sleep = rcu_sched_sleep_prep;
-                        spin_unlock_irqrestore(&rcu_ctrlblk.schedlock, flags);
-                }
-
-                /*
-                 * Wait on each CPU in turn to have either visited
-                 * a quiescent state or been in dynticks-idle mode.
-                 */
-                for_each_online_cpu(cpu) {
-                        while (rcu_qsctr_inc_needed(cpu) &&
-                               rcu_qsctr_inc_needed_dyntick(cpu)) {
-                                /* resched_cpu(cpu); @@@ */
-                                schedule_timeout_interruptible(1);
-                        }
-                }
-
-                /* Advance callbacks for each CPU.  */
-
-                for_each_online_cpu(cpu) {
-
-                        rdp = RCU_DATA_CPU(cpu);
-                        spin_lock_irqsave(&rdp->lock, flags);
-
-                        /*
-                         * We are running on this CPU irq-disabled, so no
-                         * CPU can go offline until we re-enable irqs.
-                         * The current CPU might have already gone
-                         * offline (between the for_each_offline_cpu and
-                         * the spin_lock_irqsave), but in that case all its
-                         * callback lists will be empty, so no harm done.
-                         *
-                         * Advance the callbacks!  We share normal RCU's
-                         * donelist, since callbacks are invoked the
-                         * same way in either case.
-                         */
-                        if (rdp->waitschedlist != NULL) {
-                                *rdp->donetail = rdp->waitschedlist;
-                                rdp->donetail = rdp->waitschedtail;
-
-                                /*
-                                 * Next rcu_check_callbacks() will
-                                 * do the required raise_softirq().
-                                 */
-                        }
-                        if (rdp->nextschedlist != NULL) {
-                                rdp->waitschedlist = rdp->nextschedlist;
-                                rdp->waitschedtail = rdp->nextschedtail;
-                                couldsleep = 0;
-                                couldsleepnext = 0;
-                        } else {
-                                rdp->waitschedlist = NULL;
-                                rdp->waitschedtail = &rdp->waitschedlist;
-                        }
-                        rdp->nextschedlist = NULL;
-                        rdp->nextschedtail = &rdp->nextschedlist;
-
-                        /* Mark sleep intention. */
-
-                        rdp->rcu_sched_sleeping = couldsleep;
-
-                        spin_unlock_irqrestore(&rdp->lock, flags);
-                }
-
-                /* If we saw callbacks on the last scan, go deal with them. */
-
-                if (!couldsleep)
-                        continue;
-
-                /* Attempt to block... */
-
-                spin_lock_irqsave(&rcu_ctrlblk.schedlock, flags);
-                if (rcu_ctrlblk.sched_sleep != rcu_sched_sleep_prep) {
-
-                        /*
-                         * Someone posted a callback after we scanned.
-                         * Go take care of it.
-                         */
-                        spin_unlock_irqrestore(&rcu_ctrlblk.schedlock, flags);
-                        couldsleepnext = 0;
-                        continue;
-                }
-
-                /* Block until the next person posts a callback. */
-
-                rcu_ctrlblk.sched_sleep = rcu_sched_sleeping;
-                spin_unlock_irqrestore(&rcu_ctrlblk.schedlock, flags);
-                ret = 0; /* unused */
-                __wait_event_interruptible(rcu_ctrlblk.sched_wq,
-                        rcu_ctrlblk.sched_sleep != rcu_sched_sleeping,
-                        ret);
-
-                couldsleepnext = 0;
-
-        } while (!kthread_should_stop());
-
-        return (0);
-}
-
-/*
- * Check to see if any future RCU-related work will need to be done
- * by the current CPU, even if none need be done immediately, returning
- * 1 if so.  Assumes that notifiers would take care of handling any
- * outstanding requests from the RCU core.
- *
- * This function is part of the RCU implementation; it is -not-
- * an exported member of the RCU API.
- */
-int rcu_needs_cpu(int cpu)
-{
-        struct rcu_data *rdp = RCU_DATA_CPU(cpu);
-
-        return (rdp->donelist != NULL ||
-                !!rdp->waitlistcount ||
-                rdp->nextlist != NULL ||
-                rdp->nextschedlist != NULL ||
-                rdp->waitschedlist != NULL);
-}
-
-int rcu_pending(int cpu)
-{
-        struct rcu_data *rdp = RCU_DATA_CPU(cpu);
-
-        /* The CPU has at least one callback queued somewhere. */
-
-        if (rdp->donelist != NULL ||
-            !!rdp->waitlistcount ||
-            rdp->nextlist != NULL ||
-            rdp->nextschedlist != NULL ||
-            rdp->waitschedlist != NULL)
-                return 1;
-
-        /* The RCU core needs an acknowledgement from this CPU. */
-
-        if ((per_cpu(rcu_flip_flag, cpu) == rcu_flipped) ||
-            (per_cpu(rcu_mb_flag, cpu) == rcu_mb_needed))
-                return 1;
-
-        /* This CPU has fallen behind the global grace-period number. */
-
-        if (rdp->completed != rcu_ctrlblk.completed)
-                return 1;
-
-        /* Nothing needed from this CPU. */
-
-        return 0;
-}
-
-static int __cpuinit rcu_cpu_notify(struct notifier_block *self,
-                                unsigned long action, void *hcpu)
-{
-        long cpu = (long)hcpu;
-
-        switch (action) {
-        case CPU_UP_PREPARE:
-        case CPU_UP_PREPARE_FROZEN:
-                rcu_online_cpu(cpu);
-                break;
-        case CPU_UP_CANCELED:
-        case CPU_UP_CANCELED_FROZEN:
-        case CPU_DEAD:
-        case CPU_DEAD_FROZEN:
-                rcu_offline_cpu(cpu);
-                break;
-        default:
-                break;
-        }
-        return NOTIFY_OK;
-}
-
-static struct notifier_block __cpuinitdata rcu_nb = {
-        .notifier_call = rcu_cpu_notify,
-};
-
-void __init __rcu_init(void)
-{
-        int cpu;
-        int i;
-        struct rcu_data *rdp;
-
-        printk(KERN_NOTICE "Preemptible RCU implementation.\n");
-        for_each_possible_cpu(cpu) {
-                rdp = RCU_DATA_CPU(cpu);
-                spin_lock_init(&rdp->lock);
-                rdp->completed = 0;
-                rdp->waitlistcount = 0;
-                rdp->nextlist = NULL;
-                rdp->nexttail = &rdp->nextlist;
-                for (i = 0; i < GP_STAGES; i++) {
-                        rdp->waitlist[i] = NULL;
-                        rdp->waittail[i] = &rdp->waitlist[i];
-                }
-                rdp->donelist = NULL;
-                rdp->donetail = &rdp->donelist;
-                rdp->rcu_flipctr[0] = 0;
-                rdp->rcu_flipctr[1] = 0;
-                rdp->nextschedlist = NULL;
-                rdp->nextschedtail = &rdp->nextschedlist;
-                rdp->waitschedlist = NULL;
-                rdp->waitschedtail = &rdp->waitschedlist;
-                rdp->rcu_sched_sleeping = 0;
-        }
-        register_cpu_notifier(&rcu_nb);
-
-        /*
-         * We don't need protection against CPU-Hotplug here
-         * since
-         * a) If a CPU comes online while we are iterating over the
-         *    cpu_online_mask below, we would only end up making a
-         *    duplicate call to rcu_online_cpu() which sets the corresponding
-         *    CPU's mask in the rcu_cpu_online_map.
-         *
-         * b) A CPU cannot go offline at this point in time since the user
-         *    does not have access to the sysfs interface, nor do we
-         *    suspend the system.
-         */
-        for_each_online_cpu(cpu)
-                rcu_cpu_notify(&rcu_nb, CPU_UP_PREPARE, (void *)(long) cpu);
-
-        open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
-}
-
-/*
- * Late-boot-time RCU initialization that must wait until after scheduler
- * has been initialized.
- */
-void __init rcu_init_sched(void)
-{
-        rcu_sched_grace_period_task = kthread_run(rcu_sched_grace_period,
-                                                  NULL,
-                                                  "rcu_sched_grace_period");
-        WARN_ON(IS_ERR(rcu_sched_grace_period_task));
-}
-
-#ifdef CONFIG_RCU_TRACE
-long *rcupreempt_flipctr(int cpu)
-{
-        return &RCU_DATA_CPU(cpu)->rcu_flipctr[0];
-}
-EXPORT_SYMBOL_GPL(rcupreempt_flipctr);
-
-int rcupreempt_flip_flag(int cpu)
-{
-        return per_cpu(rcu_flip_flag, cpu);
-}
-EXPORT_SYMBOL_GPL(rcupreempt_flip_flag);
-
-int rcupreempt_mb_flag(int cpu)
-{
-        return per_cpu(rcu_mb_flag, cpu);
-}
-EXPORT_SYMBOL_GPL(rcupreempt_mb_flag);
-
-char *rcupreempt_try_flip_state_name(void)
-{
-        return rcu_try_flip_state_names[rcu_ctrlblk.rcu_try_flip_state];
-}
-EXPORT_SYMBOL_GPL(rcupreempt_try_flip_state_name);
-
-struct rcupreempt_trace *rcupreempt_trace_cpu(int cpu)
-{
-        struct rcu_data *rdp = RCU_DATA_CPU(cpu);
-
-        return &rdp->trace;
-}
-EXPORT_SYMBOL_GPL(rcupreempt_trace_cpu);
-
-#endif /* #ifdef RCU_TRACE */
diff --git a/kernel/rcupreempt_trace.c b/kernel/rcupreempt_trace.c
deleted file mode 100644
index 7c2665cac172..000000000000
--- a/kernel/rcupreempt_trace.c
+++ /dev/null
@@ -1,334 +0,0 @@
-/*
- * Read-Copy Update tracing for realtime implementation
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
- *
- * Copyright IBM Corporation, 2006
- *
- * Papers:  http://www.rdrop.com/users/paulmck/RCU
- *
- * For detailed explanation of Read-Copy Update mechanism see -
- *              Documentation/RCU/ *.txt
- *
- */
-#include <linux/types.h>
-#include <linux/kernel.h>
-#include <linux/init.h>
-#include <linux/spinlock.h>
-#include <linux/smp.h>
-#include <linux/rcupdate.h>
-#include <linux/interrupt.h>
-#include <linux/sched.h>
-#include <asm/atomic.h>
-#include <linux/bitops.h>
-#include <linux/module.h>
-#include <linux/completion.h>
-#include <linux/moduleparam.h>
-#include <linux/percpu.h>
-#include <linux/notifier.h>
-#include <linux/cpu.h>
-#include <linux/mutex.h>
-#include <linux/rcupreempt_trace.h>
-#include <linux/debugfs.h>
-
-static struct mutex rcupreempt_trace_mutex;
-static char *rcupreempt_trace_buf;
-#define RCUPREEMPT_TRACE_BUF_SIZE 4096
-
-void rcupreempt_trace_move2done(struct rcupreempt_trace *trace)
-{
-        trace->done_length += trace->wait_length;
-        trace->done_add += trace->wait_length;
-        trace->wait_length = 0;
-}
-void rcupreempt_trace_move2wait(struct rcupreempt_trace *trace)
-{
-        trace->wait_length += trace->next_length;
-        trace->wait_add += trace->next_length;
-        trace->next_length = 0;
-}
-void rcupreempt_trace_try_flip_1(struct rcupreempt_trace *trace)
-{
-        atomic_inc(&trace->rcu_try_flip_1);
-}
-void rcupreempt_trace_try_flip_e1(struct rcupreempt_trace *trace)
-{
-        atomic_inc(&trace->rcu_try_flip_e1);
-}
-void rcupreempt_trace_try_flip_i1(struct rcupreempt_trace *trace)
-{
-        trace->rcu_try_flip_i1++;
-}
-void rcupreempt_trace_try_flip_ie1(struct rcupreempt_trace *trace)
-{
-        trace->rcu_try_flip_ie1++;
-}
-void rcupreempt_trace_try_flip_g1(struct rcupreempt_trace *trace)
-{
-        trace->rcu_try_flip_g1++;
-}
-void rcupreempt_trace_try_flip_a1(struct rcupreempt_trace *trace)
-{
-        trace->rcu_try_flip_a1++;
-}
-void rcupreempt_trace_try_flip_ae1(struct rcupreempt_trace *trace)
-{
-        trace->rcu_try_flip_ae1++;
-}
-void rcupreempt_trace_try_flip_a2(struct rcupreempt_trace *trace)
-{
-        trace->rcu_try_flip_a2++;
-}
-void rcupreempt_trace_try_flip_z1(struct rcupreempt_trace *trace)
-{
-        trace->rcu_try_flip_z1++;
-}
-void rcupreempt_trace_try_flip_ze1(struct rcupreempt_trace *trace)
-{
-        trace->rcu_try_flip_ze1++;
-}
-void rcupreempt_trace_try_flip_z2(struct rcupreempt_trace *trace)
-{
-        trace->rcu_try_flip_z2++;
-}
-void rcupreempt_trace_try_flip_m1(struct rcupreempt_trace *trace)
-{
-        trace->rcu_try_flip_m1++;
-}
-void rcupreempt_trace_try_flip_me1(struct rcupreempt_trace *trace)
-{
-        trace->rcu_try_flip_me1++;
-}
-void rcupreempt_trace_try_flip_m2(struct rcupreempt_trace *trace)
-{
-        trace->rcu_try_flip_m2++;
-}
-void rcupreempt_trace_check_callbacks(struct rcupreempt_trace *trace)
-{
-        trace->rcu_check_callbacks++;
-}
-void rcupreempt_trace_done_remove(struct rcupreempt_trace *trace)
-{
-        trace->done_remove += trace->done_length;
-        trace->done_length = 0;
-}
-void rcupreempt_trace_invoke(struct rcupreempt_trace *trace)
-{
-        atomic_inc(&trace->done_invoked);
-}
-void rcupreempt_trace_next_add(struct rcupreempt_trace *trace)
-{
-        trace->next_add++;
-        trace->next_length++;
-}
-
-static void rcupreempt_trace_sum(struct rcupreempt_trace *sp)
-{
-        struct rcupreempt_trace *cp;
-        int cpu;
-
-        memset(sp, 0, sizeof(*sp));
-        for_each_possible_cpu(cpu) {
-                cp = rcupreempt_trace_cpu(cpu);
-                sp->next_length += cp->next_length;
-                sp->next_add += cp->next_add;
-                sp->wait_length += cp->wait_length;
-                sp->wait_add += cp->wait_add;
-                sp->done_length += cp->done_length;
-                sp->done_add += cp->done_add;
-                sp->done_remove += cp->done_remove;
-                atomic_add(atomic_read(&cp->done_invoked), &sp->done_invoked);
-                sp->rcu_check_callbacks += cp->rcu_check_callbacks;
-                atomic_add(atomic_read(&cp->rcu_try_flip_1),
-                           &sp->rcu_try_flip_1);
-                atomic_add(atomic_read(&cp->rcu_try_flip_e1),
-                           &sp->rcu_try_flip_e1);
-                sp->rcu_try_flip_i1 += cp->rcu_try_flip_i1;
-                sp->rcu_try_flip_ie1 += cp->rcu_try_flip_ie1;
-                sp->rcu_try_flip_g1 += cp->rcu_try_flip_g1;
-                sp->rcu_try_flip_a1 += cp->rcu_try_flip_a1;
-                sp->rcu_try_flip_ae1 += cp->rcu_try_flip_ae1;
-                sp->rcu_try_flip_a2 += cp->rcu_try_flip_a2;
-                sp->rcu_try_flip_z1 += cp->rcu_try_flip_z1;
-                sp->rcu_try_flip_ze1 += cp->rcu_try_flip_ze1;
-                sp->rcu_try_flip_z2 += cp->rcu_try_flip_z2;
-                sp->rcu_try_flip_m1 += cp->rcu_try_flip_m1;
-                sp->rcu_try_flip_me1 += cp->rcu_try_flip_me1;
-                sp->rcu_try_flip_m2 += cp->rcu_try_flip_m2;
-        }
-}
-
-static ssize_t rcustats_read(struct file *filp, char __user *buffer,
-                                size_t count, loff_t *ppos)
-{
-        struct rcupreempt_trace trace;
-        ssize_t bcount;
-        int cnt = 0;
-
-        rcupreempt_trace_sum(&trace);
-        mutex_lock(&rcupreempt_trace_mutex);
-        snprintf(&rcupreempt_trace_buf[cnt], RCUPREEMPT_TRACE_BUF_SIZE - cnt,
-                 "ggp=%ld rcc=%ld\n",
-                 rcu_batches_completed(),
-                 trace.rcu_check_callbacks);
-        snprintf(&rcupreempt_trace_buf[cnt], RCUPREEMPT_TRACE_BUF_SIZE - cnt,
-                 "na=%ld nl=%ld wa=%ld wl=%ld da=%ld dl=%ld dr=%ld di=%d\n"
-                 "1=%d e1=%d i1=%ld ie1=%ld g1=%ld a1=%ld ae1=%ld a2=%ld\n"
-                 "z1=%ld ze1=%ld z2=%ld m1=%ld me1=%ld m2=%ld\n",
-
-                 trace.next_add, trace.next_length,
-                 trace.wait_add, trace.wait_length,
-                 trace.done_add, trace.done_length,
-                 trace.done_remove, atomic_read(&trace.done_invoked),
-                 atomic_read(&trace.rcu_try_flip_1),
-                 atomic_read(&trace.rcu_try_flip_e1),
-                 trace.rcu_try_flip_i1, trace.rcu_try_flip_ie1,
-                 trace.rcu_try_flip_g1,
-                 trace.rcu_try_flip_a1, trace.rcu_try_flip_ae1,
-                         trace.rcu_try_flip_a2,
-                 trace.rcu_try_flip_z1, trace.rcu_try_flip_ze1,
-                         trace.rcu_try_flip_z2,
-                 trace.rcu_try_flip_m1, trace.rcu_try_flip_me1,
-                        trace.rcu_try_flip_m2);
-        bcount = simple_read_from_buffer(buffer, count, ppos,
-                        rcupreempt_trace_buf, strlen(rcupreempt_trace_buf));
-        mutex_unlock(&rcupreempt_trace_mutex);
-        return bcount;
-}
-
-static ssize_t rcugp_read(struct file *filp, char __user *buffer,
-                                size_t count, loff_t *ppos)
-{
-        long oldgp = rcu_batches_completed();
-        ssize_t bcount;
-
-        mutex_lock(&rcupreempt_trace_mutex);
-        synchronize_rcu();
-        snprintf(rcupreempt_trace_buf, RCUPREEMPT_TRACE_BUF_SIZE,
-                "oldggp=%ld  newggp=%ld\n", oldgp, rcu_batches_completed());
-        bcount = simple_read_from_buffer(buffer, count, ppos,
-                        rcupreempt_trace_buf, strlen(rcupreempt_trace_buf));
-        mutex_unlock(&rcupreempt_trace_mutex);
-        return bcount;
-}
-
-static ssize_t rcuctrs_read(struct file *filp, char __user *buffer,
-                                size_t count, loff_t *ppos)
-{
-        int cnt = 0;
-        int cpu;
-        int f = rcu_batches_completed() & 0x1;
-        ssize_t bcount;
-
-        mutex_lock(&rcupreempt_trace_mutex);
-
-        cnt += snprintf(&rcupreempt_trace_buf[cnt], RCUPREEMPT_TRACE_BUF_SIZE,
-                                "CPU last cur F M\n");
-        for_each_online_cpu(cpu) {
-                long *flipctr = rcupreempt_flipctr(cpu);
-                cnt += snprintf(&rcupreempt_trace_buf[cnt],
-                                RCUPREEMPT_TRACE_BUF_SIZE - cnt,
-                                        "%3d %4ld %3ld %d %d\n",
-                               cpu,
-                               flipctr[!f],
-                               flipctr[f],
-                               rcupreempt_flip_flag(cpu),
-                               rcupreempt_mb_flag(cpu));
-        }
-        cnt += snprintf(&rcupreempt_trace_buf[cnt],
-                        RCUPREEMPT_TRACE_BUF_SIZE - cnt,
-                        "ggp = %ld, state = %s\n",
-                        rcu_batches_completed(),
-                        rcupreempt_try_flip_state_name());
-        cnt += snprintf(&rcupreempt_trace_buf[cnt],
-                        RCUPREEMPT_TRACE_BUF_SIZE - cnt,
-                        "\n");
-        bcount = simple_read_from_buffer(buffer, count, ppos,
-                        rcupreempt_trace_buf, strlen(rcupreempt_trace_buf));
-        mutex_unlock(&rcupreempt_trace_mutex);
-        return bcount;
-}
-
-static struct file_operations rcustats_fops = {
-        .owner = THIS_MODULE,
-        .read = rcustats_read,
-};
-
-static struct file_operations rcugp_fops = {
-        .owner = THIS_MODULE,
-        .read = rcugp_read,
-};
-
-static struct file_operations rcuctrs_fops = {
-        .owner = THIS_MODULE,
-        .read = rcuctrs_read,
-};
-
-static struct dentry *rcudir, *statdir, *ctrsdir, *gpdir;
-static int rcupreempt_debugfs_init(void)
-{
-        rcudir = debugfs_create_dir("rcu", NULL);
-        if (!rcudir)
-                goto out;
-        statdir = debugfs_create_file("rcustats", 0444, rcudir,
-                                                NULL, &rcustats_fops);
-        if (!statdir)
-                goto free_out;
-
-        gpdir = debugfs_create_file("rcugp", 0444, rcudir, NULL, &rcugp_fops);
-        if (!gpdir)
-                goto free_out;
-
-        ctrsdir = debugfs_create_file("rcuctrs", 0444, rcudir,
-                                                NULL, &rcuctrs_fops);
-        if (!ctrsdir)
-                goto free_out;
-        return 0;
-free_out:
-        if (statdir)
-                debugfs_remove(statdir);
-        if (gpdir)
-                debugfs_remove(gpdir);
-        debugfs_remove(rcudir);
-out:
-        return 1;
-}
-
-static int __init rcupreempt_trace_init(void)
-{
-        int ret;
-
-        mutex_init(&rcupreempt_trace_mutex);
-        rcupreempt_trace_buf = kmalloc(RCUPREEMPT_TRACE_BUF_SIZE, GFP_KERNEL);
-        if (!rcupreempt_trace_buf)
-                return 1;
-        ret = rcupreempt_debugfs_init();
-        if (ret)
-                kfree(rcupreempt_trace_buf);
-        return ret;
-}
-
-static void __exit rcupreempt_trace_cleanup(void)
-{
-        debugfs_remove(statdir);
-        debugfs_remove(gpdir);
-        debugfs_remove(ctrsdir);
-        debugfs_remove(rcudir);
-        kfree(rcupreempt_trace_buf);
-}
-
-
-module_init(rcupreempt_trace_init);
-module_exit(rcupreempt_trace_cleanup);
diff --git a/kernel/rcutorture.c b/kernel/rcutorture.c
index 9b4a975a4b4a..b33db539a8ad 100644
--- a/kernel/rcutorture.c
+++ b/kernel/rcutorture.c
@@ -257,14 +257,14 @@ struct rcu_torture_ops {
         void (*init)(void);
         void (*cleanup)(void);
         int (*readlock)(void);
-        void (*readdelay)(struct rcu_random_state *rrsp);
+        void (*read_delay)(struct rcu_random_state *rrsp);
         void (*readunlock)(int idx);
         int (*completed)(void);
-        void (*deferredfree)(struct rcu_torture *p);
+        void (*deferred_free)(struct rcu_torture *p);
         void (*sync)(void);
         void (*cb_barrier)(void);
         int (*stats)(char *page);
-        int irqcapable;
+        int irq_capable;
         char *name;
 };
 static struct rcu_torture_ops *cur_ops = NULL;
@@ -320,7 +320,7 @@ rcu_torture_cb(struct rcu_head *p)
                 rp->rtort_mbtest = 0;
                 rcu_torture_free(rp);
         } else
-                cur_ops->deferredfree(rp);
+                cur_ops->deferred_free(rp);
 }
 
 static void rcu_torture_deferred_free(struct rcu_torture *p)
@@ -329,18 +329,18 @@ static void rcu_torture_deferred_free(struct rcu_torture *p)
 }
 
 static struct rcu_torture_ops rcu_ops = {
-        .init = NULL,
-        .cleanup = NULL,
-        .readlock = rcu_torture_read_lock,
-        .readdelay = rcu_read_delay,
-        .readunlock = rcu_torture_read_unlock,
-        .completed = rcu_torture_completed,
-        .deferredfree = rcu_torture_deferred_free,
-        .sync = synchronize_rcu,
-        .cb_barrier = rcu_barrier,
-        .stats = NULL,
-        .irqcapable = 1,
-        .name = "rcu"
+        .init           = NULL,
+        .cleanup        = NULL,
+        .readlock       = rcu_torture_read_lock,
+        .read_delay     = rcu_read_delay,
+        .readunlock     = rcu_torture_read_unlock,
+        .completed      = rcu_torture_completed,
+        .deferred_free  = rcu_torture_deferred_free,
+        .sync           = synchronize_rcu,
+        .cb_barrier     = rcu_barrier,
+        .stats          = NULL,
+        .irq_capable    = 1,
+        .name           = "rcu"
 };
 
 static void rcu_sync_torture_deferred_free(struct rcu_torture *p)
@@ -370,18 +370,18 @@ static void rcu_sync_torture_init(void)
 }
 
 static struct rcu_torture_ops rcu_sync_ops = {
-        .init = rcu_sync_torture_init,
-        .cleanup = NULL,
-        .readlock = rcu_torture_read_lock,
-        .readdelay = rcu_read_delay,
-        .readunlock = rcu_torture_read_unlock,
-        .completed = rcu_torture_completed,
-        .deferredfree = rcu_sync_torture_deferred_free,
-        .sync = synchronize_rcu,
-        .cb_barrier = NULL,
-        .stats = NULL,
-        .irqcapable = 1,
-        .name = "rcu_sync"
+        .init           = rcu_sync_torture_init,
+        .cleanup        = NULL,
+        .readlock       = rcu_torture_read_lock,
+        .read_delay     = rcu_read_delay,
+        .readunlock     = rcu_torture_read_unlock,
+        .completed      = rcu_torture_completed,
+        .deferred_free  = rcu_sync_torture_deferred_free,
+        .sync           = synchronize_rcu,
+        .cb_barrier     = NULL,
+        .stats          = NULL,
+        .irq_capable    = 1,
+        .name           = "rcu_sync"
 };
 
 /*
@@ -432,33 +432,33 @@ static void rcu_bh_torture_synchronize(void)
 }
 
 static struct rcu_torture_ops rcu_bh_ops = {
-        .init = NULL,
-        .cleanup = NULL,
-        .readlock = rcu_bh_torture_read_lock,
-        .readdelay = rcu_read_delay,  /* just reuse rcu's version. */
-        .readunlock = rcu_bh_torture_read_unlock,
-        .completed = rcu_bh_torture_completed,
-        .deferredfree = rcu_bh_torture_deferred_free,
-        .sync = rcu_bh_torture_synchronize,
-        .cb_barrier = rcu_barrier_bh,
-        .stats = NULL,
-        .irqcapable = 1,
-        .name = "rcu_bh"
+        .init           = NULL,
+        .cleanup        = NULL,
+        .readlock       = rcu_bh_torture_read_lock,
+        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
+        .readunlock     = rcu_bh_torture_read_unlock,
+        .completed      = rcu_bh_torture_completed,
+        .deferred_free  = rcu_bh_torture_deferred_free,
+        .sync           = rcu_bh_torture_synchronize,
+        .cb_barrier     = rcu_barrier_bh,
+        .stats          = NULL,
+        .irq_capable    = 1,
+        .name           = "rcu_bh"
 };
 
 static struct rcu_torture_ops rcu_bh_sync_ops = {
-        .init = rcu_sync_torture_init,
-        .cleanup = NULL,
-        .readlock = rcu_bh_torture_read_lock,
-        .readdelay = rcu_read_delay,  /* just reuse rcu's version. */
-        .readunlock = rcu_bh_torture_read_unlock,
-        .completed = rcu_bh_torture_completed,
-        .deferredfree = rcu_sync_torture_deferred_free,
-        .sync = rcu_bh_torture_synchronize,
-        .cb_barrier = NULL,
-        .stats = NULL,
-        .irqcapable = 1,
-        .name = "rcu_bh_sync"
+        .init           = rcu_sync_torture_init,
+        .cleanup        = NULL,
+        .readlock       = rcu_bh_torture_read_lock,
+        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
+        .readunlock     = rcu_bh_torture_read_unlock,
+        .completed      = rcu_bh_torture_completed,
+        .deferred_free  = rcu_sync_torture_deferred_free,
+        .sync           = rcu_bh_torture_synchronize,
+        .cb_barrier     = NULL,
+        .stats          = NULL,
+        .irq_capable    = 1,
+        .name           = "rcu_bh_sync"
 };
 
 /*
@@ -530,17 +530,17 @@ static int srcu_torture_stats(char *page)
 }
 
 static struct rcu_torture_ops srcu_ops = {
-        .init = srcu_torture_init,
-        .cleanup = srcu_torture_cleanup,
-        .readlock = srcu_torture_read_lock,
-        .readdelay = srcu_read_delay,
-        .readunlock = srcu_torture_read_unlock,
-        .completed = srcu_torture_completed,
-        .deferredfree = rcu_sync_torture_deferred_free,
-        .sync = srcu_torture_synchronize,
-        .cb_barrier = NULL,
-        .stats = srcu_torture_stats,
-        .name = "srcu"
+        .init           = srcu_torture_init,
+        .cleanup        = srcu_torture_cleanup,
+        .readlock       = srcu_torture_read_lock,
+        .read_delay     = srcu_read_delay,
+        .readunlock     = srcu_torture_read_unlock,
+        .completed      = srcu_torture_completed,
+        .deferred_free  = rcu_sync_torture_deferred_free,
+        .sync           = srcu_torture_synchronize,
+        .cb_barrier     = NULL,
+        .stats          = srcu_torture_stats,
+        .name           = "srcu"
 };
 
 /*
@@ -574,32 +574,49 @@ static void sched_torture_synchronize(void)
 }
 
 static struct rcu_torture_ops sched_ops = {
-        .init = rcu_sync_torture_init,
-        .cleanup = NULL,
-        .readlock = sched_torture_read_lock,
-        .readdelay = rcu_read_delay,  /* just reuse rcu's version. */
-        .readunlock = sched_torture_read_unlock,
-        .completed = sched_torture_completed,
-        .deferredfree = rcu_sched_torture_deferred_free,
-        .sync = sched_torture_synchronize,
-        .cb_barrier = rcu_barrier_sched,
-        .stats = NULL,
-        .irqcapable = 1,
-        .name = "sched"
+        .init           = rcu_sync_torture_init,
+        .cleanup        = NULL,
+        .readlock       = sched_torture_read_lock,
+        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
+        .readunlock     = sched_torture_read_unlock,
+        .completed      = sched_torture_completed,
+        .deferred_free  = rcu_sched_torture_deferred_free,
+        .sync           = sched_torture_synchronize,
+        .cb_barrier     = rcu_barrier_sched,
+        .stats          = NULL,
+        .irq_capable    = 1,
+        .name           = "sched"
 };
 
 static struct rcu_torture_ops sched_ops_sync = {
-        .init = rcu_sync_torture_init,
-        .cleanup = NULL,
-        .readlock = sched_torture_read_lock,
-        .readdelay = rcu_read_delay,  /* just reuse rcu's version. */
-        .readunlock = sched_torture_read_unlock,
-        .completed = sched_torture_completed,
-        .deferredfree = rcu_sync_torture_deferred_free,
-        .sync = sched_torture_synchronize,
-        .cb_barrier = NULL,
-        .stats = NULL,
-        .name = "sched_sync"
+        .init           = rcu_sync_torture_init,
+        .cleanup        = NULL,
+        .readlock       = sched_torture_read_lock,
+        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
+        .readunlock     = sched_torture_read_unlock,
+        .completed      = sched_torture_completed,
+        .deferred_free  = rcu_sync_torture_deferred_free,
+        .sync           = sched_torture_synchronize,
+        .cb_barrier     = NULL,
+        .stats          = NULL,
+        .name           = "sched_sync"
+};
+
+extern int rcu_expedited_torture_stats(char *page);
+
+static struct rcu_torture_ops sched_expedited_ops = {
+        .init           = rcu_sync_torture_init,
+        .cleanup        = NULL,
+        .readlock       = sched_torture_read_lock,
+        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
+        .readunlock     = sched_torture_read_unlock,
+        .completed      = sched_torture_completed,
+        .deferred_free  = rcu_sync_torture_deferred_free,
+        .sync           = synchronize_sched_expedited,
+        .cb_barrier     = NULL,
+        .stats          = rcu_expedited_torture_stats,
+        .irq_capable    = 1,
+        .name           = "sched_expedited"
 };
 
 /*
@@ -635,7 +652,7 @@ rcu_torture_writer(void *arg)
                                 i = RCU_TORTURE_PIPE_LEN;
                         atomic_inc(&rcu_torture_wcount[i]);
                         old_rp->rtort_pipe_count++;
-                        cur_ops->deferredfree(old_rp);
+                        cur_ops->deferred_free(old_rp);
                 }
                 rcu_torture_current_version++;
                 oldbatch = cur_ops->completed();
@@ -700,7 +717,7 @@ static void rcu_torture_timer(unsigned long unused)
         if (p->rtort_mbtest == 0)
                 atomic_inc(&n_rcu_torture_mberror);
         spin_lock(&rand_lock);
-        cur_ops->readdelay(&rand);
+        cur_ops->read_delay(&rand);
         n_rcu_torture_timers++;
         spin_unlock(&rand_lock);
         preempt_disable();
@@ -738,11 +755,11 @@ rcu_torture_reader(void *arg)
 
         VERBOSE_PRINTK_STRING("rcu_torture_reader task started");
         set_user_nice(current, 19);
-        if (irqreader && cur_ops->irqcapable)
+        if (irqreader && cur_ops->irq_capable)
                 setup_timer_on_stack(&t, rcu_torture_timer, 0);
 
         do {
-                if (irqreader && cur_ops->irqcapable) {
+                if (irqreader && cur_ops->irq_capable) {
                         if (!timer_pending(&t))
                                 mod_timer(&t, 1);
                 }
@@ -757,7 +774,7 @@ rcu_torture_reader(void *arg)
                 }
                 if (p->rtort_mbtest == 0)
                         atomic_inc(&n_rcu_torture_mberror);
-                cur_ops->readdelay(&rand);
+                cur_ops->read_delay(&rand);
                 preempt_disable();
                 pipe_count = p->rtort_pipe_count;
                 if (pipe_count > RCU_TORTURE_PIPE_LEN) {
@@ -778,7 +795,7 @@ rcu_torture_reader(void *arg)
         } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
         VERBOSE_PRINTK_STRING("rcu_torture_reader task stopping");
         rcutorture_shutdown_absorb("rcu_torture_reader");
-        if (irqreader && cur_ops->irqcapable)
+        if (irqreader && cur_ops->irq_capable)
                 del_timer_sync(&t);
         while (!kthread_should_stop())
                 schedule_timeout_uninterruptible(1);
@@ -1078,6 +1095,7 @@ rcu_torture_init(void)
         int firsterr = 0;
         static struct rcu_torture_ops *torture_ops[] =
                 { &rcu_ops, &rcu_sync_ops, &rcu_bh_ops, &rcu_bh_sync_ops,
+                  &sched_expedited_ops,
                   &srcu_ops, &sched_ops, &sched_ops_sync, };
 
         mutex_lock(&fullstop_mutex);
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index 0dccfbba6d26..6b11b07cfe7f 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -35,6 +35,7 @@
 #include <linux/rcupdate.h>
 #include <linux/interrupt.h>
 #include <linux/sched.h>
+#include <linux/nmi.h>
 #include <asm/atomic.h>
 #include <linux/bitops.h>
 #include <linux/module.h>
@@ -46,6 +47,8 @@
 #include <linux/mutex.h>
 #include <linux/time.h>
 
+#include "rcutree.h"
+
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 static struct lock_class_key rcu_lock_key;
 struct lockdep_map rcu_lock_map =
@@ -72,30 +75,59 @@ EXPORT_SYMBOL_GPL(rcu_lock_map);
         .n_force_qs_ngp = 0, \
 }
 
-struct rcu_state rcu_state = RCU_STATE_INITIALIZER(rcu_state);
-DEFINE_PER_CPU(struct rcu_data, rcu_data);
+struct rcu_state rcu_sched_state = RCU_STATE_INITIALIZER(rcu_sched_state);
+DEFINE_PER_CPU(struct rcu_data, rcu_sched_data);
 
 struct rcu_state rcu_bh_state = RCU_STATE_INITIALIZER(rcu_bh_state);
 DEFINE_PER_CPU(struct rcu_data, rcu_bh_data);
 
+extern long rcu_batches_completed_sched(void);
+static struct rcu_node *rcu_get_root(struct rcu_state *rsp);
+static void cpu_quiet_msk(unsigned long mask, struct rcu_state *rsp,
+                          struct rcu_node *rnp, unsigned long flags);
+static void cpu_quiet_msk_finish(struct rcu_state *rsp, unsigned long flags);
+#ifdef CONFIG_HOTPLUG_CPU
+static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp);
+#endif /* #ifdef CONFIG_HOTPLUG_CPU */
+static void __rcu_process_callbacks(struct rcu_state *rsp,
+                                    struct rcu_data *rdp);
+static void __call_rcu(struct rcu_head *head,
+                       void (*func)(struct rcu_head *rcu),
+                       struct rcu_state *rsp);
+static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp);
+static void __cpuinit rcu_init_percpu_data(int cpu, struct rcu_state *rsp,
+                                           int preemptable);
+
+#include "rcutree_plugin.h"
+
 /*
- * Increment the quiescent state counter.
- * The counter is a bit degenerated: We do not need to know
+ * Note a quiescent state.  Because we do not need to know
  * how many quiescent states passed, just if there was at least
- * one since the start of the grace period. Thus just a flag.
+ * one since the start of the grace period, this just sets a flag.
  */
-void rcu_qsctr_inc(int cpu)
+void rcu_sched_qs(int cpu)
 {
-        struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
+        unsigned long flags;
+        struct rcu_data *rdp;
+
+        local_irq_save(flags);
+        rdp = &per_cpu(rcu_sched_data, cpu);
         rdp->passed_quiesc = 1;
         rdp->passed_quiesc_completed = rdp->completed;
+        rcu_preempt_qs(cpu);
+        local_irq_restore(flags);
 }
 
-void rcu_bh_qsctr_inc(int cpu)
+void rcu_bh_qs(int cpu)
 {
-        struct rcu_data *rdp = &per_cpu(rcu_bh_data, cpu);
+        unsigned long flags;
+        struct rcu_data *rdp;
+
+        local_irq_save(flags);
+        rdp = &per_cpu(rcu_bh_data, cpu);
         rdp->passed_quiesc = 1;
         rdp->passed_quiesc_completed = rdp->completed;
+        local_irq_restore(flags);
 }
 
 #ifdef CONFIG_NO_HZ
@@ -110,15 +142,16 @@ static int qhimark = 10000;	/* If this many pending, ignore blimit. */
 static int qlowmark = 100;      /* Once only this many pending, use blimit. */
 
 static void force_quiescent_state(struct rcu_state *rsp, int relaxed);
+static int rcu_pending(int cpu);
 
 /*
- * Return the number of RCU batches processed thus far for debug & stats.
+ * Return the number of RCU-sched batches processed thus far for debug & stats.
  */
-long rcu_batches_completed(void)
+long rcu_batches_completed_sched(void)
 {
-        return rcu_state.completed;
+        return rcu_sched_state.completed;
 }
-EXPORT_SYMBOL_GPL(rcu_batches_completed);
+EXPORT_SYMBOL_GPL(rcu_batches_completed_sched);
 
 /*
  * Return the number of RCU BH batches processed thus far for debug & stats.
@@ -181,6 +214,10 @@ static int rcu_implicit_offline_qs(struct rcu_data *rdp)
                 return 1;
         }
 
+        /* If preemptable RCU, no point in sending reschedule IPI. */
+        if (rdp->preemptable)
+                return 0;
+
         /* The CPU is online, so send it a reschedule IPI. */
         if (rdp->cpu != smp_processor_id())
                 smp_send_reschedule(rdp->cpu);
@@ -193,7 +230,6 @@ static int rcu_implicit_offline_qs(struct rcu_data *rdp)
 #endif /* #ifdef CONFIG_SMP */
 
 #ifdef CONFIG_NO_HZ
-static DEFINE_RATELIMIT_STATE(rcu_rs, 10 * HZ, 5);
 
 /**
  * rcu_enter_nohz - inform RCU that current CPU is entering nohz
@@ -213,7 +249,7 @@ void rcu_enter_nohz(void)
         rdtp = &__get_cpu_var(rcu_dynticks);
         rdtp->dynticks++;
         rdtp->dynticks_nesting--;
-        WARN_ON_RATELIMIT(rdtp->dynticks & 0x1, &rcu_rs);
+        WARN_ON_ONCE(rdtp->dynticks & 0x1);
         local_irq_restore(flags);
 }
 
@@ -232,7 +268,7 @@ void rcu_exit_nohz(void)
         rdtp = &__get_cpu_var(rcu_dynticks);
         rdtp->dynticks++;
         rdtp->dynticks_nesting++;
-        WARN_ON_RATELIMIT(!(rdtp->dynticks & 0x1), &rcu_rs);
+        WARN_ON_ONCE(!(rdtp->dynticks & 0x1));
         local_irq_restore(flags);
         smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */
 }
@@ -251,7 +287,7 @@ void rcu_nmi_enter(void)
         if (rdtp->dynticks & 0x1)
                 return;
         rdtp->dynticks_nmi++;
-        WARN_ON_RATELIMIT(!(rdtp->dynticks_nmi & 0x1), &rcu_rs);
+        WARN_ON_ONCE(!(rdtp->dynticks_nmi & 0x1));
         smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */
 }
 
@@ -270,7 +306,7 @@ void rcu_nmi_exit(void)
                 return;
         smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */
         rdtp->dynticks_nmi++;
-        WARN_ON_RATELIMIT(rdtp->dynticks_nmi & 0x1, &rcu_rs);
+        WARN_ON_ONCE(rdtp->dynticks_nmi & 0x1);
 }
 
 /**
@@ -286,7 +322,7 @@ void rcu_irq_enter(void)
         if (rdtp->dynticks_nesting++)
                 return;
         rdtp->dynticks++;
-        WARN_ON_RATELIMIT(!(rdtp->dynticks & 0x1), &rcu_rs);
+        WARN_ON_ONCE(!(rdtp->dynticks & 0x1));
         smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */
 }
 
@@ -305,10 +341,10 @@ void rcu_irq_exit(void)
                 return;
         smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */
         rdtp->dynticks++;
-        WARN_ON_RATELIMIT(rdtp->dynticks & 0x1, &rcu_rs);
+        WARN_ON_ONCE(rdtp->dynticks & 0x1);
 
         /* If the interrupt queued a callback, get out of dyntick mode. */
-        if (__get_cpu_var(rcu_data).nxtlist ||
+        if (__get_cpu_var(rcu_sched_data).nxtlist ||
             __get_cpu_var(rcu_bh_data).nxtlist)
                 set_need_resched();
 }
@@ -461,6 +497,7 @@ static void print_other_cpu_stall(struct rcu_state *rsp)
 
         printk(KERN_ERR "INFO: RCU detected CPU stalls:");
         for (; rnp_cur < rnp_end; rnp_cur++) {
+                rcu_print_task_stall(rnp);
                 if (rnp_cur->qsmask == 0)
                         continue;
                 for (cpu = 0; cpu <= rnp_cur->grphi - rnp_cur->grplo; cpu++)
@@ -469,6 +506,8 @@ static void print_other_cpu_stall(struct rcu_state *rsp)
         }
         printk(" (detected by %d, t=%ld jiffies)\n",
                smp_processor_id(), (long)(jiffies - rsp->gp_start));
+        trigger_all_cpu_backtrace();
+
         force_quiescent_state(rsp, 0);  /* Kick them all. */
 }
 
@@ -479,12 +518,14 @@ static void print_cpu_stall(struct rcu_state *rsp)
 
         printk(KERN_ERR "INFO: RCU detected CPU %d stall (t=%lu jiffies)\n",
                         smp_processor_id(), jiffies - rsp->gp_start);
-        dump_stack();
+        trigger_all_cpu_backtrace();
+
         spin_lock_irqsave(&rnp->lock, flags);
         if ((long)(jiffies - rsp->jiffies_stall) >= 0)
                 rsp->jiffies_stall =
                         jiffies + RCU_SECONDS_TILL_STALL_RECHECK;
         spin_unlock_irqrestore(&rnp->lock, flags);
+
         set_need_resched();  /* kick ourselves to get things going. */
 }
 
@@ -674,6 +715,19 @@ rcu_process_gp_end(struct rcu_state *rsp, struct rcu_data *rdp)
 }
 
 /*
+ * Clean up after the prior grace period and let rcu_start_gp() start up
+ * the next grace period if one is needed.  Note that the caller must
+ * hold rnp->lock, as required by rcu_start_gp(), which will release it.
+ */
+static void cpu_quiet_msk_finish(struct rcu_state *rsp, unsigned long flags)
+        __releases(rnp->lock)
+{
+        rsp->completed = rsp->gpnum;
+        rcu_process_gp_end(rsp, rsp->rda[smp_processor_id()]);
+        rcu_start_gp(rsp, flags);  /* releases root node's rnp->lock. */
+}
+
+/*
  * Similar to cpu_quiet(), for which it is a helper function.  Allows
  * a group of CPUs to be quieted at one go, though all the CPUs in the
  * group must be represented by the same leaf rcu_node structure.
@@ -694,7 +748,7 @@ cpu_quiet_msk(unsigned long mask, struct rcu_state *rsp, struct rcu_node *rnp,
                         return;
                 }
                 rnp->qsmask &= ~mask;
-                if (rnp->qsmask != 0) {
+                if (rnp->qsmask != 0 || rcu_preempted_readers(rnp)) {
 
                         /* Other bits still set at this level, so done. */
                         spin_unlock_irqrestore(&rnp->lock, flags);
@@ -714,14 +768,10 @@ cpu_quiet_msk(unsigned long mask, struct rcu_state *rsp, struct rcu_node *rnp,
 
         /*
          * Get here if we are the last CPU to pass through a quiescent
-         * state for this grace period.  Clean up and let rcu_start_gp()
-         * start up the next grace period if one is needed.  Note that
-         * we still hold rnp->lock, as required by rcu_start_gp(), which
-         * will release it.
+         * state for this grace period.  Invoke cpu_quiet_msk_finish()
+         * to clean up and start the next grace period if one is needed.
          */
-        rsp->completed = rsp->gpnum;
-        rcu_process_gp_end(rsp, rsp->rda[smp_processor_id()]);
-        rcu_start_gp(rsp, flags);  /* releases rnp->lock. */
+        cpu_quiet_msk_finish(rsp, flags); /* releases rnp->lock. */
 }
 
 /*
@@ -828,11 +878,12 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)
                 spin_lock(&rnp->lock);          /* irqs already disabled. */
                 rnp->qsmaskinit &= ~mask;
                 if (rnp->qsmaskinit != 0) {
-                        spin_unlock(&rnp->lock); /* irqs already disabled. */
+                        spin_unlock(&rnp->lock); /* irqs remain disabled. */
                         break;
                 }
+                rcu_preempt_offline_tasks(rsp, rnp);
                 mask = rnp->grpmask;
-                spin_unlock(&rnp->lock);        /* irqs already disabled. */
+                spin_unlock(&rnp->lock);        /* irqs remain disabled. */
                 rnp = rnp->parent;
         } while (rnp != NULL);
         lastcomp = rsp->completed;
@@ -845,7 +896,7 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)
         /*
          * Move callbacks from the outgoing CPU to the running CPU.
          * Note that the outgoing CPU is now quiscent, so it is now
-         * (uncharacteristically) safe to access it rcu_data structure.
+         * (uncharacteristically) safe to access its rcu_data structure.
          * Note also that we must carefully retain the order of the
          * outgoing CPU's callbacks in order for rcu_barrier() to work
          * correctly.  Finally, note that we start all the callbacks
@@ -876,8 +927,9 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)
  */
 static void rcu_offline_cpu(int cpu)
 {
-        __rcu_offline_cpu(cpu, &rcu_state);
+        __rcu_offline_cpu(cpu, &rcu_sched_state);
         __rcu_offline_cpu(cpu, &rcu_bh_state);
+        rcu_preempt_offline_cpu(cpu);
 }
 
 #else /* #ifdef CONFIG_HOTPLUG_CPU */
@@ -963,6 +1015,8 @@ static void rcu_do_batch(struct rcu_data *rdp)
  */
 void rcu_check_callbacks(int cpu, int user)
 {
+        if (!rcu_pending(cpu))
+                return; /* if nothing for RCU to do. */
         if (user ||
             (idle_cpu(cpu) && rcu_scheduler_active &&
              !in_softirq() && hardirq_count() <= (1 << HARDIRQ_SHIFT))) {
@@ -971,17 +1025,16 @@ void rcu_check_callbacks(int cpu, int user)
                  * Get here if this CPU took its interrupt from user
                  * mode or from the idle loop, and if this is not a
                  * nested interrupt.  In this case, the CPU is in
-                 * a quiescent state, so count it.
+                 * a quiescent state, so note it.
                  *
                  * No memory barrier is required here because both
-                 * rcu_qsctr_inc() and rcu_bh_qsctr_inc() reference
-                 * only CPU-local variables that other CPUs neither
-                 * access nor modify, at least not while the corresponding
-                 * CPU is online.
+                 * rcu_sched_qs() and rcu_bh_qs() reference only CPU-local
+                 * variables that other CPUs neither access nor modify,
+                 * at least not while the corresponding CPU is online.
                  */
 
-                rcu_qsctr_inc(cpu);
-                rcu_bh_qsctr_inc(cpu);
+                rcu_sched_qs(cpu);
+                rcu_bh_qs(cpu);
 
         } else if (!in_softirq()) {
 
@@ -989,11 +1042,12 @@ void rcu_check_callbacks(int cpu, int user)
                  * Get here if this CPU did not take its interrupt from
                  * softirq, in other words, if it is not interrupting
                  * a rcu_bh read-side critical section.  This is an _bh
-                 * critical section, so count it.
+                 * critical section, so note it.
                  */
 
-                rcu_bh_qsctr_inc(cpu);
+                rcu_bh_qs(cpu);
         }
+        rcu_preempt_check_callbacks(cpu);
         raise_softirq(RCU_SOFTIRQ);
 }
 
@@ -1132,6 +1186,8 @@ __rcu_process_callbacks(struct rcu_state *rsp, struct rcu_data *rdp)
 {
         unsigned long flags;
 
+        WARN_ON_ONCE(rdp->beenonline == 0);
+
         /*
          * If an RCU GP has gone long enough, go check for dyntick
          * idle CPUs and, if needed, send resched IPIs.
@@ -1170,8 +1226,10 @@ static void rcu_process_callbacks(struct softirq_action *unused)
          */
         smp_mb(); /* See above block comment. */
 
-        __rcu_process_callbacks(&rcu_state, &__get_cpu_var(rcu_data));
+        __rcu_process_callbacks(&rcu_sched_state,
+                                &__get_cpu_var(rcu_sched_data));
         __rcu_process_callbacks(&rcu_bh_state, &__get_cpu_var(rcu_bh_data));
+        rcu_preempt_process_callbacks();
 
         /*
          * Memory references from any later RCU read-side critical sections
@@ -1227,13 +1285,13 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
 }
 
 /*
- * Queue an RCU callback for invocation after a grace period.
+ * Queue an RCU-sched callback for invocation after a grace period.
  */
-void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
+void call_rcu_sched(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
 {
-        __call_rcu(head, func, &rcu_state);
+        __call_rcu(head, func, &rcu_sched_state);
 }
-EXPORT_SYMBOL_GPL(call_rcu);
+EXPORT_SYMBOL_GPL(call_rcu_sched);
 
 /*
  * Queue an RCU for invocation after a quicker grace period.
@@ -1305,10 +1363,11 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp)
  * by the current CPU, returning 1 if so.  This function is part of the
  * RCU implementation; it is -not- an exported member of the RCU API.
  */
-int rcu_pending(int cpu)
+static int rcu_pending(int cpu)
 {
-        return __rcu_pending(&rcu_state, &per_cpu(rcu_data, cpu)) ||
-               __rcu_pending(&rcu_bh_state, &per_cpu(rcu_bh_data, cpu));
+        return __rcu_pending(&rcu_sched_state, &per_cpu(rcu_sched_data, cpu)) ||
+               __rcu_pending(&rcu_bh_state, &per_cpu(rcu_bh_data, cpu)) ||
+               rcu_preempt_pending(cpu);
 }
 
 /*
@@ -1320,27 +1379,46 @@ int rcu_pending(int cpu)
 int rcu_needs_cpu(int cpu)
 {
         /* RCU callbacks either ready or pending? */
-        return per_cpu(rcu_data, cpu).nxtlist ||
-               per_cpu(rcu_bh_data, cpu).nxtlist;
+        return per_cpu(rcu_sched_data, cpu).nxtlist ||
+               per_cpu(rcu_bh_data, cpu).nxtlist ||
+               rcu_preempt_needs_cpu(cpu);
 }
 
 /*
- * Initialize a CPU's per-CPU RCU data.  We take this "scorched earth"
- * approach so that we don't have to worry about how long the CPU has
- * been gone, or whether it ever was online previously.  We do trust the
- * ->mynode field, as it is constant for a given struct rcu_data and
- * initialized during early boot.
- *
- * Note that only one online or offline event can be happening at a given
- * time.  Note also that we can accept some slop in the rsp->completed
- * access due to the fact that this CPU cannot possibly have any RCU
- * callbacks in flight yet.
+ * Do boot-time initialization of a CPU's per-CPU RCU data.
  */
-static void __cpuinit
-rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
+static void __init
+rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp)
 {
         unsigned long flags;
         int i;
+        struct rcu_data *rdp = rsp->rda[cpu];
+        struct rcu_node *rnp = rcu_get_root(rsp);
+
+        /* Set up local state, ensuring consistent view of global state. */
+        spin_lock_irqsave(&rnp->lock, flags);
+        rdp->grpmask = 1UL << (cpu - rdp->mynode->grplo);
+        rdp->nxtlist = NULL;
+        for (i = 0; i < RCU_NEXT_SIZE; i++)
+                rdp->nxttail[i] = &rdp->nxtlist;
+        rdp->qlen = 0;
+#ifdef CONFIG_NO_HZ
+        rdp->dynticks = &per_cpu(rcu_dynticks, cpu);
+#endif /* #ifdef CONFIG_NO_HZ */
+        rdp->cpu = cpu;
+        spin_unlock_irqrestore(&rnp->lock, flags);
+}
+
+/*
+ * Initialize a CPU's per-CPU RCU data.  Note that only one online or
+ * offline event can be happening at a given time.  Note also that we
+ * can accept some slop in the rsp->completed access due to the fact
+ * that this CPU cannot possibly have any RCU callbacks in flight yet.
+ */
+static void __cpuinit
+rcu_init_percpu_data(int cpu, struct rcu_state *rsp, int preemptable)
+{
+        unsigned long flags;
         long lastcomp;
         unsigned long mask;
         struct rcu_data *rdp = rsp->rda[cpu];
@@ -1354,17 +1432,9 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
         rdp->passed_quiesc = 0;  /* We could be racing with new GP, */
         rdp->qs_pending = 1;     /*  so set up to respond to current GP. */
         rdp->beenonline = 1;     /* We have now been online. */
+        rdp->preemptable = preemptable;
         rdp->passed_quiesc_completed = lastcomp - 1;
-        rdp->grpmask = 1UL << (cpu - rdp->mynode->grplo);
-        rdp->nxtlist = NULL;
-        for (i = 0; i < RCU_NEXT_SIZE; i++)
-                rdp->nxttail[i] = &rdp->nxtlist;
-        rdp->qlen = 0;
         rdp->blimit = blimit;
-#ifdef CONFIG_NO_HZ
-        rdp->dynticks = &per_cpu(rcu_dynticks, cpu);
-#endif /* #ifdef CONFIG_NO_HZ */
-        rdp->cpu = cpu;
         spin_unlock(&rnp->lock);                /* irqs remain disabled. */
 
         /*
@@ -1405,16 +1475,16 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
 
 static void __cpuinit rcu_online_cpu(int cpu)
 {
-        rcu_init_percpu_data(cpu, &rcu_state);
-        rcu_init_percpu_data(cpu, &rcu_bh_state);
-        open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
+        rcu_init_percpu_data(cpu, &rcu_sched_state, 0);
+        rcu_init_percpu_data(cpu, &rcu_bh_state, 0);
+        rcu_preempt_init_percpu_data(cpu);
 }
 
 /*
- * Handle CPU online/offline notifcation events.
+ * Handle CPU online/offline notification events.
  */
-static int __cpuinit rcu_cpu_notify(struct notifier_block *self,
-                                unsigned long action, void *hcpu)
+int __cpuinit rcu_cpu_notify(struct notifier_block *self,
+                             unsigned long action, void *hcpu)
 {
         long cpu = (long)hcpu;
 
@@ -1486,6 +1556,7 @@ static void __init rcu_init_one(struct rcu_state *rsp)
                 rnp = rsp->level[i];
                 for (j = 0; j < rsp->levelcnt[i]; j++, rnp++) {
                         spin_lock_init(&rnp->lock);
+                        rnp->gpnum = 0;
                         rnp->qsmask = 0;
                         rnp->qsmaskinit = 0;
                         rnp->grplo = j * cpustride;
@@ -1503,16 +1574,20 @@ static void __init rcu_init_one(struct rcu_state *rsp)
                                               j / rsp->levelspread[i - 1];
                         }
                         rnp->level = i;
+                        INIT_LIST_HEAD(&rnp->blocked_tasks[0]);
+                        INIT_LIST_HEAD(&rnp->blocked_tasks[1]);
                 }
         }
 }
 
 /*
- * Helper macro for __rcu_init().  To be used nowhere else!
- * Assigns leaf node pointers into each CPU's rcu_data structure.
+ * Helper macro for __rcu_init() and __rcu_init_preempt().  To be used
+ * nowhere else!  Assigns leaf node pointers into each CPU's rcu_data
+ * structure.
  */
-#define RCU_DATA_PTR_INIT(rsp, rcu_data) \
+#define RCU_INIT_FLAVOR(rsp, rcu_data) \
 do { \
+        rcu_init_one(rsp); \
         rnp = (rsp)->level[NUM_RCU_LVLS - 1]; \
         j = 0; \
         for_each_possible_cpu(i) { \
@@ -1520,33 +1595,43 @@ do { \
                         j++; \
                 per_cpu(rcu_data, i).mynode = &rnp[j]; \
                 (rsp)->rda[i] = &per_cpu(rcu_data, i); \
+                rcu_boot_init_percpu_data(i, rsp); \
         } \
 } while (0)
 
-static struct notifier_block __cpuinitdata rcu_nb = {
-        .notifier_call  = rcu_cpu_notify,
-};
+#ifdef CONFIG_TREE_PREEMPT_RCU
+
+void __init __rcu_init_preempt(void)
+{
+        int i;                  /* All used by RCU_INIT_FLAVOR(). */
+        int j;
+        struct rcu_node *rnp;
+
+        RCU_INIT_FLAVOR(&rcu_preempt_state, rcu_preempt_data);
+}
+
+#else /* #ifdef CONFIG_TREE_PREEMPT_RCU */
+
+void __init __rcu_init_preempt(void)
+{
+}
+
+#endif /* #else #ifdef CONFIG_TREE_PREEMPT_RCU */
 
 void __init __rcu_init(void)
 {
-        int i;                  /* All used by RCU_DATA_PTR_INIT(). */
+        int i;                  /* All used by RCU_INIT_FLAVOR(). */
         int j;
         struct rcu_node *rnp;
 
-        printk(KERN_WARNING "Experimental hierarchical RCU implementation.\n");
+        rcu_bootup_announce();
 #ifdef CONFIG_RCU_CPU_STALL_DETECTOR
         printk(KERN_INFO "RCU-based detection of stalled CPUs is enabled.\n");
 #endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
-        rcu_init_one(&rcu_state);
-        RCU_DATA_PTR_INIT(&rcu_state, rcu_data);
-        rcu_init_one(&rcu_bh_state);
-        RCU_DATA_PTR_INIT(&rcu_bh_state, rcu_bh_data);
-
-        for_each_online_cpu(i)
-                rcu_cpu_notify(&rcu_nb, CPU_UP_PREPARE, (void *)(long)i);
-        /* Register notifier for non-boot CPUs */
-        register_cpu_notifier(&rcu_nb);
-        printk(KERN_WARNING "Experimental hierarchical RCU init done.\n");
+        RCU_INIT_FLAVOR(&rcu_sched_state, rcu_sched_data);
+        RCU_INIT_FLAVOR(&rcu_bh_state, rcu_bh_data);
+        __rcu_init_preempt();
+        open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
 }
 
 module_param(blimit, int, 0);
diff --git a/kernel/rcutree.h b/kernel/rcutree.h
index 5e872bbf07f5..bf8a6f9f134d 100644
--- a/kernel/rcutree.h
+++ b/kernel/rcutree.h
@@ -1,10 +1,259 @@
+/*
+ * Read-Copy Update mechanism for mutual exclusion (tree-based version)
+ * Internal non-public definitions.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Copyright IBM Corporation, 2008
+ *
+ * Author: Ingo Molnar <mingo@elte.hu>
+ *         Paul E. McKenney <paulmck@linux.vnet.ibm.com>
+ */
+
+#include <linux/cache.h>
+#include <linux/spinlock.h>
+#include <linux/threads.h>
+#include <linux/cpumask.h>
+#include <linux/seqlock.h>
+
+/*
+ * Define shape of hierarchy based on NR_CPUS and CONFIG_RCU_FANOUT.
+ * In theory, it should be possible to add more levels straightforwardly.
+ * In practice, this has not been tested, so there is probably some
+ * bug somewhere.
+ */
+#define MAX_RCU_LVLS 3
+#define RCU_FANOUT            (CONFIG_RCU_FANOUT)
+#define RCU_FANOUT_SQ         (RCU_FANOUT * RCU_FANOUT)
+#define RCU_FANOUT_CUBE       (RCU_FANOUT_SQ * RCU_FANOUT)
+
+#if NR_CPUS <= RCU_FANOUT
+#  define NUM_RCU_LVLS        1
+#  define NUM_RCU_LVL_0       1
+#  define NUM_RCU_LVL_1       (NR_CPUS)
+#  define NUM_RCU_LVL_2       0
+#  define NUM_RCU_LVL_3       0
+#elif NR_CPUS <= RCU_FANOUT_SQ
+#  define NUM_RCU_LVLS        2
+#  define NUM_RCU_LVL_0       1
+#  define NUM_RCU_LVL_1       (((NR_CPUS) + RCU_FANOUT - 1) / RCU_FANOUT)
+#  define NUM_RCU_LVL_2       (NR_CPUS)
+#  define NUM_RCU_LVL_3       0
+#elif NR_CPUS <= RCU_FANOUT_CUBE
+#  define NUM_RCU_LVLS        3
+#  define NUM_RCU_LVL_0       1
+#  define NUM_RCU_LVL_1       (((NR_CPUS) + RCU_FANOUT_SQ - 1) / RCU_FANOUT_SQ)
+#  define NUM_RCU_LVL_2       (((NR_CPUS) + (RCU_FANOUT) - 1) / (RCU_FANOUT))
+#  define NUM_RCU_LVL_3       NR_CPUS
+#else
+# error "CONFIG_RCU_FANOUT insufficient for NR_CPUS"
+#endif /* #if (NR_CPUS) <= RCU_FANOUT */
+
+#define RCU_SUM (NUM_RCU_LVL_0 + NUM_RCU_LVL_1 + NUM_RCU_LVL_2 + NUM_RCU_LVL_3)
+#define NUM_RCU_NODES (RCU_SUM - NR_CPUS)
+
+/*
+ * Dynticks per-CPU state.
+ */
+struct rcu_dynticks {
+        int dynticks_nesting;   /* Track nesting level, sort of. */
+        int dynticks;           /* Even value for dynticks-idle, else odd. */
+        int dynticks_nmi;       /* Even value for either dynticks-idle or */
+                                /*  not in nmi handler, else odd.  So this */
+                                /*  remains even for nmi from irq handler. */
+};
+
+/*
+ * Definition for node within the RCU grace-period-detection hierarchy.
+ */
+struct rcu_node {
+        spinlock_t lock;
+        long    gpnum;          /* Current grace period for this node. */
+                                /*  This will either be equal to or one */
+                                /*  behind the root rcu_node's gpnum. */
+        unsigned long qsmask;   /* CPUs or groups that need to switch in */
+                                /*  order for current grace period to proceed.*/
+        unsigned long qsmaskinit;
+                                /* Per-GP initialization for qsmask. */
+        unsigned long grpmask;  /* Mask to apply to parent qsmask. */
+        int     grplo;          /* lowest-numbered CPU or group here. */
+        int     grphi;          /* highest-numbered CPU or group here. */
+        u8      grpnum;         /* CPU/group number for next level up. */
+        u8      level;          /* root is at level 0. */
+        struct rcu_node *parent;
+        struct list_head blocked_tasks[2];
+                                /* Tasks blocked in RCU read-side critsect. */
+} ____cacheline_internodealigned_in_smp;
+
+/* Index values for nxttail array in struct rcu_data. */
+#define RCU_DONE_TAIL           0       /* Also RCU_WAIT head. */
+#define RCU_WAIT_TAIL           1       /* Also RCU_NEXT_READY head. */
+#define RCU_NEXT_READY_TAIL     2       /* Also RCU_NEXT head. */
+#define RCU_NEXT_TAIL           3
+#define RCU_NEXT_SIZE           4
+
+/* Per-CPU data for read-copy update. */
+struct rcu_data {
+        /* 1) quiescent-state and grace-period handling : */
+        long            completed;      /* Track rsp->completed gp number */
+                                        /*  in order to detect GP end. */
+        long            gpnum;          /* Highest gp number that this CPU */
+                                        /*  is aware of having started. */
+        long            passed_quiesc_completed;
+                                        /* Value of completed at time of qs. */
+        bool            passed_quiesc;  /* User-mode/idle loop etc. */
+        bool            qs_pending;     /* Core waits for quiesc state. */
+        bool            beenonline;     /* CPU online at least once. */
+        bool            preemptable;    /* Preemptable RCU? */
+        struct rcu_node *mynode;        /* This CPU's leaf of hierarchy */
+        unsigned long grpmask;          /* Mask to apply to leaf qsmask. */
+
+        /* 2) batch handling */
+        /*
+         * If nxtlist is not NULL, it is partitioned as follows.
+         * Any of the partitions might be empty, in which case the
+         * pointer to that partition will be equal to the pointer for
+         * the following partition.  When the list is empty, all of
+         * the nxttail elements point to nxtlist, which is NULL.
+         *
+         * [*nxttail[RCU_NEXT_READY_TAIL], NULL = *nxttail[RCU_NEXT_TAIL]):
+         *      Entries that might have arrived after current GP ended
+         * [*nxttail[RCU_WAIT_TAIL], *nxttail[RCU_NEXT_READY_TAIL]):
+         *      Entries known to have arrived before current GP ended
+         * [*nxttail[RCU_DONE_TAIL], *nxttail[RCU_WAIT_TAIL]):
+         *      Entries that batch # <= ->completed - 1: waiting for current GP
+         * [nxtlist, *nxttail[RCU_DONE_TAIL]):
+         *      Entries that batch # <= ->completed
+         *      The grace period for these entries has completed, and
+         *      the other grace-period-completed entries may be moved
+         *      here temporarily in rcu_process_callbacks().
+         */
+        struct rcu_head *nxtlist;
+        struct rcu_head **nxttail[RCU_NEXT_SIZE];
+        long            qlen;           /* # of queued callbacks */
+        long            blimit;         /* Upper limit on a processed batch */
+
+#ifdef CONFIG_NO_HZ
+        /* 3) dynticks interface. */
+        struct rcu_dynticks *dynticks;  /* Shared per-CPU dynticks state. */
+        int dynticks_snap;              /* Per-GP tracking for dynticks. */
+        int dynticks_nmi_snap;          /* Per-GP tracking for dynticks_nmi. */
+#endif /* #ifdef CONFIG_NO_HZ */
+
+        /* 4) reasons this CPU needed to be kicked by force_quiescent_state */
+#ifdef CONFIG_NO_HZ
+        unsigned long dynticks_fqs;     /* Kicked due to dynticks idle. */
+#endif /* #ifdef CONFIG_NO_HZ */
+        unsigned long offline_fqs;      /* Kicked due to being offline. */
+        unsigned long resched_ipi;      /* Sent a resched IPI. */
+
+        /* 5) __rcu_pending() statistics. */
+        long n_rcu_pending;             /* rcu_pending() calls since boot. */
+        long n_rp_qs_pending;
+        long n_rp_cb_ready;
+        long n_rp_cpu_needs_gp;
+        long n_rp_gp_completed;
+        long n_rp_gp_started;
+        long n_rp_need_fqs;
+        long n_rp_need_nothing;
+
+        int cpu;
+};
+
+/* Values for signaled field in struct rcu_state. */
+#define RCU_GP_INIT             0       /* Grace period being initialized. */
+#define RCU_SAVE_DYNTICK        1       /* Need to scan dyntick state. */
+#define RCU_FORCE_QS            2       /* Need to force quiescent state. */
+#ifdef CONFIG_NO_HZ
+#define RCU_SIGNAL_INIT         RCU_SAVE_DYNTICK
+#else /* #ifdef CONFIG_NO_HZ */
+#define RCU_SIGNAL_INIT         RCU_FORCE_QS
+#endif /* #else #ifdef CONFIG_NO_HZ */
+
+#define RCU_JIFFIES_TILL_FORCE_QS        3      /* for rsp->jiffies_force_qs */
+#ifdef CONFIG_RCU_CPU_STALL_DETECTOR
+#define RCU_SECONDS_TILL_STALL_CHECK   (10 * HZ)  /* for rsp->jiffies_stall */
+#define RCU_SECONDS_TILL_STALL_RECHECK (30 * HZ)  /* for rsp->jiffies_stall */
+#define RCU_STALL_RAT_DELAY             2         /* Allow other CPUs time */
+                                                  /*  to take at least one */
+                                                  /*  scheduling clock irq */
+                                                  /*  before ratting on them. */
+
+#endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
+
+/*
+ * RCU global state, including node hierarchy.  This hierarchy is
+ * represented in "heap" form in a dense array.  The root (first level)
+ * of the hierarchy is in ->node[0] (referenced by ->level[0]), the second
+ * level in ->node[1] through ->node[m] (->node[1] referenced by ->level[1]),
+ * and the third level in ->node[m+1] and following (->node[m+1] referenced
+ * by ->level[2]).  The number of levels is determined by the number of
+ * CPUs and by CONFIG_RCU_FANOUT.  Small systems will have a "hierarchy"
+ * consisting of a single rcu_node.
+ */
+struct rcu_state {
+        struct rcu_node node[NUM_RCU_NODES];    /* Hierarchy. */
+        struct rcu_node *level[NUM_RCU_LVLS];   /* Hierarchy levels. */
+        u32 levelcnt[MAX_RCU_LVLS + 1];         /* # nodes in each level. */
+        u8 levelspread[NUM_RCU_LVLS];           /* kids/node in each level. */
+        struct rcu_data *rda[NR_CPUS];          /* array of rdp pointers. */
+
+        /* The following fields are guarded by the root rcu_node's lock. */
+
+        u8      signaled ____cacheline_internodealigned_in_smp;
+                                                /* Force QS state. */
+        long    gpnum;                          /* Current gp number. */
+        long    completed;                      /* # of last completed gp. */
+        spinlock_t onofflock;                   /* exclude on/offline and */
+                                                /*  starting new GP. */
+        spinlock_t fqslock;                     /* Only one task forcing */
+                                                /*  quiescent states. */
+        unsigned long jiffies_force_qs;         /* Time at which to invoke */
+                                                /*  force_quiescent_state(). */
+        unsigned long n_force_qs;               /* Number of calls to */
+                                                /*  force_quiescent_state(). */
+        unsigned long n_force_qs_lh;            /* ~Number of calls leaving */
+                                                /*  due to lock unavailable. */
+        unsigned long n_force_qs_ngp;           /* Number of calls leaving */
+                                                /*  due to no GP active. */
+#ifdef CONFIG_RCU_CPU_STALL_DETECTOR
+        unsigned long gp_start;                 /* Time at which GP started, */
+                                                /*  but in jiffies. */
+        unsigned long jiffies_stall;            /* Time at which to check */
+                                                /*  for CPU stalls. */
+#endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
+#ifdef CONFIG_NO_HZ
+        long dynticks_completed;                /* Value of completed @ snap. */
+#endif /* #ifdef CONFIG_NO_HZ */
+};
+
+#ifdef RCU_TREE_NONCORE
 
 /*
  * RCU implementation internal declarations:
  */
-extern struct rcu_state rcu_state;
-DECLARE_PER_CPU(struct rcu_data, rcu_data);
+extern struct rcu_state rcu_sched_state;
+DECLARE_PER_CPU(struct rcu_data, rcu_sched_data);
 
 extern struct rcu_state rcu_bh_state;
 DECLARE_PER_CPU(struct rcu_data, rcu_bh_data);
 
+#ifdef CONFIG_TREE_PREEMPT_RCU
+extern struct rcu_state rcu_preempt_state;
+DECLARE_PER_CPU(struct rcu_data, rcu_preempt_data);
+#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
+
+#endif /* #ifdef RCU_TREE_NONCORE */
+
diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h
new file mode 100644
index 000000000000..47789369ea59
--- /dev/null
+++ b/kernel/rcutree_plugin.h
@@ -0,0 +1,532 @@
+/*
+ * Read-Copy Update mechanism for mutual exclusion (tree-based version)
+ * Internal non-public definitions that provide either classic
+ * or preemptable semantics.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Copyright Red Hat, 2009
+ * Copyright IBM Corporation, 2009
+ *
+ * Author: Ingo Molnar <mingo@elte.hu>
+ *         Paul E. McKenney <paulmck@linux.vnet.ibm.com>
+ */
+
+
+#ifdef CONFIG_TREE_PREEMPT_RCU
+
+struct rcu_state rcu_preempt_state = RCU_STATE_INITIALIZER(rcu_preempt_state);
+DEFINE_PER_CPU(struct rcu_data, rcu_preempt_data);
+
+/*
+ * Tell them what RCU they are running.
+ */
+static inline void rcu_bootup_announce(void)
+{
+        printk(KERN_INFO
+               "Experimental preemptable hierarchical RCU implementation.\n");
+}
+
+/*
+ * Return the number of RCU-preempt batches processed thus far
+ * for debug and statistics.
+ */
+long rcu_batches_completed_preempt(void)
+{
+        return rcu_preempt_state.completed;
+}
+EXPORT_SYMBOL_GPL(rcu_batches_completed_preempt);
+
+/*
+ * Return the number of RCU batches processed thus far for debug & stats.
+ */
+long rcu_batches_completed(void)
+{
+        return rcu_batches_completed_preempt();
+}
+EXPORT_SYMBOL_GPL(rcu_batches_completed);
+
+/*
+ * Record a preemptable-RCU quiescent state for the specified CPU.  Note
+ * that this just means that the task currently running on the CPU is
+ * not in a quiescent state.  There might be any number of tasks blocked
+ * while in an RCU read-side critical section.
+ */
+static void rcu_preempt_qs_record(int cpu)
+{
+        struct rcu_data *rdp = &per_cpu(rcu_preempt_data, cpu);
+        rdp->passed_quiesc = 1;
+        rdp->passed_quiesc_completed = rdp->completed;
+}
+
+/*
+ * We have entered the scheduler or are between softirqs in ksoftirqd.
+ * If we are in an RCU read-side critical section, we need to reflect
+ * that in the state of the rcu_node structure corresponding to this CPU.
+ * Caller must disable hardirqs.
+ */
+static void rcu_preempt_qs(int cpu)
+{
+        struct task_struct *t = current;
+        int phase;
+        struct rcu_data *rdp;
+        struct rcu_node *rnp;
+
+        if (t->rcu_read_lock_nesting &&
+            (t->rcu_read_unlock_special & RCU_READ_UNLOCK_BLOCKED) == 0) {
+
+                /* Possibly blocking in an RCU read-side critical section. */
+                rdp = rcu_preempt_state.rda[cpu];
+                rnp = rdp->mynode;
+                spin_lock(&rnp->lock);
+                t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BLOCKED;
+                t->rcu_blocked_node = rnp;
+
+                /*
+                 * If this CPU has already checked in, then this task
+                 * will hold up the next grace period rather than the
+                 * current grace period.  Queue the task accordingly.
+                 * If the task is queued for the current grace period
+                 * (i.e., this CPU has not yet passed through a quiescent
+                 * state for the current grace period), then as long
+                 * as that task remains queued, the current grace period
+                 * cannot end.
+                 */
+                phase = !(rnp->qsmask & rdp->grpmask) ^ (rnp->gpnum & 0x1);
+                list_add(&t->rcu_node_entry, &rnp->blocked_tasks[phase]);
+                smp_mb();  /* Ensure later ctxt swtch seen after above. */
+                spin_unlock(&rnp->lock);
+        }
+
+        /*
+         * Either we were not in an RCU read-side critical section to
+         * begin with, or we have now recorded that critical section
+         * globally.  Either way, we can now note a quiescent state
+         * for this CPU.  Again, if we were in an RCU read-side critical
+         * section, and if that critical section was blocking the current
+         * grace period, then the fact that the task has been enqueued
+         * means that we continue to block the current grace period.
+         */
+        rcu_preempt_qs_record(cpu);
+        t->rcu_read_unlock_special &= ~(RCU_READ_UNLOCK_NEED_QS |
+                                        RCU_READ_UNLOCK_GOT_QS);
+}
+
+/*
+ * Tree-preemptable RCU implementation for rcu_read_lock().
+ * Just increment ->rcu_read_lock_nesting, shared state will be updated
+ * if we block.
+ */
+void __rcu_read_lock(void)
+{
+        ACCESS_ONCE(current->rcu_read_lock_nesting)++;
+        barrier();  /* needed if we ever invoke rcu_read_lock in rcutree.c */
+}
+EXPORT_SYMBOL_GPL(__rcu_read_lock);
+
+static void rcu_read_unlock_special(struct task_struct *t)
+{
+        int empty;
+        unsigned long flags;
+        unsigned long mask;
+        struct rcu_node *rnp;
+        int special;
+
+        /* NMI handlers cannot block and cannot safely manipulate state. */
+        if (in_nmi())
+                return;
+
+        local_irq_save(flags);
+
+        /*
+         * If RCU core is waiting for this CPU to exit critical section,
+         * let it know that we have done so.
+         */
+        special = t->rcu_read_unlock_special;
+        if (special & RCU_READ_UNLOCK_NEED_QS) {
+                t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_NEED_QS;
+                t->rcu_read_unlock_special |= RCU_READ_UNLOCK_GOT_QS;
+        }
+
+        /* Hardware IRQ handlers cannot block. */
+        if (in_irq()) {
+                local_irq_restore(flags);
+                return;
+        }
+
+        /* Clean up if blocked during RCU read-side critical section. */
+        if (special & RCU_READ_UNLOCK_BLOCKED) {
+                t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_BLOCKED;
+
+                /*
+                 * Remove this task from the list it blocked on.  The
+                 * task can migrate while we acquire the lock, but at
+                 * most one time.  So at most two passes through loop.
+                 */
+                for (;;) {
+                        rnp = t->rcu_blocked_node;
+                        spin_lock(&rnp->lock);
+                        if (rnp == t->rcu_blocked_node)
+                                break;
+                        spin_unlock(&rnp->lock);
+                }
+                empty = list_empty(&rnp->blocked_tasks[rnp->gpnum & 0x1]);
+                list_del_init(&t->rcu_node_entry);
+                t->rcu_blocked_node = NULL;
+
+                /*
+                 * If this was the last task on the current list, and if
+                 * we aren't waiting on any CPUs, report the quiescent state.
+                 * Note that both cpu_quiet_msk_finish() and cpu_quiet_msk()
+                 * drop rnp->lock and restore irq.
+                 */
+                if (!empty && rnp->qsmask == 0 &&
+                    list_empty(&rnp->blocked_tasks[rnp->gpnum & 0x1])) {
+                        t->rcu_read_unlock_special &=
+                                ~(RCU_READ_UNLOCK_NEED_QS |
+                                  RCU_READ_UNLOCK_GOT_QS);
+                        if (rnp->parent == NULL) {
+                                /* Only one rcu_node in the tree. */
+                                cpu_quiet_msk_finish(&rcu_preempt_state, flags);
+                                return;
+                        }
+                        /* Report up the rest of the hierarchy. */
+                        mask = rnp->grpmask;
+                        spin_unlock_irqrestore(&rnp->lock, flags);
+                        rnp = rnp->parent;
+                        spin_lock_irqsave(&rnp->lock, flags);
+                        cpu_quiet_msk(mask, &rcu_preempt_state, rnp, flags);
+                        return;
+                }
+                spin_unlock(&rnp->lock);
+        }
+        local_irq_restore(flags);
+}
+
+/*
+ * Tree-preemptable RCU implementation for rcu_read_unlock().
+ * Decrement ->rcu_read_lock_nesting.  If the result is zero (outermost
+ * rcu_read_unlock()) and ->rcu_read_unlock_special is non-zero, then
+ * invoke rcu_read_unlock_special() to clean up after a context switch
+ * in an RCU read-side critical section and other special cases.
+ */
+void __rcu_read_unlock(void)
+{
+        struct task_struct *t = current;
+
+        barrier();  /* needed if we ever invoke rcu_read_unlock in rcutree.c */
+        if (--ACCESS_ONCE(t->rcu_read_lock_nesting) == 0 &&
+            unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
+                rcu_read_unlock_special(t);
+}
+EXPORT_SYMBOL_GPL(__rcu_read_unlock);
+
+#ifdef CONFIG_RCU_CPU_STALL_DETECTOR
+
+/*
+ * Scan the current list of tasks blocked within RCU read-side critical
+ * sections, printing out the tid of each.
+ */
+static void rcu_print_task_stall(struct rcu_node *rnp)
+{
+        unsigned long flags;
+        struct list_head *lp;
+        int phase = rnp->gpnum & 0x1;
+        struct task_struct *t;
+
+        if (!list_empty(&rnp->blocked_tasks[phase])) {
+                spin_lock_irqsave(&rnp->lock, flags);
+                phase = rnp->gpnum & 0x1; /* re-read under lock. */
+                lp = &rnp->blocked_tasks[phase];
+                list_for_each_entry(t, lp, rcu_node_entry)
+                        printk(" P%d", t->pid);
+                spin_unlock_irqrestore(&rnp->lock, flags);
+        }
+}
+
+#endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
+
+/*
+ * Check for preempted RCU readers for the specified rcu_node structure.
+ * If the caller needs a reliable answer, it must hold the rcu_node's
+ * >lock.
+ */
+static int rcu_preempted_readers(struct rcu_node *rnp)
+{
+        return !list_empty(&rnp->blocked_tasks[rnp->gpnum & 0x1]);
+}
+
+#ifdef CONFIG_HOTPLUG_CPU
+
+/*
+ * Handle tasklist migration for case in which all CPUs covered by the
+ * specified rcu_node have gone offline.  Move them up to the root
+ * rcu_node.  The reason for not just moving them to the immediate
+ * parent is to remove the need for rcu_read_unlock_special() to
+ * make more than two attempts to acquire the target rcu_node's lock.
+ *
+ * The caller must hold rnp->lock with irqs disabled.
+ */
+static void rcu_preempt_offline_tasks(struct rcu_state *rsp,
+                                      struct rcu_node *rnp)
+{
+        int i;
+        struct list_head *lp;
+        struct list_head *lp_root;
+        struct rcu_node *rnp_root = rcu_get_root(rsp);
+        struct task_struct *tp;
+
+        if (rnp == rnp_root) {
+                WARN_ONCE(1, "Last CPU thought to be offlined?");
+                return;  /* Shouldn't happen: at least one CPU online. */
+        }
+
+        /*
+         * Move tasks up to root rcu_node.  Rely on the fact that the
+         * root rcu_node can be at most one ahead of the rest of the
+         * rcu_nodes in terms of gp_num value.  This fact allows us to
+         * move the blocked_tasks[] array directly, element by element.
+         */
+        for (i = 0; i < 2; i++) {
+                lp = &rnp->blocked_tasks[i];
+                lp_root = &rnp_root->blocked_tasks[i];
+                while (!list_empty(lp)) {
+                        tp = list_entry(lp->next, typeof(*tp), rcu_node_entry);
+                        spin_lock(&rnp_root->lock); /* irqs already disabled */
+                        list_del(&tp->rcu_node_entry);
+                        tp->rcu_blocked_node = rnp_root;
+                        list_add(&tp->rcu_node_entry, lp_root);
+                        spin_unlock(&rnp_root->lock); /* irqs remain disabled */
+                }
+        }
+}
+
+/*
+ * Do CPU-offline processing for preemptable RCU.
+ */
+static void rcu_preempt_offline_cpu(int cpu)
+{
+        __rcu_offline_cpu(cpu, &rcu_preempt_state);
+}
+
+#endif /* #ifdef CONFIG_HOTPLUG_CPU */
+
+/*
+ * Check for a quiescent state from the current CPU.  When a task blocks,
+ * the task is recorded in the corresponding CPU's rcu_node structure,
+ * which is checked elsewhere.
+ *
+ * Caller must disable hard irqs.
+ */
+static void rcu_preempt_check_callbacks(int cpu)
+{
+        struct task_struct *t = current;
+
+        if (t->rcu_read_lock_nesting == 0) {
+                t->rcu_read_unlock_special &=
+                        ~(RCU_READ_UNLOCK_NEED_QS | RCU_READ_UNLOCK_GOT_QS);
+                rcu_preempt_qs_record(cpu);
+                return;
+        }
+        if (per_cpu(rcu_preempt_data, cpu).qs_pending) {
+                if (t->rcu_read_unlock_special & RCU_READ_UNLOCK_GOT_QS) {
+                        rcu_preempt_qs_record(cpu);
+                        t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_GOT_QS;
+                } else if (!(t->rcu_read_unlock_special &
+                             RCU_READ_UNLOCK_NEED_QS)) {
+                        t->rcu_read_unlock_special |= RCU_READ_UNLOCK_NEED_QS;
+                }
+        }
+}
+
+/*
+ * Process callbacks for preemptable RCU.
+ */
+static void rcu_preempt_process_callbacks(void)
+{
+        __rcu_process_callbacks(&rcu_preempt_state,
+                                &__get_cpu_var(rcu_preempt_data));
+}
+
+/*
+ * Queue a preemptable-RCU callback for invocation after a grace period.
+ */
+void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
+{
+        __call_rcu(head, func, &rcu_preempt_state);
+}
+EXPORT_SYMBOL_GPL(call_rcu);
+
+/*
+ * Check to see if there is any immediate preemptable-RCU-related work
+ * to be done.
+ */
+static int rcu_preempt_pending(int cpu)
+{
+        return __rcu_pending(&rcu_preempt_state,
+                             &per_cpu(rcu_preempt_data, cpu));
+}
+
+/*
+ * Does preemptable RCU need the CPU to stay out of dynticks mode?
+ */
+static int rcu_preempt_needs_cpu(int cpu)
+{
+        return !!per_cpu(rcu_preempt_data, cpu).nxtlist;
+}
+
+/*
+ * Initialize preemptable RCU's per-CPU data.
+ */
+static void __cpuinit rcu_preempt_init_percpu_data(int cpu)
+{
+        rcu_init_percpu_data(cpu, &rcu_preempt_state, 1);
+}
+
+/*
+ * Check for a task exiting while in a preemptable-RCU read-side
+ * critical section, clean up if so.  No need to issue warnings,
+ * as debug_check_no_locks_held() already does this if lockdep
+ * is enabled.
+ */
+void exit_rcu(void)
+{
+        struct task_struct *t = current;
+
+        if (t->rcu_read_lock_nesting == 0)
+                return;
+        t->rcu_read_lock_nesting = 1;
+        rcu_read_unlock();
+}
+
+#else /* #ifdef CONFIG_TREE_PREEMPT_RCU */
+
+/*
+ * Tell them what RCU they are running.
+ */
+static inline void rcu_bootup_announce(void)
+{
+        printk(KERN_INFO "Hierarchical RCU implementation.\n");
+}
+
+/*
+ * Return the number of RCU batches processed thus far for debug & stats.
+ */
+long rcu_batches_completed(void)
+{
+        return rcu_batches_completed_sched();
+}
+EXPORT_SYMBOL_GPL(rcu_batches_completed);
+
+/*
+ * Because preemptable RCU does not exist, we never have to check for
+ * CPUs being in quiescent states.
+ */
+static void rcu_preempt_qs(int cpu)
+{
+}
+
+#ifdef CONFIG_RCU_CPU_STALL_DETECTOR
+
+/*
+ * Because preemptable RCU does not exist, we never have to check for
+ * tasks blocked within RCU read-side critical sections.
+ */
+static void rcu_print_task_stall(struct rcu_node *rnp)
+{
+}
+
+#endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
+
+/*
+ * Because preemptable RCU does not exist, there are never any preempted
+ * RCU readers.
+ */
+static int rcu_preempted_readers(struct rcu_node *rnp)
+{
+        return 0;
+}
+
+#ifdef CONFIG_HOTPLUG_CPU
+
+/*
+ * Because preemptable RCU does not exist, it never needs to migrate
+ * tasks that were blocked within RCU read-side critical sections.
+ */
+static void rcu_preempt_offline_tasks(struct rcu_state *rsp,
+                                      struct rcu_node *rnp)
+{
+}
+
+/*
+ * Because preemptable RCU does not exist, it never needs CPU-offline
+ * processing.
+ */
+static void rcu_preempt_offline_cpu(int cpu)
+{
+}
+
+#endif /* #ifdef CONFIG_HOTPLUG_CPU */
+
+/*
+ * Because preemptable RCU does not exist, it never has any callbacks
+ * to check.
+ */
+void rcu_preempt_check_callbacks(int cpu)
+{
+}
+
+/*
+ * Because preemptable RCU does not exist, it never has any callbacks
+ * to process.
+ */
+void rcu_preempt_process_callbacks(void)
+{
+}
+
+/*
+ * In classic RCU, call_rcu() is just call_rcu_sched().
+ */
+void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
+{
+        call_rcu_sched(head, func);
+}
+EXPORT_SYMBOL_GPL(call_rcu);
+
+/*
+ * Because preemptable RCU does not exist, it never has any work to do.
+ */
+static int rcu_preempt_pending(int cpu)
+{
+        return 0;
+}
+
+/*
+ * Because preemptable RCU does not exist, it never needs any CPU.
+ */
+static int rcu_preempt_needs_cpu(int cpu)
+{
+        return 0;
+}
+
+/*
+ * Because preemptable RCU does not exist, there is no per-CPU
+ * data to initialize.
+ */
+static void __cpuinit rcu_preempt_init_percpu_data(int cpu)
+{
+}
+
+#endif /* #else #ifdef CONFIG_TREE_PREEMPT_RCU */
diff --git a/kernel/rcutree_trace.c b/kernel/rcutree_trace.c
index fe1dcdbf1ca3..0ea1bff69727 100644
--- a/kernel/rcutree_trace.c
+++ b/kernel/rcutree_trace.c
@@ -43,6 +43,7 @@
 #include <linux/debugfs.h>
 #include <linux/seq_file.h>
 
+#define RCU_TREE_NONCORE
 #include "rcutree.h"
 
 static void print_one_rcu_data(struct seq_file *m, struct rcu_data *rdp)
@@ -76,8 +77,12 @@ static void print_one_rcu_data(struct seq_file *m, struct rcu_data *rdp)
 
 static int show_rcudata(struct seq_file *m, void *unused)
 {
-        seq_puts(m, "rcu:\n");
-        PRINT_RCU_DATA(rcu_data, print_one_rcu_data, m);
+#ifdef CONFIG_TREE_PREEMPT_RCU
+        seq_puts(m, "rcu_preempt:\n");
+        PRINT_RCU_DATA(rcu_preempt_data, print_one_rcu_data, m);
+#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
+        seq_puts(m, "rcu_sched:\n");
+        PRINT_RCU_DATA(rcu_sched_data, print_one_rcu_data, m);
         seq_puts(m, "rcu_bh:\n");
         PRINT_RCU_DATA(rcu_bh_data, print_one_rcu_data, m);
         return 0;
@@ -102,7 +107,7 @@ static void print_one_rcu_data_csv(struct seq_file *m, struct rcu_data *rdp)
                 return;
         seq_printf(m, "%d,%s,%ld,%ld,%d,%ld,%d",
                    rdp->cpu,
-                   cpu_is_offline(rdp->cpu) ? "\"Y\"" : "\"N\"",
+                   cpu_is_offline(rdp->cpu) ? "\"N\"" : "\"Y\"",
                    rdp->completed, rdp->gpnum,
                    rdp->passed_quiesc, rdp->passed_quiesc_completed,
                    rdp->qs_pending);
@@ -124,8 +129,12 @@ static int show_rcudata_csv(struct seq_file *m, void *unused)
         seq_puts(m, "\"dt\",\"dt nesting\",\"dn\",\"df\",");
 #endif /* #ifdef CONFIG_NO_HZ */
         seq_puts(m, "\"of\",\"ri\",\"ql\",\"b\"\n");
-        seq_puts(m, "\"rcu:\"\n");
-        PRINT_RCU_DATA(rcu_data, print_one_rcu_data_csv, m);
+#ifdef CONFIG_TREE_PREEMPT_RCU
+        seq_puts(m, "\"rcu_preempt:\"\n");
+        PRINT_RCU_DATA(rcu_preempt_data, print_one_rcu_data_csv, m);
+#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
+        seq_puts(m, "\"rcu_sched:\"\n");
+        PRINT_RCU_DATA(rcu_sched_data, print_one_rcu_data_csv, m);
         seq_puts(m, "\"rcu_bh:\"\n");
         PRINT_RCU_DATA(rcu_bh_data, print_one_rcu_data_csv, m);
         return 0;
@@ -171,8 +180,12 @@ static void print_one_rcu_state(struct seq_file *m, struct rcu_state *rsp)
 
 static int show_rcuhier(struct seq_file *m, void *unused)
 {
-        seq_puts(m, "rcu:\n");
-        print_one_rcu_state(m, &rcu_state);
+#ifdef CONFIG_TREE_PREEMPT_RCU
+        seq_puts(m, "rcu_preempt:\n");
+        print_one_rcu_state(m, &rcu_preempt_state);
+#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
+        seq_puts(m, "rcu_sched:\n");
+        print_one_rcu_state(m, &rcu_sched_state);
         seq_puts(m, "rcu_bh:\n");
         print_one_rcu_state(m, &rcu_bh_state);
         return 0;
@@ -193,8 +206,12 @@ static struct file_operations rcuhier_fops = {
 
 static int show_rcugp(struct seq_file *m, void *unused)
 {
-        seq_printf(m, "rcu: completed=%ld  gpnum=%ld\n",
-                   rcu_state.completed, rcu_state.gpnum);
+#ifdef CONFIG_TREE_PREEMPT_RCU
+        seq_printf(m, "rcu_preempt: completed=%ld  gpnum=%ld\n",
+                   rcu_preempt_state.completed, rcu_preempt_state.gpnum);
+#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
+        seq_printf(m, "rcu_sched: completed=%ld  gpnum=%ld\n",
+                   rcu_sched_state.completed, rcu_sched_state.gpnum);
         seq_printf(m, "rcu_bh: completed=%ld  gpnum=%ld\n",
                    rcu_bh_state.completed, rcu_bh_state.gpnum);
         return 0;
@@ -243,8 +260,12 @@ static void print_rcu_pendings(struct seq_file *m, struct rcu_state *rsp)
 
 static int show_rcu_pending(struct seq_file *m, void *unused)
 {
-        seq_puts(m, "rcu:\n");
-        print_rcu_pendings(m, &rcu_state);
+#ifdef CONFIG_TREE_PREEMPT_RCU
+        seq_puts(m, "rcu_preempt:\n");
+        print_rcu_pendings(m, &rcu_preempt_state);
+#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
+        seq_puts(m, "rcu_sched:\n");
+        print_rcu_pendings(m, &rcu_sched_state);
         seq_puts(m, "rcu_bh:\n");
         print_rcu_pendings(m, &rcu_bh_state);
         return 0;
@@ -264,62 +285,47 @@ static struct file_operations rcu_pending_fops = {
 };
 
 static struct dentry *rcudir;
-static struct dentry *datadir;
-static struct dentry *datadir_csv;
-static struct dentry *gpdir;
-static struct dentry *hierdir;
-static struct dentry *rcu_pendingdir;
 
 static int __init rcuclassic_trace_init(void)
 {
+        struct dentry *retval;
+
         rcudir = debugfs_create_dir("rcu", NULL);
         if (!rcudir)
-                goto out;
+                goto free_out;
 
-        datadir = debugfs_create_file("rcudata", 0444, rcudir,
+        retval = debugfs_create_file("rcudata", 0444, rcudir,
                                                 NULL, &rcudata_fops);
-        if (!datadir)
+        if (!retval)
                 goto free_out;
 
-        datadir_csv = debugfs_create_file("rcudata.csv", 0444, rcudir,
+        retval = debugfs_create_file("rcudata.csv", 0444, rcudir,
                                                 NULL, &rcudata_csv_fops);
-        if (!datadir_csv)
+        if (!retval)
                 goto free_out;
 
-        gpdir = debugfs_create_file("rcugp", 0444, rcudir, NULL, &rcugp_fops);
-        if (!gpdir)
+        retval = debugfs_create_file("rcugp", 0444, rcudir, NULL, &rcugp_fops);
+        if (!retval)
                 goto free_out;
 
-        hierdir = debugfs_create_file("rcuhier", 0444, rcudir,
+        retval = debugfs_create_file("rcuhier", 0444, rcudir,
                                                 NULL, &rcuhier_fops);
-        if (!hierdir)
+        if (!retval)
                 goto free_out;
 
-        rcu_pendingdir = debugfs_create_file("rcu_pending", 0444, rcudir,
+        retval = debugfs_create_file("rcu_pending", 0444, rcudir,
                                                 NULL, &rcu_pending_fops);
-        if (!rcu_pendingdir)
+        if (!retval)
                 goto free_out;
         return 0;
 free_out:
-        if (datadir)
-                debugfs_remove(datadir);
-        if (datadir_csv)
-                debugfs_remove(datadir_csv);
-        if (gpdir)
-                debugfs_remove(gpdir);
-        debugfs_remove(rcudir);
-out:
+        debugfs_remove_recursive(rcudir);
         return 1;
 }
 
 static void __exit rcuclassic_trace_cleanup(void)
 {
-        debugfs_remove(datadir);
-        debugfs_remove(datadir_csv);
-        debugfs_remove(gpdir);
-        debugfs_remove(hierdir);
-        debugfs_remove(rcu_pendingdir);
-        debugfs_remove(rcudir);
+        debugfs_remove_recursive(rcudir);
 }
 
 
diff --git a/kernel/res_counter.c b/kernel/res_counter.c
index bf8e7534c803..e1338f074314 100644
--- a/kernel/res_counter.c
+++ b/kernel/res_counter.c
@@ -18,7 +18,7 @@
 void res_counter_init(struct res_counter *counter, struct res_counter *parent)
 {
         spin_lock_init(&counter->lock);
-        counter->limit = (unsigned long long)LLONG_MAX;
+        counter->limit = RESOURCE_MAX;
         counter->parent = parent;
 }
 
@@ -133,6 +133,16 @@ int res_counter_memparse_write_strategy(const char *buf,
                                         unsigned long long *res)
 {
         char *end;
+
+        /* return RESOURCE_MAX(unlimited) if "-1" is specified */
+        if (*buf == '-') {
+                *res = simple_strtoull(buf + 1, &end, 10);
+                if (*res != 1 || *end != '\0')
+                        return -EINVAL;
+                *res = RESOURCE_MAX;
+                return 0;
+        }
+
         /* FIXME - make memparse() take const char* args */
         *res = memparse((char *)buf, &end);
         if (*end != '\0')
diff --git a/kernel/resource.c b/kernel/resource.c
index ac5f3a36923f..78b087221c15 100644
--- a/kernel/resource.c
+++ b/kernel/resource.c
@@ -787,7 +787,7 @@ static int __init reserve_setup(char *str)
         static struct resource reserve[MAXRESERVE];
 
         for (;;) {
-                int io_start, io_num;
+                unsigned int io_start, io_num;
                 int x = reserved;
 
                 if (get_option (&str, &io_start) != 2)
diff --git a/kernel/rtmutex.c b/kernel/rtmutex.c
index fcd107a78c5a..29bd4baf9e75 100644
--- a/kernel/rtmutex.c
+++ b/kernel/rtmutex.c
@@ -1039,16 +1039,14 @@ int rt_mutex_start_proxy_lock(struct rt_mutex *lock,
         if (!rt_mutex_owner(lock) || try_to_steal_lock(lock, task)) {
                 /* We got the lock for task. */
                 debug_rt_mutex_lock(lock);
-
                 rt_mutex_set_owner(lock, task, 0);
-
+                spin_unlock(&lock->wait_lock);
                 rt_mutex_deadlock_account_lock(lock, task);
                 return 1;
         }
 
         ret = task_blocks_on_rt_mutex(lock, waiter, task, detect_deadlock);
 
-
         if (ret && !waiter->task) {
                 /*
                  * Reset the return value. We might have
diff --git a/kernel/sched.c b/kernel/sched.c
index 8fb88a906aaa..faf4d463bbff 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -64,7 +64,6 @@
 #include <linux/tsacct_kern.h>
 #include <linux/kprobes.h>
 #include <linux/delayacct.h>
-#include <linux/reciprocal_div.h>
 #include <linux/unistd.h>
 #include <linux/pagemap.h>
 #include <linux/hrtimer.h>
@@ -120,30 +119,6 @@
  */
 #define RUNTIME_INF     ((u64)~0ULL)
 
-#ifdef CONFIG_SMP
-
-static void double_rq_lock(struct rq *rq1, struct rq *rq2);
-
-/*
- * Divide a load by a sched group cpu_power : (load / sg->__cpu_power)
- * Since cpu_power is a 'constant', we can use a reciprocal divide.
- */
-static inline u32 sg_div_cpu_power(const struct sched_group *sg, u32 load)
-{
-        return reciprocal_divide(load, sg->reciprocal_cpu_power);
-}
-
-/*
- * Each time a sched group cpu_power is changed,
- * we must compute its reciprocal value
- */
-static inline void sg_inc_cpu_power(struct sched_group *sg, u32 val)
-{
-        sg->__cpu_power += val;
-        sg->reciprocal_cpu_power = reciprocal_value(sg->__cpu_power);
-}
-#endif
-
 static inline int rt_policy(int policy)
 {
         if (unlikely(policy == SCHED_FIFO || policy == SCHED_RR))
@@ -309,8 +284,8 @@ void set_tg_uid(struct user_struct *user)
 
 /*
  * Root task group.
- *      Every UID task group (including init_task_group aka UID-0) will
- *      be a child to this group.
+ *      Every UID task group (including init_task_group aka UID-0) will
+ *      be a child to this group.
  */
 struct task_group root_task_group;
 
@@ -318,12 +293,12 @@ struct task_group root_task_group;
 /* Default task group's sched entity on each cpu */
 static DEFINE_PER_CPU(struct sched_entity, init_sched_entity);
 /* Default task group's cfs_rq on each cpu */
-static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp;
+static DEFINE_PER_CPU_SHARED_ALIGNED(struct cfs_rq, init_tg_cfs_rq);
 #endif /* CONFIG_FAIR_GROUP_SCHED */
 
 #ifdef CONFIG_RT_GROUP_SCHED
 static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity);
-static DEFINE_PER_CPU(struct rt_rq, init_rt_rq) ____cacheline_aligned_in_smp;
+static DEFINE_PER_CPU_SHARED_ALIGNED(struct rt_rq, init_rt_rq);
 #endif /* CONFIG_RT_GROUP_SCHED */
 #else /* !CONFIG_USER_SCHED */
 #define root_task_group init_task_group
@@ -401,13 +376,6 @@ static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
 
 #else
 
-#ifdef CONFIG_SMP
-static int root_task_group_empty(void)
-{
-        return 1;
-}
-#endif
-
 static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { }
 static inline struct task_group *task_group(struct task_struct *p)
 {
@@ -493,6 +461,7 @@ struct rt_rq {
 #endif
 #ifdef CONFIG_SMP
         unsigned long rt_nr_migratory;
+        unsigned long rt_nr_total;
         int overloaded;
         struct plist_head pushable_tasks;
 #endif
@@ -536,14 +505,6 @@ struct root_domain {
 #ifdef CONFIG_SMP
         struct cpupri cpupri;
 #endif
-#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
-        /*
-         * Preferred wake up cpu nominated by sched_mc balance that will be
-         * used when most cpus are idle in the system indicating overall very
-         * low system utilisation. Triggered at POWERSAVINGS_BALANCE_WAKEUP(2)
-         */
-        unsigned int sched_mc_preferred_wakeup_cpu;
-#endif
 };
 
 /*
@@ -615,6 +576,7 @@ struct rq {
 
         unsigned char idle_at_tick;
         /* For active balancing */
+        int post_schedule;
         int active_balance;
         int push_cpu;
         /* cpu of this runqueue: */
@@ -625,6 +587,9 @@ struct rq {
 
         struct task_struct *migration_thread;
         struct list_head migration_queue;
+
+        u64 rt_avg;
+        u64 age_stamp;
 #endif
 
         /* calc_load related fields */
@@ -664,9 +629,10 @@ struct rq {
 
 static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
 
-static inline void check_preempt_curr(struct rq *rq, struct task_struct *p, int sync)
+static inline
+void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
 {
-        rq->curr->sched_class->check_preempt_curr(rq, p, sync);
+        rq->curr->sched_class->check_preempt_curr(rq, p, flags);
 }
 
 static inline int cpu_of(struct rq *rq)
@@ -692,6 +658,7 @@ static inline int cpu_of(struct rq *rq)
 #define this_rq()               (&__get_cpu_var(runqueues))
 #define task_rq(p)              cpu_rq(task_cpu(p))
 #define cpu_curr(cpu)           (cpu_rq(cpu)->curr)
+#define raw_rq()                (&__raw_get_cpu_var(runqueues))
 
 inline void update_rq_clock(struct rq *rq)
 {
@@ -860,6 +827,14 @@ unsigned int sysctl_sched_shares_ratelimit = 250000;
 unsigned int sysctl_sched_shares_thresh = 4;
 
 /*
+ * period over which we average the RT time consumption, measured
+ * in ms.
+ *
+ * default: 1s
+ */
+const_debug unsigned int sysctl_sched_time_avg = MSEC_PER_SEC;
+
+/*
  * period over which we measure -rt task cpu usage in us.
  * default: 1s
  */
@@ -1277,12 +1252,37 @@ void wake_up_idle_cpu(int cpu)
 }
 #endif /* CONFIG_NO_HZ */
 
+static u64 sched_avg_period(void)
+{
+        return (u64)sysctl_sched_time_avg * NSEC_PER_MSEC / 2;
+}
+
+static void sched_avg_update(struct rq *rq)
+{
+        s64 period = sched_avg_period();
+
+        while ((s64)(rq->clock - rq->age_stamp) > period) {
+                rq->age_stamp += period;
+                rq->rt_avg /= 2;
+        }
+}
+
+static void sched_rt_avg_update(struct rq *rq, u64 rt_delta)
+{
+        rq->rt_avg += rt_delta;
+        sched_avg_update(rq);
+}
+
 #else /* !CONFIG_SMP */
 static void resched_task(struct task_struct *p)
 {
         assert_spin_locked(&task_rq(p)->lock);
         set_tsk_need_resched(p);
 }
+
+static void sched_rt_avg_update(struct rq *rq, u64 rt_delta)
+{
+}
 #endif /* CONFIG_SMP */
 
 #if BITS_PER_LONG == 32
@@ -1493,8 +1493,65 @@ static int tg_nop(struct task_group *tg, void *data)
 #endif
 
 #ifdef CONFIG_SMP
-static unsigned long source_load(int cpu, int type);
-static unsigned long target_load(int cpu, int type);
+/* Used instead of source_load when we know the type == 0 */
+static unsigned long weighted_cpuload(const int cpu)
+{
+        return cpu_rq(cpu)->load.weight;
+}
+
+/*
+ * Return a low guess at the load of a migration-source cpu weighted
+ * according to the scheduling class and "nice" value.
+ *
+ * We want to under-estimate the load of migration sources, to
+ * balance conservatively.
+ */
+static unsigned long source_load(int cpu, int type)
+{
+        struct rq *rq = cpu_rq(cpu);
+        unsigned long total = weighted_cpuload(cpu);
+
+        if (type == 0 || !sched_feat(LB_BIAS))
+                return total;
+
+        return min(rq->cpu_load[type-1], total);
+}
+
+/*
+ * Return a high guess at the load of a migration-target cpu weighted
+ * according to the scheduling class and "nice" value.
+ */
+static unsigned long target_load(int cpu, int type)
+{
+        struct rq *rq = cpu_rq(cpu);
+        unsigned long total = weighted_cpuload(cpu);
+
+        if (type == 0 || !sched_feat(LB_BIAS))
+                return total;
+
+        return max(rq->cpu_load[type-1], total);
+}
+
+static struct sched_group *group_of(int cpu)
+{
+        struct sched_domain *sd = rcu_dereference(cpu_rq(cpu)->sd);
+
+        if (!sd)
+                return NULL;
+
+        return sd->groups;
+}
+
+static unsigned long power_of(int cpu)
+{
+        struct sched_group *group = group_of(cpu);
+
+        if (!group)
+                return SCHED_LOAD_SCALE;
+
+        return group->cpu_power;
+}
+
 static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd);
 
 static unsigned long cpu_avg_load_per_task(int cpu)
@@ -1512,28 +1569,35 @@ static unsigned long cpu_avg_load_per_task(int cpu)
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
 
+struct update_shares_data {
+        unsigned long rq_weight[NR_CPUS];
+};
+
+static DEFINE_PER_CPU(struct update_shares_data, update_shares_data);
+
 static void __set_se_shares(struct sched_entity *se, unsigned long shares);
 
 /*
  * Calculate and set the cpu's group shares.
  */
-static void
-update_group_shares_cpu(struct task_group *tg, int cpu,
-                        unsigned long sd_shares, unsigned long sd_rq_weight)
+static void update_group_shares_cpu(struct task_group *tg, int cpu,
+                                    unsigned long sd_shares,
+                                    unsigned long sd_rq_weight,
+                                    struct update_shares_data *usd)
 {
-        unsigned long shares;
-        unsigned long rq_weight;
-
-        if (!tg->se[cpu])
-                return;
+        unsigned long shares, rq_weight;
+        int boost = 0;
 
-        rq_weight = tg->cfs_rq[cpu]->rq_weight;
+        rq_weight = usd->rq_weight[cpu];
+        if (!rq_weight) {
+                boost = 1;
+                rq_weight = NICE_0_LOAD;
+        }
 
         /*
-         *           \Sum shares * rq_weight
-         * shares =  -----------------------
-         *               \Sum rq_weight
-         *
+         *             \Sum_j shares_j * rq_weight_i
+         * shares_i =  -----------------------------
+         *                  \Sum_j rq_weight_j
          */
         shares = (sd_shares * rq_weight) / sd_rq_weight;
         shares = clamp_t(unsigned long, shares, MIN_SHARES, MAX_SHARES);
@@ -1544,8 +1608,8 @@ update_group_shares_cpu(struct task_group *tg, int cpu,
                 unsigned long flags;
 
                 spin_lock_irqsave(&rq->lock, flags);
-                tg->cfs_rq[cpu]->shares = shares;
-
+                tg->cfs_rq[cpu]->rq_weight = boost ? 0 : rq_weight;
+                tg->cfs_rq[cpu]->shares = boost ? 0 : shares;
                 __set_se_shares(tg->se[cpu], shares);
                 spin_unlock_irqrestore(&rq->lock, flags);
         }
@@ -1558,22 +1622,30 @@ update_group_shares_cpu(struct task_group *tg, int cpu,
  */
 static int tg_shares_up(struct task_group *tg, void *data)
 {
-        unsigned long weight, rq_weight = 0;
-        unsigned long shares = 0;
+        unsigned long weight, rq_weight = 0, shares = 0;
+        struct update_shares_data *usd;
         struct sched_domain *sd = data;
+        unsigned long flags;
         int i;
 
+        if (!tg->se[0])
+                return 0;
+
+        local_irq_save(flags);
+        usd = &__get_cpu_var(update_shares_data);
+
         for_each_cpu(i, sched_domain_span(sd)) {
+                weight = tg->cfs_rq[i]->load.weight;
+                usd->rq_weight[i] = weight;
+
                 /*
                  * If there are currently no tasks on the cpu pretend there
                  * is one of average load so that when a new task gets to
                  * run here it will not get delayed by group starvation.
                  */
-                weight = tg->cfs_rq[i]->load.weight;
                 if (!weight)
                         weight = NICE_0_LOAD;
 
-                tg->cfs_rq[i]->rq_weight = weight;
                 rq_weight += weight;
                 shares += tg->cfs_rq[i]->shares;
         }
@@ -1585,7 +1657,9 @@ static int tg_shares_up(struct task_group *tg, void *data)
                 shares = tg->shares;
 
         for_each_cpu(i, sched_domain_span(sd))
-                update_group_shares_cpu(tg, i, shares, rq_weight);
+                update_group_shares_cpu(tg, i, shares, rq_weight, usd);
+
+        local_irq_restore(flags);
 
         return 0;
 }
@@ -1615,8 +1689,14 @@ static int tg_load_down(struct task_group *tg, void *data)
 
 static void update_shares(struct sched_domain *sd)
 {
-        u64 now = cpu_clock(raw_smp_processor_id());
-        s64 elapsed = now - sd->last_update;
+        s64 elapsed;
+        u64 now;
+
+        if (root_task_group_empty())
+                return;
+
+        now = cpu_clock(raw_smp_processor_id());
+        elapsed = now - sd->last_update;
 
         if (elapsed >= (s64)(u64)sysctl_sched_shares_ratelimit) {
                 sd->last_update = now;
@@ -1626,6 +1706,9 @@ static void update_shares(struct sched_domain *sd)
 
 static void update_shares_locked(struct rq *rq, struct sched_domain *sd)
 {
+        if (root_task_group_empty())
+                return;
+
         spin_unlock(&rq->lock);
         update_shares(sd);
         spin_lock(&rq->lock);
@@ -1633,6 +1716,9 @@ static void update_shares_locked(struct rq *rq, struct sched_domain *sd)
 
 static void update_h_load(long cpu)
 {
+        if (root_task_group_empty())
+                return;
+
         walk_tg_tree(tg_load_down, tg_nop, (void *)cpu);
 }
 
@@ -1650,6 +1736,8 @@ static inline void update_shares_locked(struct rq *rq, struct sched_domain *sd)
 
 #ifdef CONFIG_PREEMPT
 
+static void double_rq_lock(struct rq *rq1, struct rq *rq2);
+
 /*
  * fair double_lock_balance: Safely acquires both rq->locks in a fair
  * way at the expense of forcing extra atomic operations in all
@@ -1914,13 +2002,6 @@ static inline void check_class_changed(struct rq *rq, struct task_struct *p,
 }
 
 #ifdef CONFIG_SMP
-
-/* Used instead of source_load when we know the type == 0 */
-static unsigned long weighted_cpuload(const int cpu)
-{
-        return cpu_rq(cpu)->load.weight;
-}
-
 /*
  * Is this task likely cache-hot:
  */
@@ -1978,7 +2059,8 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
                 if (task_hot(p, old_rq->clock, NULL))
                         schedstat_inc(p, se.nr_forced2_migrations);
 #endif
-                perf_counter_task_migration(p, new_cpu);
+                perf_swcounter_event(PERF_COUNT_SW_CPU_MIGRATIONS,
+                                     1, 1, NULL, 0);
         }
         p->se.vruntime -= old_cfsrq->min_vruntime -
                                          new_cfsrq->min_vruntime;
@@ -2193,186 +2275,6 @@ void kick_process(struct task_struct *p)
         preempt_enable();
 }
 EXPORT_SYMBOL_GPL(kick_process);
-
-/*
- * Return a low guess at the load of a migration-source cpu weighted
- * according to the scheduling class and "nice" value.
- *
- * We want to under-estimate the load of migration sources, to
- * balance conservatively.
- */
-static unsigned long source_load(int cpu, int type)
-{
-        struct rq *rq = cpu_rq(cpu);
-        unsigned long total = weighted_cpuload(cpu);
-
-        if (type == 0 || !sched_feat(LB_BIAS))
-                return total;
-
-        return min(rq->cpu_load[type-1], total);
-}
-
-/*
- * Return a high guess at the load of a migration-target cpu weighted
- * according to the scheduling class and "nice" value.
- */
-static unsigned long target_load(int cpu, int type)
-{
-        struct rq *rq = cpu_rq(cpu);
-        unsigned long total = weighted_cpuload(cpu);
-
-        if (type == 0 || !sched_feat(LB_BIAS))
-                return total;
-
-        return max(rq->cpu_load[type-1], total);
-}
-
-/*
- * find_idlest_group finds and returns the least busy CPU group within the
- * domain.
- */
-static struct sched_group *
-find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
-{
-        struct sched_group *idlest = NULL, *this = NULL, *group = sd->groups;
-        unsigned long min_load = ULONG_MAX, this_load = 0;
-        int load_idx = sd->forkexec_idx;
-        int imbalance = 100 + (sd->imbalance_pct-100)/2;
-
-        do {
-                unsigned long load, avg_load;
-                int local_group;
-                int i;
-
-                /* Skip over this group if it has no CPUs allowed */
-                if (!cpumask_intersects(sched_group_cpus(group),
-                                        &p->cpus_allowed))
-                        continue;
-
-                local_group = cpumask_test_cpu(this_cpu,
-                                               sched_group_cpus(group));
-
-                /* Tally up the load of all CPUs in the group */
-                avg_load = 0;
-
-                for_each_cpu(i, sched_group_cpus(group)) {
-                        /* Bias balancing toward cpus of our domain */
-                        if (local_group)
-                                load = source_load(i, load_idx);
-                        else
-                                load = target_load(i, load_idx);
-
-                        avg_load += load;
-                }
-
-                /* Adjust by relative CPU power of the group */
-                avg_load = sg_div_cpu_power(group,
-                                avg_load * SCHED_LOAD_SCALE);
-
-                if (local_group) {
-                        this_load = avg_load;
-                        this = group;
-                } else if (avg_load < min_load) {
-                        min_load = avg_load;
-                        idlest = group;
-                }
-        } while (group = group->next, group != sd->groups);
-
-        if (!idlest || 100*this_load < imbalance*min_load)
-                return NULL;
-        return idlest;
-}
-
-/*
- * find_idlest_cpu - find the idlest cpu among the cpus in group.
- */
-static int
-find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
-{
-        unsigned long load, min_load = ULONG_MAX;
-        int idlest = -1;
-        int i;
-
-        /* Traverse only the allowed CPUs */
-        for_each_cpu_and(i, sched_group_cpus(group), &p->cpus_allowed) {
-                load = weighted_cpuload(i);
-
-                if (load < min_load || (load == min_load && i == this_cpu)) {
-                        min_load = load;
-                        idlest = i;
-                }
-        }
-
-        return idlest;
-}
-
-/*
- * sched_balance_self: balance the current task (running on cpu) in domains
- * that have the 'flag' flag set. In practice, this is SD_BALANCE_FORK and
- * SD_BALANCE_EXEC.
- *
- * Balance, ie. select the least loaded group.
- *
- * Returns the target CPU number, or the same CPU if no balancing is needed.
- *
- * preempt must be disabled.
- */
-static int sched_balance_self(int cpu, int flag)
-{
-        struct task_struct *t = current;
-        struct sched_domain *tmp, *sd = NULL;
-
-        for_each_domain(cpu, tmp) {
-                /*
-                 * If power savings logic is enabled for a domain, stop there.
-                 */
-                if (tmp->flags & SD_POWERSAVINGS_BALANCE)
-                        break;
-                if (tmp->flags & flag)
-                        sd = tmp;
-        }
-
-        if (sd)
-                update_shares(sd);
-
-        while (sd) {
-                struct sched_group *group;
-                int new_cpu, weight;
-
-                if (!(sd->flags & flag)) {
-                        sd = sd->child;
-                        continue;
-                }
-
-                group = find_idlest_group(sd, t, cpu);
-                if (!group) {
-                        sd = sd->child;
-                        continue;
-                }
-
-                new_cpu = find_idlest_cpu(group, t, cpu);
-                if (new_cpu == -1 || new_cpu == cpu) {
-                        /* Now try balancing at a lower domain level of cpu */
-                        sd = sd->child;
-                        continue;
-                }
-
-                /* Now try balancing at a lower domain level of new_cpu */
-                cpu = new_cpu;
-                weight = cpumask_weight(sched_domain_span(sd));
-                sd = NULL;
-                for_each_domain(cpu, tmp) {
-                        if (weight <= cpumask_weight(sched_domain_span(tmp)))
-                                break;
-                        if (tmp->flags & flag)
-                                sd = tmp;
-                }
-                /* while loop will break here if sd == NULL */
-        }
-
-        return cpu;
-}
-
 #endif /* CONFIG_SMP */
 
 /**
@@ -2410,37 +2312,22 @@ void task_oncpu_function_call(struct task_struct *p,
  *
  * returns failure only if the task is already active.
  */
-static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
+static int try_to_wake_up(struct task_struct *p, unsigned int state,
+                          int wake_flags)
 {
         int cpu, orig_cpu, this_cpu, success = 0;
         unsigned long flags;
-        long old_state;
         struct rq *rq;
 
         if (!sched_feat(SYNC_WAKEUPS))
-                sync = 0;
+                wake_flags &= ~WF_SYNC;
 
-#ifdef CONFIG_SMP
-        if (sched_feat(LB_WAKEUP_UPDATE) && !root_task_group_empty()) {
-                struct sched_domain *sd;
-
-                this_cpu = raw_smp_processor_id();
-                cpu = task_cpu(p);
-
-                for_each_domain(this_cpu, sd) {
-                        if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
-                                update_shares(sd);
-                                break;
-                        }
-                }
-        }
-#endif
+        this_cpu = get_cpu();
 
         smp_wmb();
         rq = task_rq_lock(p, &flags);
         update_rq_clock(rq);
-        old_state = p->state;
-        if (!(old_state & state))
+        if (!(p->state & state))
                 goto out;
 
         if (p->se.on_rq)
@@ -2448,27 +2335,29 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
 
         cpu = task_cpu(p);
         orig_cpu = cpu;
-        this_cpu = smp_processor_id();
 
 #ifdef CONFIG_SMP
         if (unlikely(task_running(rq, p)))
                 goto out_activate;
 
-        cpu = p->sched_class->select_task_rq(p, sync);
-        if (cpu != orig_cpu) {
+        /*
+         * In order to handle concurrent wakeups and release the rq->lock
+         * we put the task in TASK_WAKING state.
+         *
+         * First fix up the nr_uninterruptible count:
+         */
+        if (task_contributes_to_load(p))
+                rq->nr_uninterruptible--;
+        p->state = TASK_WAKING;
+        task_rq_unlock(rq, &flags);
+
+        cpu = p->sched_class->select_task_rq(p, SD_BALANCE_WAKE, wake_flags);
+        if (cpu != orig_cpu)
                 set_task_cpu(p, cpu);
-                task_rq_unlock(rq, &flags);
-                /* might preempt at this point */
-                rq = task_rq_lock(p, &flags);
-                old_state = p->state;
-                if (!(old_state & state))
-                        goto out;
-                if (p->se.on_rq)
-                        goto out_running;
 
-                this_cpu = smp_processor_id();
-                cpu = task_cpu(p);
-        }
+        rq = task_rq_lock(p, &flags);
+        WARN_ON(p->state != TASK_WAKING);
+        cpu = task_cpu(p);
 
 #ifdef CONFIG_SCHEDSTATS
         schedstat_inc(rq, ttwu_count);
@@ -2488,7 +2377,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
 out_activate:
 #endif /* CONFIG_SMP */
         schedstat_inc(p, se.nr_wakeups);
-        if (sync)
+        if (wake_flags & WF_SYNC)
                 schedstat_inc(p, se.nr_wakeups_sync);
         if (orig_cpu != cpu)
                 schedstat_inc(p, se.nr_wakeups_migrate);
@@ -2517,7 +2406,7 @@ out_activate:
 
 out_running:
         trace_sched_wakeup(rq, p, success);
-        check_preempt_curr(rq, p, sync);
+        check_preempt_curr(rq, p, wake_flags);
 
         p->state = TASK_RUNNING;
 #ifdef CONFIG_SMP
@@ -2526,6 +2415,7 @@ out_running:
 #endif
 out:
         task_rq_unlock(rq, &flags);
+        put_cpu();
 
         return success;
 }
@@ -2568,17 +2458,40 @@ static void __sched_fork(struct task_struct *p)
         p->se.avg_overlap               = 0;
         p->se.start_runtime             = 0;
         p->se.avg_wakeup                = sysctl_sched_wakeup_granularity;
+        p->se.avg_running               = 0;
 
 #ifdef CONFIG_SCHEDSTATS
-        p->se.wait_start                = 0;
-        p->se.sum_sleep_runtime         = 0;
-        p->se.sleep_start               = 0;
-        p->se.block_start               = 0;
-        p->se.sleep_max                 = 0;
-        p->se.block_max                 = 0;
-        p->se.exec_max                  = 0;
-        p->se.slice_max                 = 0;
-        p->se.wait_max                  = 0;
+        p->se.wait_start                        = 0;
+        p->se.wait_max                          = 0;
+        p->se.wait_count                        = 0;
+        p->se.wait_sum                          = 0;
+
+        p->se.sleep_start                       = 0;
+        p->se.sleep_max                         = 0;
+        p->se.sum_sleep_runtime                 = 0;
+
+        p->se.block_start                       = 0;
+        p->se.block_max                         = 0;
+        p->se.exec_max                          = 0;
+        p->se.slice_max                         = 0;
+
+        p->se.nr_migrations_cold                = 0;
+        p->se.nr_failed_migrations_affine       = 0;
+        p->se.nr_failed_migrations_running      = 0;
+        p->se.nr_failed_migrations_hot          = 0;
+        p->se.nr_forced_migrations              = 0;
+        p->se.nr_forced2_migrations             = 0;
+
+        p->se.nr_wakeups                        = 0;
+        p->se.nr_wakeups_sync                   = 0;
+        p->se.nr_wakeups_migrate                = 0;
+        p->se.nr_wakeups_local                  = 0;
+        p->se.nr_wakeups_remote                 = 0;
+        p->se.nr_wakeups_affine                 = 0;
+        p->se.nr_wakeups_affine_attempts        = 0;
+        p->se.nr_wakeups_passive                = 0;
+        p->se.nr_wakeups_idle                   = 0;
+
 #endif
 
         INIT_LIST_HEAD(&p->rt.run_list);
@@ -2607,18 +2520,41 @@ void sched_fork(struct task_struct *p, int clone_flags)
 
         __sched_fork(p);
 
-#ifdef CONFIG_SMP
-        cpu = sched_balance_self(cpu, SD_BALANCE_FORK);
-#endif
-        set_task_cpu(p, cpu);
-
         /*
-         * Make sure we do not leak PI boosting priority to the child:
+         * Make sure we do not leak PI boosting priority to the child.
          */
         p->prio = current->normal_prio;
+
+        /*
+         * Revert to default priority/policy on fork if requested.
+         */
+        if (unlikely(p->sched_reset_on_fork)) {
+                if (p->policy == SCHED_FIFO || p->policy == SCHED_RR)
+                        p->policy = SCHED_NORMAL;
+
+                if (p->normal_prio < DEFAULT_PRIO)
+                        p->prio = DEFAULT_PRIO;
+
+                if (PRIO_TO_NICE(p->static_prio) < 0) {
+                        p->static_prio = NICE_TO_PRIO(0);
+                        set_load_weight(p);
+                }
+
+                /*
+                 * We don't need the reset flag anymore after the fork. It has
+                 * fulfilled its duty:
+                 */
+                p->sched_reset_on_fork = 0;
+        }
+
         if (!rt_prio(p->prio))
                 p->sched_class = &fair_sched_class;
 
+#ifdef CONFIG_SMP
+        cpu = p->sched_class->select_task_rq(p, SD_BALANCE_FORK, 0);
+#endif
+        set_task_cpu(p, cpu);
+
 #if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
         if (likely(sched_info_on()))
                 memset(&p->sched_info, 0, sizeof(p->sched_info));
@@ -2664,7 +2600,7 @@ void wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
                 inc_nr_running(rq);
         }
         trace_sched_wakeup_new(rq, p, 1);
-        check_preempt_curr(rq, p, 0);
+        check_preempt_curr(rq, p, WF_FORK);
 #ifdef CONFIG_SMP
         if (p->sched_class->task_wake_up)
                 p->sched_class->task_wake_up(rq, p);
@@ -2772,12 +2708,6 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev)
 {
         struct mm_struct *mm = rq->prev_mm;
         long prev_state;
-#ifdef CONFIG_SMP
-        int post_schedule = 0;
-
-        if (current->sched_class->needs_post_schedule)
-                post_schedule = current->sched_class->needs_post_schedule(rq);
-#endif
 
         rq->prev_mm = NULL;
 
@@ -2796,10 +2726,6 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev)
         finish_arch_switch(prev);
         perf_counter_task_sched_in(current, cpu_of(rq));
         finish_lock_switch(rq, prev);
-#ifdef CONFIG_SMP
-        if (post_schedule)
-                current->sched_class->post_schedule(rq);
-#endif
 
         fire_sched_in_preempt_notifiers(current);
         if (mm)
@@ -2814,6 +2740,42 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev)
         }
 }
 
+#ifdef CONFIG_SMP
+
+/* assumes rq->lock is held */
+static inline void pre_schedule(struct rq *rq, struct task_struct *prev)
+{
+        if (prev->sched_class->pre_schedule)
+                prev->sched_class->pre_schedule(rq, prev);
+}
+
+/* rq->lock is NOT held, but preemption is disabled */
+static inline void post_schedule(struct rq *rq)
+{
+        if (rq->post_schedule) {
+                unsigned long flags;
+
+                spin_lock_irqsave(&rq->lock, flags);
+                if (rq->curr->sched_class->post_schedule)
+                        rq->curr->sched_class->post_schedule(rq);
+                spin_unlock_irqrestore(&rq->lock, flags);
+
+                rq->post_schedule = 0;
+        }
+}
+
+#else
+
+static inline void pre_schedule(struct rq *rq, struct task_struct *p)
+{
+}
+
+static inline void post_schedule(struct rq *rq)
+{
+}
+
+#endif
+
 /**
  * schedule_tail - first thing a freshly forked thread must call.
  * @prev: the thread we just switched away from.
@@ -2824,6 +2786,13 @@ asmlinkage void schedule_tail(struct task_struct *prev)
         struct rq *rq = this_rq();
 
         finish_task_switch(rq, prev);
+
+        /*
+         * FIXME: do we need to worry about rq being invalidated by the
+         * task_switch?
+         */
+        post_schedule(rq);
+
 #ifdef __ARCH_WANT_UNLOCKED_CTXSW
         /* In this case, finish_task_switch does not reenable preemption */
         preempt_enable();
@@ -3140,7 +3109,7 @@ out:
 void sched_exec(void)
 {
         int new_cpu, this_cpu = get_cpu();
-        new_cpu = sched_balance_self(this_cpu, SD_BALANCE_EXEC);
+        new_cpu = current->sched_class->select_task_rq(current, SD_BALANCE_EXEC, 0);
         put_cpu();
         if (new_cpu != this_cpu)
                 sched_migrate_task(current, new_cpu);
@@ -3355,9 +3324,10 @@ static int move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
 {
         const struct sched_class *class;
 
-        for (class = sched_class_highest; class; class = class->next)
+        for_each_class(class) {
                 if (class->move_one_task(this_rq, this_cpu, busiest, sd, idle))
                         return 1;
+        }
 
         return 0;
 }
@@ -3520,7 +3490,7 @@ static inline void update_sd_power_savings_stats(struct sched_group *group,
          * capacity but still has some space to pick up some load
          * from other group and save more power
          */
-        if (sgs->sum_nr_running > sgs->group_capacity - 1)
+        if (sgs->sum_nr_running + 1 > sgs->group_capacity)
                 return;
 
         if (sgs->sum_nr_running > sds->leader_nr_running ||
@@ -3559,11 +3529,6 @@ static inline int check_power_save_busiest_group(struct sd_lb_stats *sds,
         *imbalance = sds->min_load_per_task;
         sds->busiest = sds->group_min;
 
-        if (sched_mc_power_savings >= POWERSAVINGS_BALANCE_WAKEUP) {
-                cpu_rq(this_cpu)->rd->sched_mc_preferred_wakeup_cpu =
-                        group_first_cpu(sds->group_leader);
-        }
-
         return 1;
 
 }
@@ -3588,6 +3553,102 @@ static inline int check_power_save_busiest_group(struct sd_lb_stats *sds,
 #endif /* CONFIG_SCHED_MC || CONFIG_SCHED_SMT */
 
 
+unsigned long default_scale_freq_power(struct sched_domain *sd, int cpu)
+{
+        return SCHED_LOAD_SCALE;
+}
+
+unsigned long __weak arch_scale_freq_power(struct sched_domain *sd, int cpu)
+{
+        return default_scale_freq_power(sd, cpu);
+}
+
+unsigned long default_scale_smt_power(struct sched_domain *sd, int cpu)
+{
+        unsigned long weight = cpumask_weight(sched_domain_span(sd));
+        unsigned long smt_gain = sd->smt_gain;
+
+        smt_gain /= weight;
+
+        return smt_gain;
+}
+
+unsigned long __weak arch_scale_smt_power(struct sched_domain *sd, int cpu)
+{
+        return default_scale_smt_power(sd, cpu);
+}
+
+unsigned long scale_rt_power(int cpu)
+{
+        struct rq *rq = cpu_rq(cpu);
+        u64 total, available;
+
+        sched_avg_update(rq);
+
+        total = sched_avg_period() + (rq->clock - rq->age_stamp);
+        available = total - rq->rt_avg;
+
+        if (unlikely((s64)total < SCHED_LOAD_SCALE))
+                total = SCHED_LOAD_SCALE;
+
+        total >>= SCHED_LOAD_SHIFT;
+
+        return div_u64(available, total);
+}
+
+static void update_cpu_power(struct sched_domain *sd, int cpu)
+{
+        unsigned long weight = cpumask_weight(sched_domain_span(sd));
+        unsigned long power = SCHED_LOAD_SCALE;
+        struct sched_group *sdg = sd->groups;
+
+        if (sched_feat(ARCH_POWER))
+                power *= arch_scale_freq_power(sd, cpu);
+        else
+                power *= default_scale_freq_power(sd, cpu);
+
+        power >>= SCHED_LOAD_SHIFT;
+
+        if ((sd->flags & SD_SHARE_CPUPOWER) && weight > 1) {
+                if (sched_feat(ARCH_POWER))
+                        power *= arch_scale_smt_power(sd, cpu);
+                else
+                        power *= default_scale_smt_power(sd, cpu);
+
+                power >>= SCHED_LOAD_SHIFT;
+        }
+
+        power *= scale_rt_power(cpu);
+        power >>= SCHED_LOAD_SHIFT;
+
+        if (!power)
+                power = 1;
+
+        sdg->cpu_power = power;
+}
+
+static void update_group_power(struct sched_domain *sd, int cpu)
+{
+        struct sched_domain *child = sd->child;
+        struct sched_group *group, *sdg = sd->groups;
+        unsigned long power;
+
+        if (!child) {
+                update_cpu_power(sd, cpu);
+                return;
+        }
+
+        power = 0;
+
+        group = child->groups;
+        do {
+                power += group->cpu_power;
+                group = group->next;
+        } while (group != child->groups);
+
+        sdg->cpu_power = power;
+}
+
 /**
  * update_sg_lb_stats - Update sched_group's statistics for load balancing.
  * @group: sched_group whose statistics are to be updated.
@@ -3600,7 +3661,8 @@ static inline int check_power_save_busiest_group(struct sd_lb_stats *sds,
  * @balance: Should we balance.
  * @sgs: variable to hold the statistics for this group.
  */
-static inline void update_sg_lb_stats(struct sched_group *group, int this_cpu,
+static inline void update_sg_lb_stats(struct sched_domain *sd,
+                        struct sched_group *group, int this_cpu,
                         enum cpu_idle_type idle, int load_idx, int *sd_idle,
                         int local_group, const struct cpumask *cpus,
                         int *balance, struct sg_lb_stats *sgs)
@@ -3611,8 +3673,11 @@ static inline void update_sg_lb_stats(struct sched_group *group, int this_cpu,
         unsigned long sum_avg_load_per_task;
         unsigned long avg_load_per_task;
 
-        if (local_group)
+        if (local_group) {
                 balance_cpu = group_first_cpu(group);
+                if (balance_cpu == this_cpu)
+                        update_group_power(sd, this_cpu);
+        }
 
         /* Tally up the load of all CPUs in the group */
         sum_avg_load_per_task = avg_load_per_task = 0;
@@ -3661,8 +3726,7 @@ static inline void update_sg_lb_stats(struct sched_group *group, int this_cpu,
         }
 
         /* Adjust by relative CPU power of the group */
-        sgs->avg_load = sg_div_cpu_power(group,
-                        sgs->group_load * SCHED_LOAD_SCALE);
+        sgs->avg_load = (sgs->group_load * SCHED_LOAD_SCALE) / group->cpu_power;
 
 
         /*
@@ -3674,14 +3738,14 @@ static inline void update_sg_lb_stats(struct sched_group *group, int this_cpu,
          *      normalized nr_running number somewhere that negates
          *      the hierarchy?
          */
-        avg_load_per_task = sg_div_cpu_power(group,
-                        sum_avg_load_per_task * SCHED_LOAD_SCALE);
+        avg_load_per_task = (sum_avg_load_per_task * SCHED_LOAD_SCALE) /
+                group->cpu_power;
 
         if ((max_cpu_load - min_cpu_load) > 2*avg_load_per_task)
                 sgs->group_imb = 1;
 
-        sgs->group_capacity = group->__cpu_power / SCHED_LOAD_SCALE;
-
+        sgs->group_capacity =
+                DIV_ROUND_CLOSEST(group->cpu_power, SCHED_LOAD_SCALE);
 }
 
 /**
@@ -3699,9 +3763,13 @@ static inline void update_sd_lb_stats(struct sched_domain *sd, int this_cpu,
                         const struct cpumask *cpus, int *balance,
                         struct sd_lb_stats *sds)
 {
+        struct sched_domain *child = sd->child;
         struct sched_group *group = sd->groups;
         struct sg_lb_stats sgs;
-        int load_idx;
+        int load_idx, prefer_sibling = 0;
+
+        if (child && child->flags & SD_PREFER_SIBLING)
+                prefer_sibling = 1;
 
         init_sd_power_savings_stats(sd, sds, idle);
         load_idx = get_sd_load_idx(sd, idle);
@@ -3712,14 +3780,22 @@ static inline void update_sd_lb_stats(struct sched_domain *sd, int this_cpu,
                 local_group = cpumask_test_cpu(this_cpu,
                                                sched_group_cpus(group));
                 memset(&sgs, 0, sizeof(sgs));
-                update_sg_lb_stats(group, this_cpu, idle, load_idx, sd_idle,
+                update_sg_lb_stats(sd, group, this_cpu, idle, load_idx, sd_idle,
                                 local_group, cpus, balance, &sgs);
 
                 if (local_group && balance && !(*balance))
                         return;
 
                 sds->total_load += sgs.group_load;
-                sds->total_pwr += group->__cpu_power;
+                sds->total_pwr += group->cpu_power;
+
+                /*
+                 * In case the child domain prefers tasks go to siblings
+                 * first, lower the group capacity to one so that we'll try
+                 * and move all the excess tasks away.
+                 */
+                if (prefer_sibling)
+                        sgs.group_capacity = min(sgs.group_capacity, 1UL);
 
                 if (local_group) {
                         sds->this_load = sgs.avg_load;
@@ -3739,7 +3815,6 @@ static inline void update_sd_lb_stats(struct sched_domain *sd, int this_cpu,
                 update_sd_power_savings_stats(group, sds, local_group, &sgs);
                 group = group->next;
         } while (group != sd->groups);
-
 }
 
 /**
@@ -3777,28 +3852,28 @@ static inline void fix_small_imbalance(struct sd_lb_stats *sds,
          * moving them.
          */
 
-        pwr_now += sds->busiest->__cpu_power *
+        pwr_now += sds->busiest->cpu_power *
                         min(sds->busiest_load_per_task, sds->max_load);
-        pwr_now += sds->this->__cpu_power *
+        pwr_now += sds->this->cpu_power *
                         min(sds->this_load_per_task, sds->this_load);
         pwr_now /= SCHED_LOAD_SCALE;
 
         /* Amount of load we'd subtract */
-        tmp = sg_div_cpu_power(sds->busiest,
-                        sds->busiest_load_per_task * SCHED_LOAD_SCALE);
+        tmp = (sds->busiest_load_per_task * SCHED_LOAD_SCALE) /
+                sds->busiest->cpu_power;
         if (sds->max_load > tmp)
-                pwr_move += sds->busiest->__cpu_power *
+                pwr_move += sds->busiest->cpu_power *
                         min(sds->busiest_load_per_task, sds->max_load - tmp);
 
         /* Amount of load we'd add */
-        if (sds->max_load * sds->busiest->__cpu_power <
+        if (sds->max_load * sds->busiest->cpu_power <
                 sds->busiest_load_per_task * SCHED_LOAD_SCALE)
-                tmp = sg_div_cpu_power(sds->this,
-                        sds->max_load * sds->busiest->__cpu_power);
+                tmp = (sds->max_load * sds->busiest->cpu_power) /
+                        sds->this->cpu_power;
         else
-                tmp = sg_div_cpu_power(sds->this,
-                        sds->busiest_load_per_task * SCHED_LOAD_SCALE);
-        pwr_move += sds->this->__cpu_power *
+                tmp = (sds->busiest_load_per_task * SCHED_LOAD_SCALE) /
+                        sds->this->cpu_power;
+        pwr_move += sds->this->cpu_power *
                         min(sds->this_load_per_task, sds->this_load + tmp);
         pwr_move /= SCHED_LOAD_SCALE;
 
@@ -3833,8 +3908,8 @@ static inline void calculate_imbalance(struct sd_lb_stats *sds, int this_cpu,
                         sds->max_load - sds->busiest_load_per_task);
 
         /* How much load to actually move to equalise the imbalance */
-        *imbalance = min(max_pull * sds->busiest->__cpu_power,
-                (sds->avg_load - sds->this_load) * sds->this->__cpu_power)
+        *imbalance = min(max_pull * sds->busiest->cpu_power,
+                (sds->avg_load - sds->this_load) * sds->this->cpu_power)
                         / SCHED_LOAD_SCALE;
 
         /*
@@ -3964,15 +4039,18 @@ find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
         int i;
 
         for_each_cpu(i, sched_group_cpus(group)) {
+                unsigned long power = power_of(i);
+                unsigned long capacity = DIV_ROUND_CLOSEST(power, SCHED_LOAD_SCALE);
                 unsigned long wl;
 
                 if (!cpumask_test_cpu(i, cpus))
                         continue;
 
                 rq = cpu_rq(i);
-                wl = weighted_cpuload(i);
+                wl = weighted_cpuload(i) * SCHED_LOAD_SCALE;
+                wl /= power;
 
-                if (rq->nr_running == 1 && wl > imbalance)
+                if (capacity && rq->nr_running == 1 && wl > imbalance)
                         continue;
 
                 if (wl > max_load) {
@@ -5233,14 +5311,13 @@ static inline void schedule_debug(struct task_struct *prev)
 #endif
 }
 
-static void put_prev_task(struct rq *rq, struct task_struct *prev)
+static void put_prev_task(struct rq *rq, struct task_struct *p)
 {
-        if (prev->state == TASK_RUNNING) {
-                u64 runtime = prev->se.sum_exec_runtime;
+        u64 runtime = p->se.sum_exec_runtime - p->se.prev_sum_exec_runtime;
 
-                runtime -= prev->se.prev_sum_exec_runtime;
-                runtime = min_t(u64, runtime, 2*sysctl_sched_migration_cost);
+        update_avg(&p->se.avg_running, runtime);
 
+        if (p->state == TASK_RUNNING) {
                 /*
                  * In order to avoid avg_overlap growing stale when we are
                  * indeed overlapping and hence not getting put to sleep, grow
@@ -5250,9 +5327,12 @@ static void put_prev_task(struct rq *rq, struct task_struct *prev)
                  * correlates to the amount of cache footprint a task can
                  * build up.
                  */
-                update_avg(&prev->se.avg_overlap, runtime);
+                runtime = min_t(u64, runtime, 2*sysctl_sched_migration_cost);
+                update_avg(&p->se.avg_overlap, runtime);
+        } else {
+                update_avg(&p->se.avg_running, 0);
         }
-        prev->sched_class->put_prev_task(rq, prev);
+        p->sched_class->put_prev_task(rq, p);
 }
 
 /*
@@ -5301,7 +5381,7 @@ need_resched:
         preempt_disable();
         cpu = smp_processor_id();
         rq = cpu_rq(cpu);
-        rcu_qsctr_inc(cpu);
+        rcu_sched_qs(cpu);
         prev = rq->curr;
         switch_count = &prev->nivcsw;
 
@@ -5325,10 +5405,7 @@ need_resched_nonpreemptible:
                 switch_count = &prev->nvcsw;
         }
 
-#ifdef CONFIG_SMP
-        if (prev->sched_class->pre_schedule)
-                prev->sched_class->pre_schedule(rq, prev);
-#endif
+        pre_schedule(rq, prev);
 
         if (unlikely(!rq->nr_running))
                 idle_balance(cpu, rq);
@@ -5354,6 +5431,8 @@ need_resched_nonpreemptible:
         } else
                 spin_unlock_irq(&rq->lock);
 
+        post_schedule(rq);
+
         if (unlikely(reacquire_kernel_lock(current) < 0))
                 goto need_resched_nonpreemptible;
 
@@ -5485,10 +5564,10 @@ asmlinkage void __sched preempt_schedule_irq(void)
 
 #endif /* CONFIG_PREEMPT */
 
-int default_wake_function(wait_queue_t *curr, unsigned mode, int sync,
+int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags,
                           void *key)
 {
-        return try_to_wake_up(curr->private, mode, sync);
+        return try_to_wake_up(curr->private, mode, wake_flags);
 }
 EXPORT_SYMBOL(default_wake_function);
 
@@ -5502,14 +5581,14 @@ EXPORT_SYMBOL(default_wake_function);
  * zero in this (rare) case, and we handle it by continuing to scan the queue.
  */
 static void __wake_up_common(wait_queue_head_t *q, unsigned int mode,
-                        int nr_exclusive, int sync, void *key)
+                        int nr_exclusive, int wake_flags, void *key)
 {
         wait_queue_t *curr, *next;
 
         list_for_each_entry_safe(curr, next, &q->task_list, task_list) {
                 unsigned flags = curr->flags;
 
-                if (curr->func(curr, mode, sync, key) &&
+                if (curr->func(curr, mode, wake_flags, key) &&
                                 (flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive)
                         break;
         }
@@ -5570,16 +5649,16 @@ void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode,
                         int nr_exclusive, void *key)
 {
         unsigned long flags;
-        int sync = 1;
+        int wake_flags = WF_SYNC;
 
         if (unlikely(!q))
                 return;
 
         if (unlikely(!nr_exclusive))
-                sync = 0;
+                wake_flags = 0;
 
         spin_lock_irqsave(&q->lock, flags);
-        __wake_up_common(q, mode, nr_exclusive, sync, key);
+        __wake_up_common(q, mode, nr_exclusive, wake_flags, key);
         spin_unlock_irqrestore(&q->lock, flags);
 }
 EXPORT_SYMBOL_GPL(__wake_up_sync_key);
@@ -6099,17 +6178,25 @@ static int __sched_setscheduler(struct task_struct *p, int policy,
         unsigned long flags;
         const struct sched_class *prev_class = p->sched_class;
         struct rq *rq;
+        int reset_on_fork;
 
         /* may grab non-irq protected spin_locks */
         BUG_ON(in_interrupt());
 recheck:
         /* double check policy once rq lock held */
-        if (policy < 0)
+        if (policy < 0) {
+                reset_on_fork = p->sched_reset_on_fork;
                 policy = oldpolicy = p->policy;
-        else if (policy != SCHED_FIFO && policy != SCHED_RR &&
-                        policy != SCHED_NORMAL && policy != SCHED_BATCH &&
-                        policy != SCHED_IDLE)
-                return -EINVAL;
+        } else {
+                reset_on_fork = !!(policy & SCHED_RESET_ON_FORK);
+                policy &= ~SCHED_RESET_ON_FORK;
+
+                if (policy != SCHED_FIFO && policy != SCHED_RR &&
+                                policy != SCHED_NORMAL && policy != SCHED_BATCH &&
+                                policy != SCHED_IDLE)
+                        return -EINVAL;
+        }
+
         /*
          * Valid priorities for SCHED_FIFO and SCHED_RR are
          * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
@@ -6153,6 +6240,10 @@ recheck:
                 /* can't change other user's priorities */
                 if (!check_same_owner(p))
                         return -EPERM;
+
+                /* Normal users shall not reset the sched_reset_on_fork flag */
+                if (p->sched_reset_on_fork && !reset_on_fork)
+                        return -EPERM;
         }
 
         if (user) {
@@ -6196,6 +6287,8 @@ recheck:
         if (running)
                 p->sched_class->put_prev_task(rq, p);
 
+        p->sched_reset_on_fork = reset_on_fork;
+
         oldprio = p->prio;
         __setscheduler(rq, p, policy, param->sched_priority);
 
@@ -6312,14 +6405,15 @@ SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
         if (p) {
                 retval = security_task_getscheduler(p);
                 if (!retval)
-                        retval = p->policy;
+                        retval = p->policy
+                                | (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0);
         }
         read_unlock(&tasklist_lock);
         return retval;
 }
 
 /**
- * sys_sched_getscheduler - get the RT priority of a thread
+ * sys_sched_getparam - get the RT priority of a thread
  * @pid: the pid in question.
  * @param: structure containing the RT priority.
  */
@@ -6540,27 +6634,21 @@ SYSCALL_DEFINE0(sched_yield)
         return 0;
 }
 
+static inline int should_resched(void)
+{
+        return need_resched() && !(preempt_count() & PREEMPT_ACTIVE);
+}
+
 static void __cond_resched(void)
 {
-#ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
-        __might_sleep(__FILE__, __LINE__);
-#endif
-        /*
-         * The BKS might be reacquired before we have dropped
-         * PREEMPT_ACTIVE, which could trigger a second
-         * cond_resched() call.
-         */
-        do {
-                add_preempt_count(PREEMPT_ACTIVE);
-                schedule();
-                sub_preempt_count(PREEMPT_ACTIVE);
-        } while (need_resched());
+        add_preempt_count(PREEMPT_ACTIVE);
+        schedule();
+        sub_preempt_count(PREEMPT_ACTIVE);
 }
 
 int __sched _cond_resched(void)
 {
-        if (need_resched() && !(preempt_count() & PREEMPT_ACTIVE) &&
-                                        system_state == SYSTEM_RUNNING) {
+        if (should_resched()) {
                 __cond_resched();
                 return 1;
         }
@@ -6569,21 +6657,23 @@ int __sched _cond_resched(void)
 EXPORT_SYMBOL(_cond_resched);
 
 /*
- * cond_resched_lock() - if a reschedule is pending, drop the given lock,
+ * __cond_resched_lock() - if a reschedule is pending, drop the given lock,
  * call schedule, and on return reacquire the lock.
  *
  * This works OK both with and without CONFIG_PREEMPT. We do strange low-level
  * operations here to prevent schedule() from being called twice (once via
  * spin_unlock(), once by hand).
  */
-int cond_resched_lock(spinlock_t *lock)
+int __cond_resched_lock(spinlock_t *lock)
 {
-        int resched = need_resched() && system_state == SYSTEM_RUNNING;
+        int resched = should_resched();
         int ret = 0;
 
+        lockdep_assert_held(lock);
+
         if (spin_needbreak(lock) || resched) {
                 spin_unlock(lock);
-                if (resched && need_resched())
+                if (resched)
                         __cond_resched();
                 else
                         cpu_relax();
@@ -6592,13 +6682,13 @@ int cond_resched_lock(spinlock_t *lock)
         }
         return ret;
 }
-EXPORT_SYMBOL(cond_resched_lock);
+EXPORT_SYMBOL(__cond_resched_lock);
 
-int __sched cond_resched_softirq(void)
+int __sched __cond_resched_softirq(void)
 {
         BUG_ON(!in_softirq());
 
-        if (need_resched() && system_state == SYSTEM_RUNNING) {
+        if (should_resched()) {
                 local_bh_enable();
                 __cond_resched();
                 local_bh_disable();
@@ -6606,7 +6696,7 @@ int __sched cond_resched_softirq(void)
         }
         return 0;
 }
-EXPORT_SYMBOL(cond_resched_softirq);
+EXPORT_SYMBOL(__cond_resched_softirq);
 
 /**
  * yield - yield the current processor to other threads.
@@ -6630,11 +6720,13 @@ EXPORT_SYMBOL(yield);
  */
 void __sched io_schedule(void)
 {
-        struct rq *rq = &__raw_get_cpu_var(runqueues);
+        struct rq *rq = raw_rq();
 
         delayacct_blkio_start();
         atomic_inc(&rq->nr_iowait);
+        current->in_iowait = 1;
         schedule();
+        current->in_iowait = 0;
         atomic_dec(&rq->nr_iowait);
         delayacct_blkio_end();
 }
@@ -6642,12 +6734,14 @@ EXPORT_SYMBOL(io_schedule);
 
 long __sched io_schedule_timeout(long timeout)
 {
-        struct rq *rq = &__raw_get_cpu_var(runqueues);
+        struct rq *rq = raw_rq();
         long ret;
 
         delayacct_blkio_start();
         atomic_inc(&rq->nr_iowait);
+        current->in_iowait = 1;
         ret = schedule_timeout(timeout);
+        current->in_iowait = 0;
         atomic_dec(&rq->nr_iowait);
         delayacct_blkio_end();
         return ret;
@@ -6964,8 +7058,12 @@ int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
 
         if (migrate_task(p, cpumask_any_and(cpu_online_mask, new_mask), &req)) {
                 /* Need help from migration thread: drop lock and wait. */
+                struct task_struct *mt = rq->migration_thread;
+
+                get_task_struct(mt);
                 task_rq_unlock(rq, &flags);
                 wake_up_process(rq->migration_thread);
+                put_task_struct(mt);
                 wait_for_completion(&req.done);
                 tlb_migrate_finish(p->mm);
                 return 0;
@@ -7023,6 +7121,11 @@ fail:
         return ret;
 }
 
+#define RCU_MIGRATION_IDLE      0
+#define RCU_MIGRATION_NEED_QS   1
+#define RCU_MIGRATION_GOT_QS    2
+#define RCU_MIGRATION_MUST_SYNC 3
+
 /*
  * migration_thread - this is a highprio system thread that performs
  * thread migration by bumping thread off CPU then 'pushing' onto
@@ -7030,6 +7133,7 @@ fail:
  */
 static int migration_thread(void *data)
 {
+        int badcpu;
         int cpu = (long)data;
         struct rq *rq;
 
@@ -7045,7 +7149,7 @@ static int migration_thread(void *data)
 
                 if (cpu_is_offline(cpu)) {
                         spin_unlock_irq(&rq->lock);
-                        goto wait_to_die;
+                        break;
                 }
 
                 if (rq->active_balance) {
@@ -7064,23 +7168,23 @@ static int migration_thread(void *data)
                 req = list_entry(head->next, struct migration_req, list);
                 list_del_init(head->next);
 
-                spin_unlock(&rq->lock);
-                __migrate_task(req->task, cpu, req->dest_cpu);
+                if (req->task != NULL) {
+                        spin_unlock(&rq->lock);
+                        __migrate_task(req->task, cpu, req->dest_cpu);
+                } else if (likely(cpu == (badcpu = smp_processor_id()))) {
+                        req->dest_cpu = RCU_MIGRATION_GOT_QS;
+                        spin_unlock(&rq->lock);
+                } else {
+                        req->dest_cpu = RCU_MIGRATION_MUST_SYNC;
+                        spin_unlock(&rq->lock);
+                        WARN_ONCE(1, "migration_thread() on CPU %d, expected %d\n", badcpu, cpu);
+                }
                 local_irq_enable();
 
                 complete(&req->done);
         }
         __set_current_state(TASK_RUNNING);
-        return 0;
 
-wait_to_die:
-        /* Wait for kthread_stop */
-        set_current_state(TASK_INTERRUPTIBLE);
-        while (!kthread_should_stop()) {
-                schedule();
-                set_current_state(TASK_INTERRUPTIBLE);
-        }
-        __set_current_state(TASK_RUNNING);
         return 0;
 }
 
@@ -7270,6 +7374,7 @@ static void migrate_dead_tasks(unsigned int dead_cpu)
 static void calc_global_load_remove(struct rq *rq)
 {
         atomic_long_sub(rq->calc_load_active, &calc_load_tasks);
+        rq->calc_load_active = 0;
 }
 #endif /* CONFIG_HOTPLUG_CPU */
 
@@ -7494,7 +7599,9 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
                 rq = task_rq_lock(p, &flags);
                 __setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1);
                 task_rq_unlock(rq, &flags);
+                get_task_struct(p);
                 cpu_rq(cpu)->migration_thread = p;
+                rq->calc_load_update = calc_load_update;
                 break;
 
         case CPU_ONLINE:
@@ -7505,8 +7612,6 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
                 /* Update our root-domain */
                 rq = cpu_rq(cpu);
                 spin_lock_irqsave(&rq->lock, flags);
-                rq->calc_load_update = calc_load_update;
-                rq->calc_load_active = 0;
                 if (rq->rd) {
                         BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
 
@@ -7524,6 +7629,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
                 kthread_bind(cpu_rq(cpu)->migration_thread,
                              cpumask_any(cpu_online_mask));
                 kthread_stop(cpu_rq(cpu)->migration_thread);
+                put_task_struct(cpu_rq(cpu)->migration_thread);
                 cpu_rq(cpu)->migration_thread = NULL;
                 break;
 
@@ -7533,6 +7639,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
                 migrate_live_tasks(cpu);
                 rq = cpu_rq(cpu);
                 kthread_stop(rq->migration_thread);
+                put_task_struct(rq->migration_thread);
                 rq->migration_thread = NULL;
                 /* Idle task back to normal (off runqueue, low prio) */
                 spin_lock_irq(&rq->lock);
@@ -7603,7 +7710,7 @@ static int __init migration_init(void)
         migration_call(&migration_notifier, CPU_ONLINE, cpu);
         register_cpu_notifier(&migration_notifier);
 
-        return err;
+        return 0;
 }
 early_initcall(migration_init);
 #endif
@@ -7650,7 +7757,7 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
                         break;
                 }
 
-                if (!group->__cpu_power) {
+                if (!group->cpu_power) {
                         printk(KERN_CONT "\n");
                         printk(KERN_ERR "ERROR: domain->cpu_power not "
                                         "set\n");
@@ -7674,9 +7781,9 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
                 cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group));
 
                 printk(KERN_CONT " %s", str);
-                if (group->__cpu_power != SCHED_LOAD_SCALE) {
-                        printk(KERN_CONT " (__cpu_power = %d)",
-                                group->__cpu_power);
+                if (group->cpu_power != SCHED_LOAD_SCALE) {
+                        printk(KERN_CONT " (cpu_power = %d)",
+                                group->cpu_power);
                 }
 
                 group = group->next;
@@ -7741,9 +7848,7 @@ static int sd_degenerate(struct sched_domain *sd)
         }
 
         /* Following flags don't use groups */
-        if (sd->flags & (SD_WAKE_IDLE |
-                         SD_WAKE_AFFINE |
-                         SD_WAKE_BALANCE))
+        if (sd->flags & (SD_WAKE_AFFINE))
                 return 0;
 
         return 1;
@@ -7760,10 +7865,6 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
         if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
                 return 0;
 
-        /* Does parent contain flags not in child? */
-        /* WAKE_BALANCE is a subset of WAKE_AFFINE */
-        if (cflags & SD_WAKE_AFFINE)
-                pflags &= ~SD_WAKE_BALANCE;
         /* Flags needing groups don't count if only 1 group in parent */
         if (parent->groups == parent->groups->next) {
                 pflags &= ~(SD_LOAD_BALANCE |
@@ -7819,7 +7920,7 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd)
         rq->rd = rd;
 
         cpumask_set_cpu(rq->cpu, rd->span);
-        if (cpumask_test_cpu(rq->cpu, cpu_online_mask))
+        if (cpumask_test_cpu(rq->cpu, cpu_active_mask))
                 set_rq_online(rq);
 
         spin_unlock_irqrestore(&rq->lock, flags);
@@ -7828,7 +7929,7 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd)
                 free_rootdomain(old_rd);
 }
 
-static int __init_refok init_rootdomain(struct root_domain *rd, bool bootmem)
+static int init_rootdomain(struct root_domain *rd, bool bootmem)
 {
         gfp_t gfp = GFP_KERNEL;
 
@@ -7961,7 +8062,7 @@ init_sched_build_groups(const struct cpumask *span,
                         continue;
 
                 cpumask_clear(sched_group_cpus(sg));
-                sg->__cpu_power = 0;
+                sg->cpu_power = 0;
 
                 for_each_cpu(j, span) {
                         if (group_fn(j, cpu_map, NULL, tmpmask) != group)
@@ -8069,6 +8170,39 @@ struct static_sched_domain {
         DECLARE_BITMAP(span, CONFIG_NR_CPUS);
 };
 
+struct s_data {
+#ifdef CONFIG_NUMA
+        int                     sd_allnodes;
+        cpumask_var_t           domainspan;
+        cpumask_var_t           covered;
+        cpumask_var_t           notcovered;
+#endif
+        cpumask_var_t           nodemask;
+        cpumask_var_t           this_sibling_map;
+        cpumask_var_t           this_core_map;
+        cpumask_var_t           send_covered;
+        cpumask_var_t           tmpmask;
+        struct sched_group      **sched_group_nodes;
+        struct root_domain      *rd;
+};
+
+enum s_alloc {
+        sa_sched_groups = 0,
+        sa_rootdomain,
+        sa_tmpmask,
+        sa_send_covered,
+        sa_this_core_map,
+        sa_this_sibling_map,
+        sa_nodemask,
+        sa_sched_group_nodes,
+#ifdef CONFIG_NUMA
+        sa_notcovered,
+        sa_covered,
+        sa_domainspan,
+#endif
+        sa_none,
+};
+
 /*
  * SMT sched-domains:
  */
@@ -8186,11 +8320,76 @@ static void init_numa_sched_groups_power(struct sched_group *group_head)
                                 continue;
                         }
 
-                        sg_inc_cpu_power(sg, sd->groups->__cpu_power);
+                        sg->cpu_power += sd->groups->cpu_power;
                 }
                 sg = sg->next;
         } while (sg != group_head);
 }
+
+static int build_numa_sched_groups(struct s_data *d,
+                                   const struct cpumask *cpu_map, int num)
+{
+        struct sched_domain *sd;
+        struct sched_group *sg, *prev;
+        int n, j;
+
+        cpumask_clear(d->covered);
+        cpumask_and(d->nodemask, cpumask_of_node(num), cpu_map);
+        if (cpumask_empty(d->nodemask)) {
+                d->sched_group_nodes[num] = NULL;
+                goto out;
+        }
+
+        sched_domain_node_span(num, d->domainspan);
+        cpumask_and(d->domainspan, d->domainspan, cpu_map);
+
+        sg = kmalloc_node(sizeof(struct sched_group) + cpumask_size(),
+                          GFP_KERNEL, num);
+        if (!sg) {
+                printk(KERN_WARNING "Can not alloc domain group for node %d\n",
+                       num);
+                return -ENOMEM;
+        }
+        d->sched_group_nodes[num] = sg;
+
+        for_each_cpu(j, d->nodemask) {
+                sd = &per_cpu(node_domains, j).sd;
+                sd->groups = sg;
+        }
+
+        sg->cpu_power = 0;
+        cpumask_copy(sched_group_cpus(sg), d->nodemask);
+        sg->next = sg;
+        cpumask_or(d->covered, d->covered, d->nodemask);
+
+        prev = sg;
+        for (j = 0; j < nr_node_ids; j++) {
+                n = (num + j) % nr_node_ids;
+                cpumask_complement(d->notcovered, d->covered);
+                cpumask_and(d->tmpmask, d->notcovered, cpu_map);
+                cpumask_and(d->tmpmask, d->tmpmask, d->domainspan);
+                if (cpumask_empty(d->tmpmask))
+                        break;
+                cpumask_and(d->tmpmask, d->tmpmask, cpumask_of_node(n));
+                if (cpumask_empty(d->tmpmask))
+                        continue;
+                sg = kmalloc_node(sizeof(struct sched_group) + cpumask_size(),
+                                  GFP_KERNEL, num);
+                if (!sg) {
+                        printk(KERN_WARNING
+                               "Can not alloc domain group for node %d\n", j);
+                        return -ENOMEM;
+                }
+                sg->cpu_power = 0;
+                cpumask_copy(sched_group_cpus(sg), d->tmpmask);
+                sg->next = prev->next;
+                cpumask_or(d->covered, d->covered, d->tmpmask);
+                prev->next = sg;
+                prev = sg;
+        }
+out:
+        return 0;
+}
 #endif /* CONFIG_NUMA */
 
 #ifdef CONFIG_NUMA
@@ -8244,15 +8443,13 @@ static void free_sched_groups(const struct cpumask *cpu_map,
  * there are asymmetries in the topology. If there are asymmetries, group
  * having more cpu_power will pickup more load compared to the group having
  * less cpu_power.
- *
- * cpu_power will be a multiple of SCHED_LOAD_SCALE. This multiple represents
- * the maximum number of tasks a group can handle in the presence of other idle
- * or lightly loaded groups in the same sched domain.
  */
 static void init_sched_groups_power(int cpu, struct sched_domain *sd)
 {
         struct sched_domain *child;
         struct sched_group *group;
+        long power;
+        int weight;
 
         WARN_ON(!sd || !sd->groups);
 
@@ -8261,28 +8458,32 @@ static void init_sched_groups_power(int cpu, struct sched_domain *sd)
 
         child = sd->child;
 
-        sd->groups->__cpu_power = 0;
+        sd->groups->cpu_power = 0;
 
-        /*
-         * For perf policy, if the groups in child domain share resources
-         * (for example cores sharing some portions of the cache hierarchy
-         * or SMT), then set this domain groups cpu_power such that each group
-         * can handle only one task, when there are other idle groups in the
-         * same sched domain.
-         */
-        if (!child || (!(sd->flags & SD_POWERSAVINGS_BALANCE) &&
-                       (child->flags &
-                        (SD_SHARE_CPUPOWER | SD_SHARE_PKG_RESOURCES)))) {
-                sg_inc_cpu_power(sd->groups, SCHED_LOAD_SCALE);
+        if (!child) {
+                power = SCHED_LOAD_SCALE;
+                weight = cpumask_weight(sched_domain_span(sd));
+                /*
+                 * SMT siblings share the power of a single core.
+                 * Usually multiple threads get a better yield out of
+                 * that one core than a single thread would have,
+                 * reflect that in sd->smt_gain.
+                 */
+                if ((sd->flags & SD_SHARE_CPUPOWER) && weight > 1) {
+                        power *= sd->smt_gain;
+                        power /= weight;
+                        power >>= SCHED_LOAD_SHIFT;
+                }
+                sd->groups->cpu_power += power;
                 return;
         }
 
         /*
-         * add cpu_power of each child group to this groups cpu_power
+         * Add cpu_power of each child group to this groups cpu_power.
          */
         group = child->groups;
         do {
-                sg_inc_cpu_power(sd->groups, group->__cpu_power);
+                sd->groups->cpu_power += group->cpu_power;
                 group = group->next;
         } while (group != child->groups);
 }
@@ -8349,287 +8550,292 @@ static void set_domain_attribute(struct sched_domain *sd,
                 request = attr->relax_domain_level;
         if (request < sd->level) {
                 /* turn off idle balance on this domain */
-                sd->flags &= ~(SD_WAKE_IDLE|SD_BALANCE_NEWIDLE);
+                sd->flags &= ~(SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
         } else {
                 /* turn on idle balance on this domain */
-                sd->flags |= (SD_WAKE_IDLE_FAR|SD_BALANCE_NEWIDLE);
+                sd->flags |= (SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
+        }
+}
+
+static void __free_domain_allocs(struct s_data *d, enum s_alloc what,
+                                 const struct cpumask *cpu_map)
+{
+        switch (what) {
+        case sa_sched_groups:
+                free_sched_groups(cpu_map, d->tmpmask); /* fall through */
+                d->sched_group_nodes = NULL;
+        case sa_rootdomain:
+                free_rootdomain(d->rd); /* fall through */
+        case sa_tmpmask:
+                free_cpumask_var(d->tmpmask); /* fall through */
+        case sa_send_covered:
+                free_cpumask_var(d->send_covered); /* fall through */
+        case sa_this_core_map:
+                free_cpumask_var(d->this_core_map); /* fall through */
+        case sa_this_sibling_map:
+                free_cpumask_var(d->this_sibling_map); /* fall through */
+        case sa_nodemask:
+                free_cpumask_var(d->nodemask); /* fall through */
+        case sa_sched_group_nodes:
+#ifdef CONFIG_NUMA
+                kfree(d->sched_group_nodes); /* fall through */
+        case sa_notcovered:
+                free_cpumask_var(d->notcovered); /* fall through */
+        case sa_covered:
+                free_cpumask_var(d->covered); /* fall through */
+        case sa_domainspan:
+                free_cpumask_var(d->domainspan); /* fall through */
+#endif
+        case sa_none:
+                break;
         }
 }
 
-/*
- * Build sched domains for a given set of cpus and attach the sched domains
- * to the individual cpus
- */
-static int __build_sched_domains(const struct cpumask *cpu_map,
-                                 struct sched_domain_attr *attr)
+static enum s_alloc __visit_domain_allocation_hell(struct s_data *d,
+                                                   const struct cpumask *cpu_map)
 {
-        int i, err = -ENOMEM;
-        struct root_domain *rd;
-        cpumask_var_t nodemask, this_sibling_map, this_core_map, send_covered,
-                tmpmask;
 #ifdef CONFIG_NUMA
-        cpumask_var_t domainspan, covered, notcovered;
-        struct sched_group **sched_group_nodes = NULL;
-        int sd_allnodes = 0;
-
-        if (!alloc_cpumask_var(&domainspan, GFP_KERNEL))
-                goto out;
-        if (!alloc_cpumask_var(&covered, GFP_KERNEL))
-                goto free_domainspan;
-        if (!alloc_cpumask_var(&notcovered, GFP_KERNEL))
-                goto free_covered;
-#endif
-
-        if (!alloc_cpumask_var(&nodemask, GFP_KERNEL))
-                goto free_notcovered;
-        if (!alloc_cpumask_var(&this_sibling_map, GFP_KERNEL))
-                goto free_nodemask;
-        if (!alloc_cpumask_var(&this_core_map, GFP_KERNEL))
-                goto free_this_sibling_map;
-        if (!alloc_cpumask_var(&send_covered, GFP_KERNEL))
-                goto free_this_core_map;
-        if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
-                goto free_send_covered;
-
-#ifdef CONFIG_NUMA
-        /*
-         * Allocate the per-node list of sched groups
-         */
-        sched_group_nodes = kcalloc(nr_node_ids, sizeof(struct sched_group *),
-                                    GFP_KERNEL);
-        if (!sched_group_nodes) {
+        if (!alloc_cpumask_var(&d->domainspan, GFP_KERNEL))
+                return sa_none;
+        if (!alloc_cpumask_var(&d->covered, GFP_KERNEL))
+                return sa_domainspan;
+        if (!alloc_cpumask_var(&d->notcovered, GFP_KERNEL))
+                return sa_covered;
+        /* Allocate the per-node list of sched groups */
+        d->sched_group_nodes = kcalloc(nr_node_ids,
+                                      sizeof(struct sched_group *), GFP_KERNEL);
+        if (!d->sched_group_nodes) {
                 printk(KERN_WARNING "Can not alloc sched group node list\n");
-                goto free_tmpmask;
-        }
-#endif
-
-        rd = alloc_rootdomain();
-        if (!rd) {
+                return sa_notcovered;
+        }
+        sched_group_nodes_bycpu[cpumask_first(cpu_map)] = d->sched_group_nodes;
+#endif
+        if (!alloc_cpumask_var(&d->nodemask, GFP_KERNEL))
+                return sa_sched_group_nodes;
+        if (!alloc_cpumask_var(&d->this_sibling_map, GFP_KERNEL))
+                return sa_nodemask;
+        if (!alloc_cpumask_var(&d->this_core_map, GFP_KERNEL))
+                return sa_this_sibling_map;
+        if (!alloc_cpumask_var(&d->send_covered, GFP_KERNEL))
+                return sa_this_core_map;
+        if (!alloc_cpumask_var(&d->tmpmask, GFP_KERNEL))
+                return sa_send_covered;
+        d->rd = alloc_rootdomain();
+        if (!d->rd) {
                 printk(KERN_WARNING "Cannot alloc root domain\n");
-                goto free_sched_groups;
+                return sa_tmpmask;
         }
+        return sa_rootdomain;
+}
 
+static struct sched_domain *__build_numa_sched_domains(struct s_data *d,
+        const struct cpumask *cpu_map, struct sched_domain_attr *attr, int i)
+{
+        struct sched_domain *sd = NULL;
 #ifdef CONFIG_NUMA
-        sched_group_nodes_bycpu[cpumask_first(cpu_map)] = sched_group_nodes;
-#endif
-
-        /*
-         * Set up domains for cpus specified by the cpu_map.
-         */
-        for_each_cpu(i, cpu_map) {
-                struct sched_domain *sd = NULL, *p;
-
-                cpumask_and(nodemask, cpumask_of_node(cpu_to_node(i)), cpu_map);
-
-#ifdef CONFIG_NUMA
-                if (cpumask_weight(cpu_map) >
-                                SD_NODES_PER_DOMAIN*cpumask_weight(nodemask)) {
-                        sd = &per_cpu(allnodes_domains, i).sd;
-                        SD_INIT(sd, ALLNODES);
-                        set_domain_attribute(sd, attr);
-                        cpumask_copy(sched_domain_span(sd), cpu_map);
-                        cpu_to_allnodes_group(i, cpu_map, &sd->groups, tmpmask);
-                        p = sd;
-                        sd_allnodes = 1;
-                } else
-                        p = NULL;
+        struct sched_domain *parent;
 
-                sd = &per_cpu(node_domains, i).sd;
-                SD_INIT(sd, NODE);
+        d->sd_allnodes = 0;
+        if (cpumask_weight(cpu_map) >
+            SD_NODES_PER_DOMAIN * cpumask_weight(d->nodemask)) {
+                sd = &per_cpu(allnodes_domains, i).sd;
+                SD_INIT(sd, ALLNODES);
                 set_domain_attribute(sd, attr);
-                sched_domain_node_span(cpu_to_node(i), sched_domain_span(sd));
-                sd->parent = p;
-                if (p)
-                        p->child = sd;
-                cpumask_and(sched_domain_span(sd),
-                            sched_domain_span(sd), cpu_map);
+                cpumask_copy(sched_domain_span(sd), cpu_map);
+                cpu_to_allnodes_group(i, cpu_map, &sd->groups, d->tmpmask);
+                d->sd_allnodes = 1;
+        }
+        parent = sd;
+
+        sd = &per_cpu(node_domains, i).sd;
+        SD_INIT(sd, NODE);
+        set_domain_attribute(sd, attr);
+        sched_domain_node_span(cpu_to_node(i), sched_domain_span(sd));
+        sd->parent = parent;
+        if (parent)
+                parent->child = sd;
+        cpumask_and(sched_domain_span(sd), sched_domain_span(sd), cpu_map);
 #endif
+        return sd;
+}
 
-                p = sd;
-                sd = &per_cpu(phys_domains, i).sd;
-                SD_INIT(sd, CPU);
-                set_domain_attribute(sd, attr);
-                cpumask_copy(sched_domain_span(sd), nodemask);
-                sd->parent = p;
-                if (p)
-                        p->child = sd;
-                cpu_to_phys_group(i, cpu_map, &sd->groups, tmpmask);
+static struct sched_domain *__build_cpu_sched_domain(struct s_data *d,
+        const struct cpumask *cpu_map, struct sched_domain_attr *attr,
+        struct sched_domain *parent, int i)
+{
+        struct sched_domain *sd;
+        sd = &per_cpu(phys_domains, i).sd;
+        SD_INIT(sd, CPU);
+        set_domain_attribute(sd, attr);
+        cpumask_copy(sched_domain_span(sd), d->nodemask);
+        sd->parent = parent;
+        if (parent)
+                parent->child = sd;
+        cpu_to_phys_group(i, cpu_map, &sd->groups, d->tmpmask);
+        return sd;
+}
 
+static struct sched_domain *__build_mc_sched_domain(struct s_data *d,
+        const struct cpumask *cpu_map, struct sched_domain_attr *attr,
+        struct sched_domain *parent, int i)
+{
+        struct sched_domain *sd = parent;
 #ifdef CONFIG_SCHED_MC
-                p = sd;
-                sd = &per_cpu(core_domains, i).sd;
-                SD_INIT(sd, MC);
-                set_domain_attribute(sd, attr);
-                cpumask_and(sched_domain_span(sd), cpu_map,
-                                                   cpu_coregroup_mask(i));
-                sd->parent = p;
-                p->child = sd;
-                cpu_to_core_group(i, cpu_map, &sd->groups, tmpmask);
+        sd = &per_cpu(core_domains, i).sd;
+        SD_INIT(sd, MC);
+        set_domain_attribute(sd, attr);
+        cpumask_and(sched_domain_span(sd), cpu_map, cpu_coregroup_mask(i));
+        sd->parent = parent;
+        parent->child = sd;
+        cpu_to_core_group(i, cpu_map, &sd->groups, d->tmpmask);
 #endif
+        return sd;
+}
 
+static struct sched_domain *__build_smt_sched_domain(struct s_data *d,
+        const struct cpumask *cpu_map, struct sched_domain_attr *attr,
+        struct sched_domain *parent, int i)
+{
+        struct sched_domain *sd = parent;
 #ifdef CONFIG_SCHED_SMT
-                p = sd;
-                sd = &per_cpu(cpu_domains, i).sd;
-                SD_INIT(sd, SIBLING);
-                set_domain_attribute(sd, attr);
-                cpumask_and(sched_domain_span(sd),
-                            topology_thread_cpumask(i), cpu_map);
-                sd->parent = p;
-                p->child = sd;
-                cpu_to_cpu_group(i, cpu_map, &sd->groups, tmpmask);
+        sd = &per_cpu(cpu_domains, i).sd;
+        SD_INIT(sd, SIBLING);
+        set_domain_attribute(sd, attr);
+        cpumask_and(sched_domain_span(sd), cpu_map, topology_thread_cpumask(i));
+        sd->parent = parent;
+        parent->child = sd;
+        cpu_to_cpu_group(i, cpu_map, &sd->groups, d->tmpmask);
 #endif
-        }
+        return sd;
+}
 
+static void build_sched_groups(struct s_data *d, enum sched_domain_level l,
+                               const struct cpumask *cpu_map, int cpu)
+{
+        switch (l) {
 #ifdef CONFIG_SCHED_SMT
-        /* Set up CPU (sibling) groups */
-        for_each_cpu(i, cpu_map) {
-                cpumask_and(this_sibling_map,
-                            topology_thread_cpumask(i), cpu_map);
-                if (i != cpumask_first(this_sibling_map))
-                        continue;
-
-                init_sched_build_groups(this_sibling_map, cpu_map,
-                                        &cpu_to_cpu_group,
-                                        send_covered, tmpmask);
-        }
+        case SD_LV_SIBLING: /* set up CPU (sibling) groups */
+                cpumask_and(d->this_sibling_map, cpu_map,
+                            topology_thread_cpumask(cpu));
+                if (cpu == cpumask_first(d->this_sibling_map))
+                        init_sched_build_groups(d->this_sibling_map, cpu_map,
+                                                &cpu_to_cpu_group,
+                                                d->send_covered, d->tmpmask);
+                break;
 #endif
-
 #ifdef CONFIG_SCHED_MC
-        /* Set up multi-core groups */
-        for_each_cpu(i, cpu_map) {
-                cpumask_and(this_core_map, cpu_coregroup_mask(i), cpu_map);
-                if (i != cpumask_first(this_core_map))
-                        continue;
-
-                init_sched_build_groups(this_core_map, cpu_map,
-                                        &cpu_to_core_group,
-                                        send_covered, tmpmask);
-        }
+        case SD_LV_MC: /* set up multi-core groups */
+                cpumask_and(d->this_core_map, cpu_map, cpu_coregroup_mask(cpu));
+                if (cpu == cpumask_first(d->this_core_map))
+                        init_sched_build_groups(d->this_core_map, cpu_map,
+                                                &cpu_to_core_group,
+                                                d->send_covered, d->tmpmask);
+                break;
 #endif
-
-        /* Set up physical groups */
-        for (i = 0; i < nr_node_ids; i++) {
-                cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
-                if (cpumask_empty(nodemask))
-                        continue;
-
-                init_sched_build_groups(nodemask, cpu_map,
-                                        &cpu_to_phys_group,
-                                        send_covered, tmpmask);
-        }
-
+        case SD_LV_CPU: /* set up physical groups */
+                cpumask_and(d->nodemask, cpumask_of_node(cpu), cpu_map);
+                if (!cpumask_empty(d->nodemask))
+                        init_sched_build_groups(d->nodemask, cpu_map,
+                                                &cpu_to_phys_group,
+                                                d->send_covered, d->tmpmask);
+                break;
 #ifdef CONFIG_NUMA
-        /* Set up node groups */
-        if (sd_allnodes) {
-                init_sched_build_groups(cpu_map, cpu_map,
-                                        &cpu_to_allnodes_group,
-                                        send_covered, tmpmask);
+        case SD_LV_ALLNODES:
+                init_sched_build_groups(cpu_map, cpu_map, &cpu_to_allnodes_group,
+                                        d->send_covered, d->tmpmask);
+                break;
+#endif
+        default:
+                break;
         }
+}
 
-        for (i = 0; i < nr_node_ids; i++) {
-                /* Set up node groups */
-                struct sched_group *sg, *prev;
-                int j;
-
-                cpumask_clear(covered);
-                cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
-                if (cpumask_empty(nodemask)) {
-                        sched_group_nodes[i] = NULL;
-                        continue;
-                }
+/*
+ * Build sched domains for a given set of cpus and attach the sched domains
+ * to the individual cpus
+ */
+static int __build_sched_domains(const struct cpumask *cpu_map,
+                                 struct sched_domain_attr *attr)
+{
+        enum s_alloc alloc_state = sa_none;
+        struct s_data d;
+        struct sched_domain *sd;
+        int i;
+#ifdef CONFIG_NUMA
+        d.sd_allnodes = 0;
+#endif
 
-                sched_domain_node_span(i, domainspan);
-                cpumask_and(domainspan, domainspan, cpu_map);
+        alloc_state = __visit_domain_allocation_hell(&d, cpu_map);
+        if (alloc_state != sa_rootdomain)
+                goto error;
+        alloc_state = sa_sched_groups;
 
-                sg = kmalloc_node(sizeof(struct sched_group) + cpumask_size(),
-                                  GFP_KERNEL, i);
-                if (!sg) {
-                        printk(KERN_WARNING "Can not alloc domain group for "
-                                "node %d\n", i);
-                        goto error;
-                }
-                sched_group_nodes[i] = sg;
-                for_each_cpu(j, nodemask) {
-                        struct sched_domain *sd;
+        /*
+         * Set up domains for cpus specified by the cpu_map.
+         */
+        for_each_cpu(i, cpu_map) {
+                cpumask_and(d.nodemask, cpumask_of_node(cpu_to_node(i)),
+                            cpu_map);
 
-                        sd = &per_cpu(node_domains, j).sd;
-                        sd->groups = sg;
-                }
-                sg->__cpu_power = 0;
-                cpumask_copy(sched_group_cpus(sg), nodemask);
-                sg->next = sg;
-                cpumask_or(covered, covered, nodemask);
-                prev = sg;
+                sd = __build_numa_sched_domains(&d, cpu_map, attr, i);
+                sd = __build_cpu_sched_domain(&d, cpu_map, attr, sd, i);
+                sd = __build_mc_sched_domain(&d, cpu_map, attr, sd, i);
+                sd = __build_smt_sched_domain(&d, cpu_map, attr, sd, i);
+        }
 
-                for (j = 0; j < nr_node_ids; j++) {
-                        int n = (i + j) % nr_node_ids;
+        for_each_cpu(i, cpu_map) {
+                build_sched_groups(&d, SD_LV_SIBLING, cpu_map, i);
+                build_sched_groups(&d, SD_LV_MC, cpu_map, i);
+        }
 
-                        cpumask_complement(notcovered, covered);
-                        cpumask_and(tmpmask, notcovered, cpu_map);
-                        cpumask_and(tmpmask, tmpmask, domainspan);
-                        if (cpumask_empty(tmpmask))
-                                break;
+        /* Set up physical groups */
+        for (i = 0; i < nr_node_ids; i++)
+                build_sched_groups(&d, SD_LV_CPU, cpu_map, i);
 
-                        cpumask_and(tmpmask, tmpmask, cpumask_of_node(n));
-                        if (cpumask_empty(tmpmask))
-                                continue;
+#ifdef CONFIG_NUMA
+        /* Set up node groups */
+        if (d.sd_allnodes)
+                build_sched_groups(&d, SD_LV_ALLNODES, cpu_map, 0);
 
-                        sg = kmalloc_node(sizeof(struct sched_group) +
-                                          cpumask_size(),
-                                          GFP_KERNEL, i);
-                        if (!sg) {
-                                printk(KERN_WARNING
-                                "Can not alloc domain group for node %d\n", j);
-                                goto error;
-                        }
-                        sg->__cpu_power = 0;
-                        cpumask_copy(sched_group_cpus(sg), tmpmask);
-                        sg->next = prev->next;
-                        cpumask_or(covered, covered, tmpmask);
-                        prev->next = sg;
-                        prev = sg;
-                }
-        }
+        for (i = 0; i < nr_node_ids; i++)
+                if (build_numa_sched_groups(&d, cpu_map, i))
+                        goto error;
 #endif
 
         /* Calculate CPU power for physical packages and nodes */
 #ifdef CONFIG_SCHED_SMT
         for_each_cpu(i, cpu_map) {
-                struct sched_domain *sd = &per_cpu(cpu_domains, i).sd;
-
+                sd = &per_cpu(cpu_domains, i).sd;
                 init_sched_groups_power(i, sd);
         }
 #endif
 #ifdef CONFIG_SCHED_MC
         for_each_cpu(i, cpu_map) {
-                struct sched_domain *sd = &per_cpu(core_domains, i).sd;
-
+                sd = &per_cpu(core_domains, i).sd;
                 init_sched_groups_power(i, sd);
         }
 #endif
 
         for_each_cpu(i, cpu_map) {
-                struct sched_domain *sd = &per_cpu(phys_domains, i).sd;
-
+                sd = &per_cpu(phys_domains, i).sd;
                 init_sched_groups_power(i, sd);
         }
 
 #ifdef CONFIG_NUMA
         for (i = 0; i < nr_node_ids; i++)
-                init_numa_sched_groups_power(sched_group_nodes[i]);
+                init_numa_sched_groups_power(d.sched_group_nodes[i]);
 
-        if (sd_allnodes) {
+        if (d.sd_allnodes) {
                 struct sched_group *sg;
 
                 cpu_to_allnodes_group(cpumask_first(cpu_map), cpu_map, &sg,
-                                                                tmpmask);
+                                                                d.tmpmask);
                 init_numa_sched_groups_power(sg);
         }
 #endif
 
         /* Attach the domains */
         for_each_cpu(i, cpu_map) {
-                struct sched_domain *sd;
 #ifdef CONFIG_SCHED_SMT
                 sd = &per_cpu(cpu_domains, i).sd;
 #elif defined(CONFIG_SCHED_MC)
@@ -8637,44 +8843,16 @@ static int __build_sched_domains(const struct cpumask *cpu_map,
 #else
                 sd = &per_cpu(phys_domains, i).sd;
 #endif
-                cpu_attach_domain(sd, rd, i);
+                cpu_attach_domain(sd, d.rd, i);
         }
 
-        err = 0;
-
-free_tmpmask:
-        free_cpumask_var(tmpmask);
-free_send_covered:
-        free_cpumask_var(send_covered);
-free_this_core_map:
-        free_cpumask_var(this_core_map);
-free_this_sibling_map:
-        free_cpumask_var(this_sibling_map);
-free_nodemask:
-        free_cpumask_var(nodemask);
-free_notcovered:
-#ifdef CONFIG_NUMA
-        free_cpumask_var(notcovered);
-free_covered:
-        free_cpumask_var(covered);
-free_domainspan:
-        free_cpumask_var(domainspan);
-out:
-#endif
-        return err;
-
-free_sched_groups:
-#ifdef CONFIG_NUMA
-        kfree(sched_group_nodes);
-#endif
-        goto free_tmpmask;
+        d.sched_group_nodes = NULL; /* don't free this we still need it */
+        __free_domain_allocs(&d, sa_tmpmask, cpu_map);
+        return 0;
 
-#ifdef CONFIG_NUMA
 error:
-        free_sched_groups(cpu_map, tmpmask);
-        free_rootdomain(rd);
-        goto free_tmpmask;
-#endif
+        __free_domain_allocs(&d, alloc_state, cpu_map);
+        return -ENOMEM;
 }
 
 static int build_sched_domains(const struct cpumask *cpu_map)
@@ -9075,7 +9253,7 @@ static void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq)
 #ifdef CONFIG_SMP
         rt_rq->rt_nr_migratory = 0;
         rt_rq->overloaded = 0;
-        plist_head_init(&rq->rt.pushable_tasks, &rq->lock);
+        plist_head_init(&rt_rq->pushable_tasks, &rq->lock);
 #endif
 
         rt_rq->rt_time = 0;
@@ -9282,11 +9460,11 @@ void __init sched_init(void)
                  * system cpu resource, based on the weight assigned to root
                  * user's cpu share (INIT_TASK_GROUP_LOAD). This is accomplished
                  * by letting tasks of init_task_group sit in a separate cfs_rq
-                 * (init_cfs_rq) and having one entity represent this group of
+                 * (init_tg_cfs_rq) and having one entity represent this group of
                  * tasks in rq->cfs (i.e init_task_group->se[] != NULL).
                  */
                 init_tg_cfs_entry(&init_task_group,
-                                &per_cpu(init_cfs_rq, i),
+                                &per_cpu(init_tg_cfs_rq, i),
                                 &per_cpu(init_sched_entity, i), i, 1,
                                 root_task_group.se[i]);
 
@@ -9312,6 +9490,7 @@ void __init sched_init(void)
 #ifdef CONFIG_SMP
                 rq->sd = NULL;
                 rq->rd = NULL;
+                rq->post_schedule = 0;
                 rq->active_balance = 0;
                 rq->next_balance = jiffies;
                 rq->push_cpu = 0;
@@ -9376,13 +9555,20 @@ void __init sched_init(void)
 }
 
 #ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
-void __might_sleep(char *file, int line)
+static inline int preempt_count_equals(int preempt_offset)
+{
+        int nested = preempt_count() & ~PREEMPT_ACTIVE;
+
+        return (nested == PREEMPT_INATOMIC_BASE + preempt_offset);
+}
+
+void __might_sleep(char *file, int line, int preempt_offset)
 {
 #ifdef in_atomic
         static unsigned long prev_jiffy;        /* ratelimiting */
 
-        if ((!in_atomic() && !irqs_disabled()) ||
-                    system_state != SYSTEM_RUNNING || oops_in_progress)
+        if ((preempt_count_equals(preempt_offset) && !irqs_disabled()) ||
+            system_state != SYSTEM_RUNNING || oops_in_progress)
                 return;
         if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
                 return;
@@ -10559,3 +10745,113 @@ struct cgroup_subsys cpuacct_subsys = {
         .subsys_id = cpuacct_subsys_id,
 };
 #endif  /* CONFIG_CGROUP_CPUACCT */
+
+#ifndef CONFIG_SMP
+
+int rcu_expedited_torture_stats(char *page)
+{
+        return 0;
+}
+EXPORT_SYMBOL_GPL(rcu_expedited_torture_stats);
+
+void synchronize_sched_expedited(void)
+{
+}
+EXPORT_SYMBOL_GPL(synchronize_sched_expedited);
+
+#else /* #ifndef CONFIG_SMP */
+
+static DEFINE_PER_CPU(struct migration_req, rcu_migration_req);
+static DEFINE_MUTEX(rcu_sched_expedited_mutex);
+
+#define RCU_EXPEDITED_STATE_POST -2
+#define RCU_EXPEDITED_STATE_IDLE -1
+
+static int rcu_expedited_state = RCU_EXPEDITED_STATE_IDLE;
+
+int rcu_expedited_torture_stats(char *page)
+{
+        int cnt = 0;
+        int cpu;
+
+        cnt += sprintf(&page[cnt], "state: %d /", rcu_expedited_state);
+        for_each_online_cpu(cpu) {
+                 cnt += sprintf(&page[cnt], " %d:%d",
+                                cpu, per_cpu(rcu_migration_req, cpu).dest_cpu);
+        }
+        cnt += sprintf(&page[cnt], "\n");
+        return cnt;
+}
+EXPORT_SYMBOL_GPL(rcu_expedited_torture_stats);
+
+static long synchronize_sched_expedited_count;
+
+/*
+ * Wait for an rcu-sched grace period to elapse, but use "big hammer"
+ * approach to force grace period to end quickly.  This consumes
+ * significant time on all CPUs, and is thus not recommended for
+ * any sort of common-case code.
+ *
+ * Note that it is illegal to call this function while holding any
+ * lock that is acquired by a CPU-hotplug notifier.  Failing to
+ * observe this restriction will result in deadlock.
+ */
+void synchronize_sched_expedited(void)
+{
+        int cpu;
+        unsigned long flags;
+        bool need_full_sync = 0;
+        struct rq *rq;
+        struct migration_req *req;
+        long snap;
+        int trycount = 0;
+
+        smp_mb();  /* ensure prior mod happens before capturing snap. */
+        snap = ACCESS_ONCE(synchronize_sched_expedited_count) + 1;
+        get_online_cpus();
+        while (!mutex_trylock(&rcu_sched_expedited_mutex)) {
+                put_online_cpus();
+                if (trycount++ < 10)
+                        udelay(trycount * num_online_cpus());
+                else {
+                        synchronize_sched();
+                        return;
+                }
+                if (ACCESS_ONCE(synchronize_sched_expedited_count) - snap > 0) {
+                        smp_mb(); /* ensure test happens before caller kfree */
+                        return;
+                }
+                get_online_cpus();
+        }
+        rcu_expedited_state = RCU_EXPEDITED_STATE_POST;
+        for_each_online_cpu(cpu) {
+                rq = cpu_rq(cpu);
+                req = &per_cpu(rcu_migration_req, cpu);
+                init_completion(&req->done);
+                req->task = NULL;
+                req->dest_cpu = RCU_MIGRATION_NEED_QS;
+                spin_lock_irqsave(&rq->lock, flags);
+                list_add(&req->list, &rq->migration_queue);
+                spin_unlock_irqrestore(&rq->lock, flags);
+                wake_up_process(rq->migration_thread);
+        }
+        for_each_online_cpu(cpu) {
+                rcu_expedited_state = cpu;
+                req = &per_cpu(rcu_migration_req, cpu);
+                rq = cpu_rq(cpu);
+                wait_for_completion(&req->done);
+                spin_lock_irqsave(&rq->lock, flags);
+                if (unlikely(req->dest_cpu == RCU_MIGRATION_MUST_SYNC))
+                        need_full_sync = 1;
+                req->dest_cpu = RCU_MIGRATION_IDLE;
+                spin_unlock_irqrestore(&rq->lock, flags);
+        }
+        rcu_expedited_state = RCU_EXPEDITED_STATE_IDLE;
+        mutex_unlock(&rcu_sched_expedited_mutex);
+        put_online_cpus();
+        if (need_full_sync)
+                synchronize_sched();
+}
+EXPORT_SYMBOL_GPL(synchronize_sched_expedited);
+
+#endif /* #else #ifndef CONFIG_SMP */
diff --git a/kernel/sched_cpupri.c b/kernel/sched_cpupri.c
index 7deffc9f0e5f..0f052fc674d5 100644
--- a/kernel/sched_cpupri.c
+++ b/kernel/sched_cpupri.c
@@ -81,8 +81,21 @@ int cpupri_find(struct cpupri *cp, struct task_struct *p,
                 if (cpumask_any_and(&p->cpus_allowed, vec->mask) >= nr_cpu_ids)
                         continue;
 
-                if (lowest_mask)
+                if (lowest_mask) {
                         cpumask_and(lowest_mask, &p->cpus_allowed, vec->mask);
+
+                        /*
+                         * We have to ensure that we have at least one bit
+                         * still set in the array, since the map could have
+                         * been concurrently emptied between the first and
+                         * second reads of vec->mask.  If we hit this
+                         * condition, simply act as though we never hit this
+                         * priority level and continue on.
+                         */
+                        if (cpumask_any(lowest_mask) >= nr_cpu_ids)
+                                continue;
+                }
+
                 return 1;
         }
 
@@ -114,21 +127,11 @@ void cpupri_set(struct cpupri *cp, int cpu, int newpri)
 
         /*
          * If the cpu was currently mapped to a different value, we
-         * first need to unmap the old value
+         * need to map it to the new value then remove the old value.
+         * Note, we must add the new value first, otherwise we risk the
+         * cpu being cleared from pri_active, and this cpu could be
+         * missed for a push or pull.
          */
-        if (likely(oldpri != CPUPRI_INVALID)) {
-                struct cpupri_vec *vec  = &cp->pri_to_cpu[oldpri];
-
-                spin_lock_irqsave(&vec->lock, flags);
-
-                vec->count--;
-                if (!vec->count)
-                        clear_bit(oldpri, cp->pri_active);
-                cpumask_clear_cpu(cpu, vec->mask);
-
-                spin_unlock_irqrestore(&vec->lock, flags);
-        }
-
         if (likely(newpri != CPUPRI_INVALID)) {
                 struct cpupri_vec *vec = &cp->pri_to_cpu[newpri];
 
@@ -141,6 +144,18 @@ void cpupri_set(struct cpupri *cp, int cpu, int newpri)
 
                 spin_unlock_irqrestore(&vec->lock, flags);
         }
+        if (likely(oldpri != CPUPRI_INVALID)) {
+                struct cpupri_vec *vec  = &cp->pri_to_cpu[oldpri];
+
+                spin_lock_irqsave(&vec->lock, flags);
+
+                vec->count--;
+                if (!vec->count)
+                        clear_bit(oldpri, cp->pri_active);
+                cpumask_clear_cpu(cpu, vec->mask);
+
+                spin_unlock_irqrestore(&vec->lock, flags);
+        }
 
         *currpri = newpri;
 }
@@ -152,7 +167,7 @@ void cpupri_set(struct cpupri *cp, int cpu, int newpri)
  *
  * Returns: -ENOMEM if memory fails.
  */
-int __init_refok cpupri_init(struct cpupri *cp, bool bootmem)
+int cpupri_init(struct cpupri *cp, bool bootmem)
 {
         gfp_t gfp = GFP_KERNEL;
         int i;
diff --git a/kernel/sched_debug.c b/kernel/sched_debug.c
index 467ca72f1657..efb84409bc43 100644
--- a/kernel/sched_debug.c
+++ b/kernel/sched_debug.c
@@ -162,7 +162,7 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
 {
         s64 MIN_vruntime = -1, min_vruntime, max_vruntime = -1,
                 spread, rq0_min_vruntime, spread0;
-        struct rq *rq = &per_cpu(runqueues, cpu);
+        struct rq *rq = cpu_rq(cpu);
         struct sched_entity *last;
         unsigned long flags;
 
@@ -191,7 +191,7 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
         if (last)
                 max_vruntime = last->vruntime;
         min_vruntime = cfs_rq->min_vruntime;
-        rq0_min_vruntime = per_cpu(runqueues, 0).cfs.min_vruntime;
+        rq0_min_vruntime = cpu_rq(0)->cfs.min_vruntime;
         spin_unlock_irqrestore(&rq->lock, flags);
         SEQ_printf(m, "  .%-30s: %Ld.%06ld\n", "MIN_vruntime",
                         SPLIT_NS(MIN_vruntime));
@@ -248,7 +248,7 @@ void print_rt_rq(struct seq_file *m, int cpu, struct rt_rq *rt_rq)
 
 static void print_cpu(struct seq_file *m, int cpu)
 {
-        struct rq *rq = &per_cpu(runqueues, cpu);
+        struct rq *rq = cpu_rq(cpu);
 
 #ifdef CONFIG_X86
         {
@@ -395,6 +395,7 @@ void proc_sched_show_task(struct task_struct *p, struct seq_file *m)
         PN(se.sum_exec_runtime);
         PN(se.avg_overlap);
         PN(se.avg_wakeup);
+        PN(se.avg_running);
 
         nr_switches = p->nvcsw + p->nivcsw;
 
@@ -409,6 +410,8 @@ void proc_sched_show_task(struct task_struct *p, struct seq_file *m)
         PN(se.wait_max);
         PN(se.wait_sum);
         P(se.wait_count);
+        PN(se.iowait_sum);
+        P(se.iowait_count);
         P(sched_info.bkl_count);
         P(se.nr_migrations);
         P(se.nr_migrations_cold);
@@ -479,6 +482,8 @@ void proc_sched_set_task(struct task_struct *p)
         p->se.wait_max                          = 0;
         p->se.wait_sum                          = 0;
         p->se.wait_count                        = 0;
+        p->se.iowait_sum                        = 0;
+        p->se.iowait_count                      = 0;
         p->se.sleep_max                         = 0;
         p->se.sum_sleep_runtime                 = 0;
         p->se.block_max                         = 0;
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index 5f9650e8fe75..10d218ab69f2 100644
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -24,7 +24,7 @@
 
 /*
  * Targeted preemption latency for CPU-bound tasks:
- * (default: 20ms * (1 + ilog(ncpus)), units: nanoseconds)
+ * (default: 5ms * (1 + ilog(ncpus)), units: nanoseconds)
  *
  * NOTE: this latency value is not the same as the concept of
  * 'timeslice length' - timeslices in CFS are of variable length
@@ -34,13 +34,13 @@
  * (to see the precise effective timeslice length of your workload,
  *  run vmstat and monitor the context-switches (cs) field)
  */
-unsigned int sysctl_sched_latency = 20000000ULL;
+unsigned int sysctl_sched_latency = 5000000ULL;
 
 /*
  * Minimal preemption granularity for CPU-bound tasks:
- * (default: 4 msec * (1 + ilog(ncpus)), units: nanoseconds)
+ * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds)
  */
-unsigned int sysctl_sched_min_granularity = 4000000ULL;
+unsigned int sysctl_sched_min_granularity = 1000000ULL;
 
 /*
  * is kept at sysctl_sched_latency / sysctl_sched_min_granularity
@@ -48,10 +48,10 @@ unsigned int sysctl_sched_min_granularity = 4000000ULL;
 static unsigned int sched_nr_latency = 5;
 
 /*
- * After fork, child runs first. (default) If set to 0 then
+ * After fork, child runs first. If set to 0 (default) then
  * parent will (try to) run first.
  */
-const_debug unsigned int sysctl_sched_child_runs_first = 1;
+unsigned int sysctl_sched_child_runs_first __read_mostly;
 
 /*
  * sys_sched_yield() compat mode
@@ -63,13 +63,13 @@ unsigned int __read_mostly sysctl_sched_compat_yield;
 
 /*
  * SCHED_OTHER wake-up granularity.
- * (default: 5 msec * (1 + ilog(ncpus)), units: nanoseconds)
+ * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds)
  *
  * This option delays the preemption effects of decoupled workloads
  * and reduces their over-scheduling. Synchronous workloads will still
  * have immediate wakeup/sleep latencies.
  */
-unsigned int sysctl_sched_wakeup_granularity = 5000000UL;
+unsigned int sysctl_sched_wakeup_granularity = 1000000UL;
 
 const_debug unsigned int sysctl_sched_migration_cost = 500000UL;
 
@@ -79,11 +79,6 @@ static const struct sched_class fair_sched_class;
  * CFS operations on generic schedulable entities:
  */
 
-static inline struct task_struct *task_of(struct sched_entity *se)
-{
-        return container_of(se, struct task_struct, se);
-}
-
 #ifdef CONFIG_FAIR_GROUP_SCHED
 
 /* cpu runqueue to which this cfs_rq is attached */
@@ -95,6 +90,14 @@ static inline struct rq *rq_of(struct cfs_rq *cfs_rq)
 /* An entity is a task if it doesn't "own" a runqueue */
 #define entity_is_task(se)      (!se->my_q)
 
+static inline struct task_struct *task_of(struct sched_entity *se)
+{
+#ifdef CONFIG_SCHED_DEBUG
+        WARN_ON_ONCE(!entity_is_task(se));
+#endif
+        return container_of(se, struct task_struct, se);
+}
+
 /* Walk up scheduling entities hierarchy */
 #define for_each_sched_entity(se) \
                 for (; se; se = se->parent)
@@ -186,7 +189,12 @@ find_matching_se(struct sched_entity **se, struct sched_entity **pse)
         }
 }
 
-#else   /* CONFIG_FAIR_GROUP_SCHED */
+#else   /* !CONFIG_FAIR_GROUP_SCHED */
+
+static inline struct task_struct *task_of(struct sched_entity *se)
+{
+        return container_of(se, struct task_struct, se);
+}
 
 static inline struct rq *rq_of(struct cfs_rq *cfs_rq)
 {
@@ -266,6 +274,12 @@ static inline u64 min_vruntime(u64 min_vruntime, u64 vruntime)
         return min_vruntime;
 }
 
+static inline int entity_before(struct sched_entity *a,
+                                struct sched_entity *b)
+{
+        return (s64)(a->vruntime - b->vruntime) < 0;
+}
+
 static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
         return se->vruntime - cfs_rq->min_vruntime;
@@ -430,12 +444,13 @@ static u64 sched_slice(struct cfs_rq *cfs_rq, struct sched_entity *se)
 
         for_each_sched_entity(se) {
                 struct load_weight *load;
+                struct load_weight lw;
 
                 cfs_rq = cfs_rq_of(se);
                 load = &cfs_rq->load;
 
                 if (unlikely(!se->on_rq)) {
-                        struct load_weight lw = cfs_rq->load;
+                        lw = cfs_rq->load;
 
                         update_load_add(&lw, se->load.weight);
                         load = &lw;
@@ -530,6 +545,12 @@ update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se)
         schedstat_set(se->wait_count, se->wait_count + 1);
         schedstat_set(se->wait_sum, se->wait_sum +
                         rq_of(cfs_rq)->clock - se->wait_start);
+#ifdef CONFIG_SCHEDSTATS
+        if (entity_is_task(se)) {
+                trace_sched_stat_wait(task_of(se),
+                        rq_of(cfs_rq)->clock - se->wait_start);
+        }
+#endif
         schedstat_set(se->wait_start, 0);
 }
 
@@ -604,9 +625,13 @@ account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
 static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
 #ifdef CONFIG_SCHEDSTATS
+        struct task_struct *tsk = NULL;
+
+        if (entity_is_task(se))
+                tsk = task_of(se);
+
         if (se->sleep_start) {
                 u64 delta = rq_of(cfs_rq)->clock - se->sleep_start;
-                struct task_struct *tsk = task_of(se);
 
                 if ((s64)delta < 0)
                         delta = 0;
@@ -617,11 +642,13 @@ static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)
                 se->sleep_start = 0;
                 se->sum_sleep_runtime += delta;
 
-                account_scheduler_latency(tsk, delta >> 10, 1);
+                if (tsk) {
+                        account_scheduler_latency(tsk, delta >> 10, 1);
+                        trace_sched_stat_sleep(tsk, delta);
+                }
         }
         if (se->block_start) {
                 u64 delta = rq_of(cfs_rq)->clock - se->block_start;
-                struct task_struct *tsk = task_of(se);
 
                 if ((s64)delta < 0)
                         delta = 0;
@@ -632,17 +659,25 @@ static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)
                 se->block_start = 0;
                 se->sum_sleep_runtime += delta;
 
-                /*
-                 * Blocking time is in units of nanosecs, so shift by 20 to
-                 * get a milliseconds-range estimation of the amount of
-                 * time that the task spent sleeping:
-                 */
-                if (unlikely(prof_on == SLEEP_PROFILING)) {
+                if (tsk) {
+                        if (tsk->in_iowait) {
+                                se->iowait_sum += delta;
+                                se->iowait_count++;
+                                trace_sched_stat_iowait(tsk, delta);
+                        }
 
-                        profile_hits(SLEEP_PROFILING, (void *)get_wchan(tsk),
-                                     delta >> 20);
+                        /*
+                         * Blocking time is in units of nanosecs, so shift by
+                         * 20 to get a milliseconds-range estimation of the
+                         * amount of time that the task spent sleeping:
+                         */
+                        if (unlikely(prof_on == SLEEP_PROFILING)) {
+                                profile_hits(SLEEP_PROFILING,
+                                                (void *)get_wchan(tsk),
+                                                delta >> 20);
+                        }
+                        account_scheduler_latency(tsk, delta >> 10, 0);
                 }
-                account_scheduler_latency(tsk, delta >> 10, 0);
         }
 #endif
 }
@@ -676,7 +711,7 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial)
 
         if (!initial) {
                 /* sleeps upto a single latency don't count. */
-                if (sched_feat(NEW_FAIR_SLEEPERS)) {
+                if (sched_feat(FAIR_SLEEPERS)) {
                         unsigned long thresh = sysctl_sched_latency;
 
                         /*
@@ -686,16 +721,24 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial)
                          * all of which have the same weight.
                          */
                         if (sched_feat(NORMALIZED_SLEEPER) &&
-                                        task_of(se)->policy != SCHED_IDLE)
+                                        (!entity_is_task(se) ||
+                                         task_of(se)->policy != SCHED_IDLE))
                                 thresh = calc_delta_fair(thresh, se);
 
+                        /*
+                         * Halve their sleep time's effect, to allow
+                         * for a gentler effect of sleepers:
+                         */
+                        if (sched_feat(GENTLE_FAIR_SLEEPERS))
+                                thresh >>= 1;
+
                         vruntime -= thresh;
                 }
-
-                /* ensure we never gain time by being placed backwards. */
-                vruntime = max_vruntime(se->vruntime, vruntime);
         }
 
+        /* ensure we never gain time by being placed backwards. */
+        vruntime = max_vruntime(se->vruntime, vruntime);
+
         se->vruntime = vruntime;
 }
 
@@ -721,10 +764,10 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int wakeup)
 
 static void __clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
-        if (cfs_rq->last == se)
+        if (!se || cfs_rq->last == se)
                 cfs_rq->last = NULL;
 
-        if (cfs_rq->next == se)
+        if (!se || cfs_rq->next == se)
                 cfs_rq->next = NULL;
 }
 
@@ -1015,7 +1058,7 @@ static void yield_task_fair(struct rq *rq)
         /*
          * Already in the rightmost position?
          */
-        if (unlikely(!rightmost || rightmost->vruntime < se->vruntime))
+        if (unlikely(!rightmost || entity_before(rightmost, se)))
                 return;
 
         /*
@@ -1026,79 +1069,6 @@ static void yield_task_fair(struct rq *rq)
         se->vruntime = rightmost->vruntime + 1;
 }
 
-/*
- * wake_idle() will wake a task on an idle cpu if task->cpu is
- * not idle and an idle cpu is available.  The span of cpus to
- * search starts with cpus closest then further out as needed,
- * so we always favor a closer, idle cpu.
- * Domains may include CPUs that are not usable for migration,
- * hence we need to mask them out (cpu_active_mask)
- *
- * Returns the CPU we should wake onto.
- */
-#if defined(ARCH_HAS_SCHED_WAKE_IDLE)
-static int wake_idle(int cpu, struct task_struct *p)
-{
-        struct sched_domain *sd;
-        int i;
-        unsigned int chosen_wakeup_cpu;
-        int this_cpu;
-
-        /*
-         * At POWERSAVINGS_BALANCE_WAKEUP level, if both this_cpu and prev_cpu
-         * are idle and this is not a kernel thread and this task's affinity
-         * allows it to be moved to preferred cpu, then just move!
-         */
-
-        this_cpu = smp_processor_id();
-        chosen_wakeup_cpu =
-                cpu_rq(this_cpu)->rd->sched_mc_preferred_wakeup_cpu;
-
-        if (sched_mc_power_savings >= POWERSAVINGS_BALANCE_WAKEUP &&
-                idle_cpu(cpu) && idle_cpu(this_cpu) &&
-                p->mm && !(p->flags & PF_KTHREAD) &&
-                cpu_isset(chosen_wakeup_cpu, p->cpus_allowed))
-                return chosen_wakeup_cpu;
-
-        /*
-         * If it is idle, then it is the best cpu to run this task.
-         *
-         * This cpu is also the best, if it has more than one task already.
-         * Siblings must be also busy(in most cases) as they didn't already
-         * pickup the extra load from this cpu and hence we need not check
-         * sibling runqueue info. This will avoid the checks and cache miss
-         * penalities associated with that.
-         */
-        if (idle_cpu(cpu) || cpu_rq(cpu)->cfs.nr_running > 1)
-                return cpu;
-
-        for_each_domain(cpu, sd) {
-                if ((sd->flags & SD_WAKE_IDLE)
-                    || ((sd->flags & SD_WAKE_IDLE_FAR)
-                        && !task_hot(p, task_rq(p)->clock, sd))) {
-                        for_each_cpu_and(i, sched_domain_span(sd),
-                                         &p->cpus_allowed) {
-                                if (cpu_active(i) && idle_cpu(i)) {
-                                        if (i != task_cpu(p)) {
-                                                schedstat_inc(p,
-                                                       se.nr_wakeups_idle);
-                                        }
-                                        return i;
-                                }
-                        }
-                } else {
-                        break;
-                }
-        }
-        return cpu;
-}
-#else /* !ARCH_HAS_SCHED_WAKE_IDLE*/
-static inline int wake_idle(int cpu, struct task_struct *p)
-{
-        return cpu;
-}
-#endif
-
 #ifdef CONFIG_SMP
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
@@ -1185,25 +1155,34 @@ static inline unsigned long effective_load(struct task_group *tg, int cpu,
 
 #endif
 
-static int
-wake_affine(struct sched_domain *this_sd, struct rq *this_rq,
-            struct task_struct *p, int prev_cpu, int this_cpu, int sync,
-            int idx, unsigned long load, unsigned long this_load,
-            unsigned int imbalance)
+static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync)
 {
-        struct task_struct *curr = this_rq->curr;
-        struct task_group *tg;
-        unsigned long tl = this_load;
+        struct task_struct *curr = current;
+        unsigned long this_load, load;
+        int idx, this_cpu, prev_cpu;
         unsigned long tl_per_task;
+        unsigned int imbalance;
+        struct task_group *tg;
         unsigned long weight;
         int balanced;
 
-        if (!(this_sd->flags & SD_WAKE_AFFINE) || !sched_feat(AFFINE_WAKEUPS))
-                return 0;
+        idx       = sd->wake_idx;
+        this_cpu  = smp_processor_id();
+        prev_cpu  = task_cpu(p);
+        load      = source_load(prev_cpu, idx);
+        this_load = target_load(this_cpu, idx);
 
-        if (sync && (curr->se.avg_overlap > sysctl_sched_migration_cost ||
-                        p->se.avg_overlap > sysctl_sched_migration_cost))
-                sync = 0;
+        if (sync) {
+               if (sched_feat(SYNC_LESS) &&
+                   (curr->se.avg_overlap > sysctl_sched_migration_cost ||
+                    p->se.avg_overlap > sysctl_sched_migration_cost))
+                       sync = 0;
+        } else {
+                if (sched_feat(SYNC_MORE) &&
+                    (curr->se.avg_overlap < sysctl_sched_migration_cost &&
+                     p->se.avg_overlap < sysctl_sched_migration_cost))
+                        sync = 1;
+        }
 
         /*
          * If sync wakeup then subtract the (maximum possible)
@@ -1214,14 +1193,26 @@ wake_affine(struct sched_domain *this_sd, struct rq *this_rq,
                 tg = task_group(current);
                 weight = current->se.load.weight;
 
-                tl += effective_load(tg, this_cpu, -weight, -weight);
+                this_load += effective_load(tg, this_cpu, -weight, -weight);
                 load += effective_load(tg, prev_cpu, 0, -weight);
         }
 
         tg = task_group(p);
         weight = p->se.load.weight;
 
-        balanced = 100*(tl + effective_load(tg, this_cpu, weight, weight)) <=
+        imbalance = 100 + (sd->imbalance_pct - 100) / 2;
+
+        /*
+         * In low-load situations, where prev_cpu is idle and this_cpu is idle
+         * due to the sync cause above having dropped this_load to 0, we'll
+         * always have an imbalance, but there's really nothing you can do
+         * about that, so that's good too.
+         *
+         * Otherwise check if either cpus are near enough in load to allow this
+         * task to be woken on this_cpu.
+         */
+        balanced = !this_load ||
+                100*(this_load + effective_load(tg, this_cpu, weight, weight)) <=
                 imbalance*(load + effective_load(tg, prev_cpu, 0, weight));
 
         /*
@@ -1235,14 +1226,15 @@ wake_affine(struct sched_domain *this_sd, struct rq *this_rq,
         schedstat_inc(p, se.nr_wakeups_affine_attempts);
         tl_per_task = cpu_avg_load_per_task(this_cpu);
 
-        if (balanced || (tl <= load && tl + target_load(prev_cpu, idx) <=
-                        tl_per_task)) {
+        if (balanced ||
+            (this_load <= load &&
+             this_load + target_load(prev_cpu, idx) <= tl_per_task)) {
                 /*
                  * This domain has SD_WAKE_AFFINE and
                  * p is cache cold in this domain, and
                  * there is no bad imbalance.
                  */
-                schedstat_inc(this_sd, ttwu_move_affine);
+                schedstat_inc(sd, ttwu_move_affine);
                 schedstat_inc(p, se.nr_wakeups_affine);
 
                 return 1;
@@ -1250,67 +1242,215 @@ wake_affine(struct sched_domain *this_sd, struct rq *this_rq,
         return 0;
 }
 
-static int select_task_rq_fair(struct task_struct *p, int sync)
+/*
+ * find_idlest_group finds and returns the least busy CPU group within the
+ * domain.
+ */
+static struct sched_group *
+find_idlest_group(struct sched_domain *sd, struct task_struct *p,
+                  int this_cpu, int load_idx)
 {
-        struct sched_domain *sd, *this_sd = NULL;
-        int prev_cpu, this_cpu, new_cpu;
-        unsigned long load, this_load;
-        struct rq *this_rq;
-        unsigned int imbalance;
-        int idx;
+        struct sched_group *idlest = NULL, *this = NULL, *group = sd->groups;
+        unsigned long min_load = ULONG_MAX, this_load = 0;
+        int imbalance = 100 + (sd->imbalance_pct-100)/2;
 
-        prev_cpu        = task_cpu(p);
-        this_cpu        = smp_processor_id();
-        this_rq         = cpu_rq(this_cpu);
-        new_cpu         = prev_cpu;
+        do {
+                unsigned long load, avg_load;
+                int local_group;
+                int i;
 
-        if (prev_cpu == this_cpu)
-                goto out;
-        /*
-         * 'this_sd' is the first domain that both
-         * this_cpu and prev_cpu are present in:
-         */
-        for_each_domain(this_cpu, sd) {
-                if (cpumask_test_cpu(prev_cpu, sched_domain_span(sd))) {
-                        this_sd = sd;
-                        break;
+                /* Skip over this group if it has no CPUs allowed */
+                if (!cpumask_intersects(sched_group_cpus(group),
+                                        &p->cpus_allowed))
+                        continue;
+
+                local_group = cpumask_test_cpu(this_cpu,
+                                               sched_group_cpus(group));
+
+                /* Tally up the load of all CPUs in the group */
+                avg_load = 0;
+
+                for_each_cpu(i, sched_group_cpus(group)) {
+                        /* Bias balancing toward cpus of our domain */
+                        if (local_group)
+                                load = source_load(i, load_idx);
+                        else
+                                load = target_load(i, load_idx);
+
+                        avg_load += load;
+                }
+
+                /* Adjust by relative CPU power of the group */
+                avg_load = (avg_load * SCHED_LOAD_SCALE) / group->cpu_power;
+
+                if (local_group) {
+                        this_load = avg_load;
+                        this = group;
+                } else if (avg_load < min_load) {
+                        min_load = avg_load;
+                        idlest = group;
+                }
+        } while (group = group->next, group != sd->groups);
+
+        if (!idlest || 100*this_load < imbalance*min_load)
+                return NULL;
+        return idlest;
+}
+
+/*
+ * find_idlest_cpu - find the idlest cpu among the cpus in group.
+ */
+static int
+find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
+{
+        unsigned long load, min_load = ULONG_MAX;
+        int idlest = -1;
+        int i;
+
+        /* Traverse only the allowed CPUs */
+        for_each_cpu_and(i, sched_group_cpus(group), &p->cpus_allowed) {
+                load = weighted_cpuload(i);
+
+                if (load < min_load || (load == min_load && i == this_cpu)) {
+                        min_load = load;
+                        idlest = i;
                 }
         }
 
-        if (unlikely(!cpumask_test_cpu(this_cpu, &p->cpus_allowed)))
-                goto out;
+        return idlest;
+}
 
-        /*
-         * Check for affine wakeup and passive balancing possibilities.
-         */
-        if (!this_sd)
+/*
+ * sched_balance_self: balance the current task (running on cpu) in domains
+ * that have the 'flag' flag set. In practice, this is SD_BALANCE_FORK and
+ * SD_BALANCE_EXEC.
+ *
+ * Balance, ie. select the least loaded group.
+ *
+ * Returns the target CPU number, or the same CPU if no balancing is needed.
+ *
+ * preempt must be disabled.
+ */
+static int select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flags)
+{
+        struct sched_domain *tmp, *affine_sd = NULL, *sd = NULL;
+        int cpu = smp_processor_id();
+        int prev_cpu = task_cpu(p);
+        int new_cpu = cpu;
+        int want_affine = 0;
+        int want_sd = 1;
+        int sync = wake_flags & WF_SYNC;
+
+        if (sd_flag & SD_BALANCE_WAKE) {
+                if (sched_feat(AFFINE_WAKEUPS))
+                        want_affine = 1;
+                new_cpu = prev_cpu;
+        }
+
+        rcu_read_lock();
+        for_each_domain(cpu, tmp) {
+                /*
+                 * If power savings logic is enabled for a domain, see if we
+                 * are not overloaded, if so, don't balance wider.
+                 */
+                if (tmp->flags & (SD_POWERSAVINGS_BALANCE|SD_PREFER_LOCAL)) {
+                        unsigned long power = 0;
+                        unsigned long nr_running = 0;
+                        unsigned long capacity;
+                        int i;
+
+                        for_each_cpu(i, sched_domain_span(tmp)) {
+                                power += power_of(i);
+                                nr_running += cpu_rq(i)->cfs.nr_running;
+                        }
+
+                        capacity = DIV_ROUND_CLOSEST(power, SCHED_LOAD_SCALE);
+
+                        if (tmp->flags & SD_POWERSAVINGS_BALANCE)
+                                nr_running /= 2;
+
+                        if (nr_running < capacity)
+                                want_sd = 0;
+                }
+
+                if (want_affine && (tmp->flags & SD_WAKE_AFFINE) &&
+                    cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) {
+
+                        affine_sd = tmp;
+                        want_affine = 0;
+                }
+
+                if (!want_sd && !want_affine)
+                        break;
+
+                if (!(tmp->flags & sd_flag))
+                        continue;
+
+                if (want_sd)
+                        sd = tmp;
+        }
+
+        if (sched_feat(LB_SHARES_UPDATE)) {
+                /*
+                 * Pick the largest domain to update shares over
+                 */
+                tmp = sd;
+                if (affine_sd && (!tmp ||
+                                  cpumask_weight(sched_domain_span(affine_sd)) >
+                                  cpumask_weight(sched_domain_span(sd))))
+                        tmp = affine_sd;
+
+                if (tmp)
+                        update_shares(tmp);
+        }
+
+        if (affine_sd && wake_affine(affine_sd, p, sync)) {
+                new_cpu = cpu;
                 goto out;
+        }
 
-        idx = this_sd->wake_idx;
+        while (sd) {
+                int load_idx = sd->forkexec_idx;
+                struct sched_group *group;
+                int weight;
 
-        imbalance = 100 + (this_sd->imbalance_pct - 100) / 2;
+                if (!(sd->flags & sd_flag)) {
+                        sd = sd->child;
+                        continue;
+                }
 
-        load = source_load(prev_cpu, idx);
-        this_load = target_load(this_cpu, idx);
+                if (sd_flag & SD_BALANCE_WAKE)
+                        load_idx = sd->wake_idx;
 
-        if (wake_affine(this_sd, this_rq, p, prev_cpu, this_cpu, sync, idx,
-                                     load, this_load, imbalance))
-                return this_cpu;
+                group = find_idlest_group(sd, p, cpu, load_idx);
+                if (!group) {
+                        sd = sd->child;
+                        continue;
+                }
 
-        /*
-         * Start passive balancing when half the imbalance_pct
-         * limit is reached.
-         */
-        if (this_sd->flags & SD_WAKE_BALANCE) {
-                if (imbalance*this_load <= 100*load) {
-                        schedstat_inc(this_sd, ttwu_move_balance);
-                        schedstat_inc(p, se.nr_wakeups_passive);
-                        return this_cpu;
+                new_cpu = find_idlest_cpu(group, p, cpu);
+                if (new_cpu == -1 || new_cpu == cpu) {
+                        /* Now try balancing at a lower domain level of cpu */
+                        sd = sd->child;
+                        continue;
                 }
+
+                /* Now try balancing at a lower domain level of new_cpu */
+                cpu = new_cpu;
+                weight = cpumask_weight(sched_domain_span(sd));
+                sd = NULL;
+                for_each_domain(cpu, tmp) {
+                        if (weight <= cpumask_weight(sched_domain_span(tmp)))
+                                break;
+                        if (tmp->flags & sd_flag)
+                                sd = tmp;
+                }
+                /* while loop will break here if sd == NULL */
         }
 
 out:
-        return wake_idle(new_cpu, p);
+        rcu_read_unlock();
+        return new_cpu;
 }
 #endif /* CONFIG_SMP */
 
@@ -1423,11 +1563,12 @@ static void set_next_buddy(struct sched_entity *se)
 /*
  * Preempt the current task with a newly woken task if needed:
  */
-static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int sync)
+static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_flags)
 {
         struct task_struct *curr = rq->curr;
         struct sched_entity *se = &curr->se, *pse = &p->se;
         struct cfs_rq *cfs_rq = task_cfs_rq(curr);
+        int sync = wake_flags & WF_SYNC;
 
         update_curr(cfs_rq);
 
@@ -1453,7 +1594,8 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int sync)
          */
         if (sched_feat(LAST_BUDDY) && likely(se->on_rq && curr != rq->idle))
                 set_last_buddy(se);
-        set_next_buddy(pse);
+        if (sched_feat(NEXT_BUDDY) && !(wake_flags & WF_FORK))
+                set_next_buddy(pse);
 
         /*
          * We can come here with TIF_NEED_RESCHED already set from new task
@@ -1475,16 +1617,25 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int sync)
                 return;
         }
 
-        if (!sched_feat(WAKEUP_PREEMPT))
-                return;
-
-        if (sched_feat(WAKEUP_OVERLAP) && (sync ||
-                        (se->avg_overlap < sysctl_sched_migration_cost &&
-                         pse->avg_overlap < sysctl_sched_migration_cost))) {
+        if ((sched_feat(WAKEUP_SYNC) && sync) ||
+            (sched_feat(WAKEUP_OVERLAP) &&
+             (se->avg_overlap < sysctl_sched_migration_cost &&
+              pse->avg_overlap < sysctl_sched_migration_cost))) {
                 resched_task(curr);
                 return;
         }
 
+        if (sched_feat(WAKEUP_RUNNING)) {
+                if (pse->avg_running < se->avg_running) {
+                        set_next_buddy(pse);
+                        resched_task(curr);
+                        return;
+                }
+        }
+
+        if (!sched_feat(WAKEUP_PREEMPT))
+                return;
+
         find_matching_se(&se, &pse);
 
         BUG_ON(!pse);
@@ -1507,8 +1658,13 @@ static struct task_struct *pick_next_task_fair(struct rq *rq)
                 /*
                  * If se was a buddy, clear it so that it will have to earn
                  * the favour again.
+                 *
+                 * If se was not a buddy, clear the buddies because neither
+                 * was elegible to run, let them earn it again.
+                 *
+                 * IOW. unconditionally clear buddies.
                  */
-                __clear_buddies(cfs_rq, se);
+                __clear_buddies(cfs_rq, NULL);
                 set_next_entity(cfs_rq, se);
                 cfs_rq = group_cfs_rq(se);
         } while (cfs_rq);
@@ -1707,11 +1863,13 @@ static void task_new_fair(struct rq *rq, struct task_struct *p)
         sched_info_queued(p);
 
         update_curr(cfs_rq);
+        if (curr)
+                se->vruntime = curr->vruntime;
         place_entity(cfs_rq, se, 1);
 
         /* 'curr' will be NULL if the child belongs to a different group */
         if (sysctl_sched_child_runs_first && this_cpu == task_cpu(p) &&
-                        curr && curr->vruntime < se->vruntime) {
+                        curr && entity_before(curr, se)) {
                 /*
                  * Upon rescheduling, sched_class::put_prev_task() will place
                  * 'current' within the tree based on its new key value.
diff --git a/kernel/sched_features.h b/kernel/sched_features.h
index 4569bfa7df9b..0d94083582c7 100644
--- a/kernel/sched_features.h
+++ b/kernel/sched_features.h
@@ -1,17 +1,123 @@
-SCHED_FEAT(NEW_FAIR_SLEEPERS, 1)
+/*
+ * Disregards a certain amount of sleep time (sched_latency_ns) and
+ * considers the task to be running during that period. This gives it
+ * a service deficit on wakeup, allowing it to run sooner.
+ */
+SCHED_FEAT(FAIR_SLEEPERS, 1)
+
+/*
+ * Only give sleepers 50% of their service deficit. This allows
+ * them to run sooner, but does not allow tons of sleepers to
+ * rip the spread apart.
+ */
+SCHED_FEAT(GENTLE_FAIR_SLEEPERS, 1)
+
+/*
+ * By not normalizing the sleep time, heavy tasks get an effective
+ * longer period, and lighter task an effective shorter period they
+ * are considered running.
+ */
 SCHED_FEAT(NORMALIZED_SLEEPER, 0)
-SCHED_FEAT(ADAPTIVE_GRAN, 1)
-SCHED_FEAT(WAKEUP_PREEMPT, 1)
+
+/*
+ * Place new tasks ahead so that they do not starve already running
+ * tasks
+ */
 SCHED_FEAT(START_DEBIT, 1)
+
+/*
+ * Should wakeups try to preempt running tasks.
+ */
+SCHED_FEAT(WAKEUP_PREEMPT, 1)
+
+/*
+ * Compute wakeup_gran based on task behaviour, clipped to
+ *  [0, sched_wakeup_gran_ns]
+ */
+SCHED_FEAT(ADAPTIVE_GRAN, 1)
+
+/*
+ * When converting the wakeup granularity to virtual time, do it such
+ * that heavier tasks preempting a lighter task have an edge.
+ */
+SCHED_FEAT(ASYM_GRAN, 1)
+
+/*
+ * Always wakeup-preempt SYNC wakeups, see SYNC_WAKEUPS.
+ */
+SCHED_FEAT(WAKEUP_SYNC, 0)
+
+/*
+ * Wakeup preempt based on task behaviour. Tasks that do not overlap
+ * don't get preempted.
+ */
+SCHED_FEAT(WAKEUP_OVERLAP, 0)
+
+/*
+ * Wakeup preemption towards tasks that run short
+ */
+SCHED_FEAT(WAKEUP_RUNNING, 0)
+
+/*
+ * Use the SYNC wakeup hint, pipes and the likes use this to indicate
+ * the remote end is likely to consume the data we just wrote, and
+ * therefore has cache benefit from being placed on the same cpu, see
+ * also AFFINE_WAKEUPS.
+ */
+SCHED_FEAT(SYNC_WAKEUPS, 1)
+
+/*
+ * Based on load and program behaviour, see if it makes sense to place
+ * a newly woken task on the same cpu as the task that woke it --
+ * improve cache locality. Typically used with SYNC wakeups as
+ * generated by pipes and the like, see also SYNC_WAKEUPS.
+ */
 SCHED_FEAT(AFFINE_WAKEUPS, 1)
+
+/*
+ * Weaken SYNC hint based on overlap
+ */
+SCHED_FEAT(SYNC_LESS, 1)
+
+/*
+ * Add SYNC hint based on overlap
+ */
+SCHED_FEAT(SYNC_MORE, 0)
+
+/*
+ * Prefer to schedule the task we woke last (assuming it failed
+ * wakeup-preemption), since its likely going to consume data we
+ * touched, increases cache locality.
+ */
+SCHED_FEAT(NEXT_BUDDY, 0)
+
+/*
+ * Prefer to schedule the task that ran last (when we did
+ * wake-preempt) as that likely will touch the same data, increases
+ * cache locality.
+ */
+SCHED_FEAT(LAST_BUDDY, 1)
+
+/*
+ * Consider buddies to be cache hot, decreases the likelyness of a
+ * cache buddy being migrated away, increases cache locality.
+ */
 SCHED_FEAT(CACHE_HOT_BUDDY, 1)
-SCHED_FEAT(SYNC_WAKEUPS, 1)
+
+/*
+ * Use arch dependent cpu power functions
+ */
+SCHED_FEAT(ARCH_POWER, 0)
+
 SCHED_FEAT(HRTICK, 0)
 SCHED_FEAT(DOUBLE_TICK, 0)
-SCHED_FEAT(ASYM_GRAN, 1)
 SCHED_FEAT(LB_BIAS, 1)
-SCHED_FEAT(LB_WAKEUP_UPDATE, 1)
+SCHED_FEAT(LB_SHARES_UPDATE, 1)
 SCHED_FEAT(ASYM_EFF_LOAD, 1)
-SCHED_FEAT(WAKEUP_OVERLAP, 0)
-SCHED_FEAT(LAST_BUDDY, 1)
+
+/*
+ * Spin-wait on mutex acquisition when the mutex owner is running on
+ * another cpu -- assumes that when the owner is running, it will soon
+ * release the lock. Decreases scheduling overhead.
+ */
 SCHED_FEAT(OWNER_SPIN, 1)
diff --git a/kernel/sched_idletask.c b/kernel/sched_idletask.c
index 499672c10cbd..a8b448af004b 100644
--- a/kernel/sched_idletask.c
+++ b/kernel/sched_idletask.c
@@ -6,7 +6,7 @@
  */
 
 #ifdef CONFIG_SMP
-static int select_task_rq_idle(struct task_struct *p, int sync)
+static int select_task_rq_idle(struct task_struct *p, int sd_flag, int flags)
 {
         return task_cpu(p); /* IDLE tasks as never migrated */
 }
@@ -14,7 +14,7 @@ static int select_task_rq_idle(struct task_struct *p, int sync)
 /*
  * Idle tasks are unconditionally rescheduled:
  */
-static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int sync)
+static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int flags)
 {
         resched_task(rq->idle);
 }
diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c
index 9bf0d2a73045..13de7126a6ab 100644
--- a/kernel/sched_rt.c
+++ b/kernel/sched_rt.c
@@ -3,13 +3,18 @@
  * policies)
  */
 
+#ifdef CONFIG_RT_GROUP_SCHED
+
+#define rt_entity_is_task(rt_se) (!(rt_se)->my_q)
+
 static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se)
 {
+#ifdef CONFIG_SCHED_DEBUG
+        WARN_ON_ONCE(!rt_entity_is_task(rt_se));
+#endif
         return container_of(rt_se, struct task_struct, rt);
 }
 
-#ifdef CONFIG_RT_GROUP_SCHED
-
 static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)
 {
         return rt_rq->rq;
@@ -22,6 +27,13 @@ static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)
 
 #else /* CONFIG_RT_GROUP_SCHED */
 
+#define rt_entity_is_task(rt_se) (1)
+
+static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se)
+{
+        return container_of(rt_se, struct task_struct, rt);
+}
+
 static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)
 {
         return container_of(rt_rq, struct rq, rt);
@@ -73,7 +85,7 @@ static inline void rt_clear_overload(struct rq *rq)
 
 static void update_rt_migration(struct rt_rq *rt_rq)
 {
-        if (rt_rq->rt_nr_migratory && (rt_rq->rt_nr_running > 1)) {
+        if (rt_rq->rt_nr_migratory && rt_rq->rt_nr_total > 1) {
                 if (!rt_rq->overloaded) {
                         rt_set_overload(rq_of_rt_rq(rt_rq));
                         rt_rq->overloaded = 1;
@@ -86,6 +98,12 @@ static void update_rt_migration(struct rt_rq *rt_rq)
 
 static void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
 {
+        if (!rt_entity_is_task(rt_se))
+                return;
+
+        rt_rq = &rq_of_rt_rq(rt_rq)->rt;
+
+        rt_rq->rt_nr_total++;
         if (rt_se->nr_cpus_allowed > 1)
                 rt_rq->rt_nr_migratory++;
 
@@ -94,6 +112,12 @@ static void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
 
 static void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
 {
+        if (!rt_entity_is_task(rt_se))
+                return;
+
+        rt_rq = &rq_of_rt_rq(rt_rq)->rt;
+
+        rt_rq->rt_nr_total--;
         if (rt_se->nr_cpus_allowed > 1)
                 rt_rq->rt_nr_migratory--;
 
@@ -112,6 +136,11 @@ static void dequeue_pushable_task(struct rq *rq, struct task_struct *p)
         plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);
 }
 
+static inline int has_pushable_tasks(struct rq *rq)
+{
+        return !plist_head_empty(&rq->rt.pushable_tasks);
+}
+
 #else
 
 static inline void enqueue_pushable_task(struct rq *rq, struct task_struct *p)
@@ -586,6 +615,8 @@ static void update_curr_rt(struct rq *rq)
         curr->se.exec_start = rq->clock;
         cpuacct_charge(curr, delta_exec);
 
+        sched_rt_avg_update(rq, delta_exec);
+
         if (!rt_bandwidth_enabled())
                 return;
 
@@ -858,8 +889,6 @@ static void enqueue_task_rt(struct rq *rq, struct task_struct *p, int wakeup)
 
         if (!task_current(rq, p) && p->rt.nr_cpus_allowed > 1)
                 enqueue_pushable_task(rq, p);
-
-        inc_cpu_load(rq, p->se.load.weight);
 }
 
 static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int sleep)
@@ -870,8 +899,6 @@ static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int sleep)
         dequeue_rt_entity(rt_se);
 
         dequeue_pushable_task(rq, p);
-
-        dec_cpu_load(rq, p->se.load.weight);
 }
 
 /*
@@ -911,10 +938,13 @@ static void yield_task_rt(struct rq *rq)
 #ifdef CONFIG_SMP
 static int find_lowest_rq(struct task_struct *task);
 
-static int select_task_rq_rt(struct task_struct *p, int sync)
+static int select_task_rq_rt(struct task_struct *p, int sd_flag, int flags)
 {
         struct rq *rq = task_rq(p);
 
+        if (sd_flag != SD_BALANCE_WAKE)
+                return smp_processor_id();
+
         /*
          * If the current task is an RT task, then
          * try to see if we can wake this RT task up on another
@@ -972,7 +1002,7 @@ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)
 /*
  * Preempt the current task with a newly woken task if needed:
  */
-static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p, int sync)
+static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p, int flags)
 {
         if (p->prio < rq->curr->prio) {
                 resched_task(rq->curr);
@@ -1048,6 +1078,14 @@ static struct task_struct *pick_next_task_rt(struct rq *rq)
         if (p)
                 dequeue_pushable_task(rq, p);
 
+#ifdef CONFIG_SMP
+        /*
+         * We detect this state here so that we can avoid taking the RQ
+         * lock again later if there is no need to push
+         */
+        rq->post_schedule = has_pushable_tasks(rq);
+#endif
+
         return p;
 }
 
@@ -1146,13 +1184,6 @@ static int find_lowest_rq(struct task_struct *task)
                 return -1; /* No targets found */
 
         /*
-         * Only consider CPUs that are usable for migration.
-         * I guess we might want to change cpupri_find() to ignore those
-         * in the first place.
-         */
-        cpumask_and(lowest_mask, lowest_mask, cpu_active_mask);
-
-        /*
          * At this point we have built a mask of cpus representing the
          * lowest priority tasks in the system.  Now we want to elect
          * the best one based on our affinity and topology.
@@ -1246,11 +1277,6 @@ static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq)
         return lowest_rq;
 }
 
-static inline int has_pushable_tasks(struct rq *rq)
-{
-        return !plist_head_empty(&rq->rt.pushable_tasks);
-}
-
 static struct task_struct *pick_next_pushable_task(struct rq *rq)
 {
         struct task_struct *p;
@@ -1450,23 +1476,9 @@ static void pre_schedule_rt(struct rq *rq, struct task_struct *prev)
                 pull_rt_task(rq);
 }
 
-/*
- * assumes rq->lock is held
- */
-static int needs_post_schedule_rt(struct rq *rq)
-{
-        return has_pushable_tasks(rq);
-}
-
 static void post_schedule_rt(struct rq *rq)
 {
-        /*
-         * This is only called if needs_post_schedule_rt() indicates that
-         * we need to push tasks away
-         */
-        spin_lock_irq(&rq->lock);
         push_rt_tasks(rq);
-        spin_unlock_irq(&rq->lock);
 }
 
 /*
@@ -1742,7 +1754,6 @@ static const struct sched_class rt_sched_class = {
         .rq_online              = rq_online_rt,
         .rq_offline             = rq_offline_rt,
         .pre_schedule           = pre_schedule_rt,
-        .needs_post_schedule    = needs_post_schedule_rt,
         .post_schedule          = post_schedule_rt,
         .task_wake_up           = task_wake_up_rt,
         .switched_from          = switched_from_rt,
diff --git a/kernel/signal.c b/kernel/signal.c
index 809a228019ad..64c5deeaca5d 100644
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -832,6 +832,7 @@ static int __send_signal(int sig, struct siginfo *info, struct task_struct *t,
 {
         struct sigpending *pending;
         struct sigqueue *q;
+        int override_rlimit;
 
         trace_sched_signal_send(sig, t);
 
@@ -863,9 +864,13 @@ static int __send_signal(int sig, struct siginfo *info, struct task_struct *t,
            make sure at least one signal gets delivered and don't
            pass on the info struct.  */
 
-        q = __sigqueue_alloc(t, GFP_ATOMIC, (sig < SIGRTMIN &&
-                                             (is_si_special(info) ||
-                                              info->si_code >= 0)));
+        if (sig < SIGRTMIN)
+                override_rlimit = (is_si_special(info) || info->si_code >= 0);
+        else
+                override_rlimit = 0;
+
+        q = __sigqueue_alloc(t, GFP_ATOMIC | __GFP_NOTRACK_FALSE_POSITIVE,
+                override_rlimit);
         if (q) {
                 list_add_tail(&q->list, &pending->list);
                 switch ((unsigned long) info) {
@@ -1405,7 +1410,7 @@ int do_notify_parent(struct task_struct *tsk, int sig)
         /* do_notify_parent_cldstop should have been called instead.  */
         BUG_ON(task_is_stopped_or_traced(tsk));
 
-        BUG_ON(!tsk->ptrace &&
+        BUG_ON(!task_ptrace(tsk) &&
                (tsk->group_leader != tsk || !thread_group_empty(tsk)));
 
         info.si_signo = sig;
@@ -1444,7 +1449,7 @@ int do_notify_parent(struct task_struct *tsk, int sig)
 
         psig = tsk->parent->sighand;
         spin_lock_irqsave(&psig->siglock, flags);
-        if (!tsk->ptrace && sig == SIGCHLD &&
+        if (!task_ptrace(tsk) && sig == SIGCHLD &&
             (psig->action[SIGCHLD-1].sa.sa_handler == SIG_IGN ||
              (psig->action[SIGCHLD-1].sa.sa_flags & SA_NOCLDWAIT))) {
                 /*
@@ -1481,7 +1486,7 @@ static void do_notify_parent_cldstop(struct task_struct *tsk, int why)
         struct task_struct *parent;
         struct sighand_struct *sighand;
 
-        if (tsk->ptrace & PT_PTRACED)
+        if (task_ptrace(tsk))
                 parent = tsk->parent;
         else {
                 tsk = tsk->group_leader;
@@ -1494,7 +1499,7 @@ static void do_notify_parent_cldstop(struct task_struct *tsk, int why)
          * see comment in do_notify_parent() abot the following 3 lines
          */
         rcu_read_lock();
-        info.si_pid = task_pid_nr_ns(tsk, tsk->parent->nsproxy->pid_ns);
+        info.si_pid = task_pid_nr_ns(tsk, parent->nsproxy->pid_ns);
         info.si_uid = __task_cred(tsk)->uid;
         rcu_read_unlock();
 
@@ -1530,7 +1535,7 @@ static void do_notify_parent_cldstop(struct task_struct *tsk, int why)
 
 static inline int may_ptrace_stop(void)
 {
-        if (!likely(current->ptrace & PT_PTRACED))
+        if (!likely(task_ptrace(current)))
                 return 0;
         /*
          * Are we in the middle of do_coredump?
@@ -1748,7 +1753,7 @@ static int do_signal_stop(int signr)
 static int ptrace_signal(int signr, siginfo_t *info,
                          struct pt_regs *regs, void *cookie)
 {
-        if (!(current->ptrace & PT_PTRACED))
+        if (!task_ptrace(current))
                 return signr;
 
         ptrace_signal_deliver(regs, cookie);
@@ -2449,11 +2454,9 @@ do_sigaltstack (const stack_t __user *uss, stack_t __user *uoss, unsigned long s
         stack_t oss;
         int error;
 
-        if (uoss) {
-                oss.ss_sp = (void __user *) current->sas_ss_sp;
-                oss.ss_size = current->sas_ss_size;
-                oss.ss_flags = sas_ss_flags(sp);
-        }
+        oss.ss_sp = (void __user *) current->sas_ss_sp;
+        oss.ss_size = current->sas_ss_size;
+        oss.ss_flags = sas_ss_flags(sp);
 
         if (uss) {
                 void __user *ss_sp;
@@ -2461,10 +2464,12 @@ do_sigaltstack (const stack_t __user *uss, stack_t __user *uoss, unsigned long s
                 int ss_flags;
 
                 error = -EFAULT;
-                if (!access_ok(VERIFY_READ, uss, sizeof(*uss))
-                    || __get_user(ss_sp, &uss->ss_sp)
-                    || __get_user(ss_flags, &uss->ss_flags)
-                    || __get_user(ss_size, &uss->ss_size))
+                if (!access_ok(VERIFY_READ, uss, sizeof(*uss)))
+                        goto out;
+                error = __get_user(ss_sp, &uss->ss_sp) |
+                        __get_user(ss_flags, &uss->ss_flags) |
+                        __get_user(ss_size, &uss->ss_size);
+                if (error)
                         goto out;
 
                 error = -EPERM;
@@ -2496,13 +2501,16 @@ do_sigaltstack (const stack_t __user *uss, stack_t __user *uoss, unsigned long s
                 current->sas_ss_size = ss_size;
         }
 
+        error = 0;
         if (uoss) {
                 error = -EFAULT;
-                if (copy_to_user(uoss, &oss, sizeof(oss)))
+                if (!access_ok(VERIFY_WRITE, uoss, sizeof(*uoss)))
                         goto out;
+                error = __put_user(oss.ss_sp, &uoss->ss_sp) |
+                        __put_user(oss.ss_size, &uoss->ss_size) |
+                        __put_user(oss.ss_flags, &uoss->ss_flags);
         }
 
-        error = 0;
 out:
         return error;
 }
diff --git a/kernel/slow-work.c b/kernel/slow-work.c
index 521ed2004d63..09d7519557d3 100644
--- a/kernel/slow-work.c
+++ b/kernel/slow-work.c
@@ -319,6 +319,15 @@ cant_get_ref:
 EXPORT_SYMBOL(slow_work_enqueue);
 
 /*
+ * Schedule a cull of the thread pool at some time in the near future
+ */
+static void slow_work_schedule_cull(void)
+{
+        mod_timer(&slow_work_cull_timer,
+                  round_jiffies(jiffies + SLOW_WORK_CULL_TIMEOUT));
+}
+
+/*
  * Worker thread culling algorithm
  */
 static bool slow_work_cull_thread(void)
@@ -335,8 +344,7 @@ static bool slow_work_cull_thread(void)
                     list_empty(&vslow_work_queue) &&
                     atomic_read(&slow_work_thread_count) >
                     slow_work_min_threads) {
-                        mod_timer(&slow_work_cull_timer,
-                                  jiffies + SLOW_WORK_CULL_TIMEOUT);
+                        slow_work_schedule_cull();
                         do_cull = true;
                 }
         }
@@ -393,8 +401,7 @@ static int slow_work_thread(void *_data)
                             list_empty(&vslow_work_queue) &&
                             atomic_read(&slow_work_thread_count) >
                             slow_work_min_threads)
-                                mod_timer(&slow_work_cull_timer,
-                                          jiffies + SLOW_WORK_CULL_TIMEOUT);
+                                slow_work_schedule_cull();
                         continue;
                 }
 
@@ -458,7 +465,7 @@ static void slow_work_new_thread_execute(struct slow_work *work)
                 if (atomic_dec_and_test(&slow_work_thread_count))
                         BUG(); /* we're running on a slow work thread... */
                 mod_timer(&slow_work_oom_timer,
-                          jiffies + SLOW_WORK_OOM_TIMEOUT);
+                          round_jiffies(jiffies + SLOW_WORK_OOM_TIMEOUT));
         } else {
                 /* ratelimit the starting of new threads */
                 mod_timer(&slow_work_oom_timer, jiffies + 1);
@@ -502,8 +509,7 @@ static int slow_work_min_threads_sysctl(struct ctl_table *table, int write,
                         if (n < 0 && !slow_work_may_not_start_new_thread)
                                 slow_work_enqueue(&slow_work_new_thread);
                         else if (n > 0)
-                                mod_timer(&slow_work_cull_timer,
-                                          jiffies + SLOW_WORK_CULL_TIMEOUT);
+                                slow_work_schedule_cull();
                 }
                 mutex_unlock(&slow_work_user_lock);
         }
@@ -529,8 +535,7 @@ static int slow_work_max_threads_sysctl(struct ctl_table *table, int write,
                                 atomic_read(&slow_work_thread_count);
 
                         if (n < 0)
-                                mod_timer(&slow_work_cull_timer,
-                                          jiffies + SLOW_WORK_CULL_TIMEOUT);
+                                slow_work_schedule_cull();
                 }
                 mutex_unlock(&slow_work_user_lock);
         }
diff --git a/kernel/smp.c b/kernel/smp.c
index ad63d8501207..8e218500ab14 100644
--- a/kernel/smp.c
+++ b/kernel/smp.c
@@ -57,7 +57,7 @@ hotplug_cfd(struct notifier_block *nfb, unsigned long action, void *hcpu)
                         return NOTIFY_BAD;
                 break;
 
-#ifdef CONFIG_CPU_HOTPLUG
+#ifdef CONFIG_HOTPLUG_CPU
         case CPU_UP_CANCELED:
         case CPU_UP_CANCELED_FROZEN:
 
@@ -177,6 +177,11 @@ void generic_smp_call_function_interrupt(void)
         int cpu = get_cpu();
 
         /*
+         * Shouldn't receive this interrupt on a cpu that is not yet online.
+         */
+        WARN_ON_ONCE(!cpu_online(cpu));
+
+        /*
          * Ensure entry is visible on call_function_queue after we have
          * entered the IPI. See comment in smp_call_function_many.
          * If we don't have this, then we may miss an entry on the list
@@ -230,6 +235,11 @@ void generic_smp_call_function_single_interrupt(void)
         unsigned int data_flags;
         LIST_HEAD(list);
 
+        /*
+         * Shouldn't receive this interrupt on a cpu that is not yet online.
+         */
+        WARN_ON_ONCE(!cpu_online(smp_processor_id()));
+
         spin_lock(&q->lock);
         list_replace_init(&q->list, &list);
         spin_unlock(&q->lock);
@@ -285,8 +295,14 @@ int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
          */
         this_cpu = get_cpu();
 
-        /* Can deadlock when called with interrupts disabled */
-        WARN_ON_ONCE(irqs_disabled() && !oops_in_progress);
+        /*
+         * Can deadlock when called with interrupts disabled.
+         * We allow cpu's that are not yet online though, as no one else can
+         * send smp call function interrupt to this cpu and as such deadlocks
+         * can't happen.
+         */
+        WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled()
+                     && !oops_in_progress);
 
         if (cpu == this_cpu) {
                 local_irq_save(flags);
@@ -329,8 +345,14 @@ void __smp_call_function_single(int cpu, struct call_single_data *data,
 {
         csd_lock(data);
 
-        /* Can deadlock when called with interrupts disabled */
-        WARN_ON_ONCE(wait && irqs_disabled() && !oops_in_progress);
+        /*
+         * Can deadlock when called with interrupts disabled.
+         * We allow cpu's that are not yet online though, as no one else can
+         * send smp call function interrupt to this cpu and as such deadlocks
+         * can't happen.
+         */
+        WARN_ON_ONCE(cpu_online(smp_processor_id()) && wait && irqs_disabled()
+                     && !oops_in_progress);
 
         generic_exec_single(cpu, data, wait);
 }
@@ -365,8 +387,14 @@ void smp_call_function_many(const struct cpumask *mask,
         unsigned long flags;
         int cpu, next_cpu, this_cpu = smp_processor_id();
 
-        /* Can deadlock when called with interrupts disabled */
-        WARN_ON_ONCE(irqs_disabled() && !oops_in_progress);
+        /*
+         * Can deadlock when called with interrupts disabled.
+         * We allow cpu's that are not yet online though, as no one else can
+         * send smp call function interrupt to this cpu and as such deadlocks
+         * can't happen.
+         */
+        WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled()
+                     && !oops_in_progress);
 
         /* So, what's a CPU they want? Ignoring this one. */
         cpu = cpumask_first_and(mask, cpu_online_mask);
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 258885a543db..f8749e5216e0 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -57,7 +57,7 @@ static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp
 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
 
 char *softirq_to_name[NR_SOFTIRQS] = {
-        "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK",
+        "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
         "TASKLET", "SCHED", "HRTIMER",  "RCU"
 };
 
@@ -213,6 +213,7 @@ restart:
         do {
                 if (pending & 1) {
                         int prev_count = preempt_count();
+                        kstat_incr_softirqs_this_cpu(h - softirq_vec);
 
                         trace_softirq_entry(h, softirq_vec);
                         h->action(h);
@@ -226,7 +227,7 @@ restart:
                                 preempt_count() = prev_count;
                         }
 
-                        rcu_bh_qsctr_inc(cpu);
+                        rcu_bh_qs(cpu);
                 }
                 h++;
                 pending >>= 1;
@@ -344,7 +345,9 @@ void open_softirq(int nr, void (*action)(struct softirq_action *))
         softirq_vec[nr].action = action;
 }
 
-/* Tasklets */
+/*
+ * Tasklets
+ */
 struct tasklet_head
 {
         struct tasklet_struct *head;
@@ -382,6 +385,17 @@ void __tasklet_hi_schedule(struct tasklet_struct *t)
 
 EXPORT_SYMBOL(__tasklet_hi_schedule);
 
+void __tasklet_hi_schedule_first(struct tasklet_struct *t)
+{
+        BUG_ON(!irqs_disabled());
+
+        t->next = __get_cpu_var(tasklet_hi_vec).head;
+        __get_cpu_var(tasklet_hi_vec).head = t;
+        __raise_softirq_irqoff(HI_SOFTIRQ);
+}
+
+EXPORT_SYMBOL(__tasklet_hi_schedule_first);
+
 static void tasklet_action(struct softirq_action *a)
 {
         struct tasklet_struct *list;
@@ -481,6 +495,66 @@ void tasklet_kill(struct tasklet_struct *t)
 
 EXPORT_SYMBOL(tasklet_kill);
 
+/*
+ * tasklet_hrtimer
+ */
+
+/*
+ * The trampoline is called when the hrtimer expires. If this is
+ * called from the hrtimer interrupt then we schedule the tasklet as
+ * the timer callback function expects to run in softirq context. If
+ * it's called in softirq context anyway (i.e. high resolution timers
+ * disabled) then the hrtimer callback is called right away.
+ */
+static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
+{
+        struct tasklet_hrtimer *ttimer =
+                container_of(timer, struct tasklet_hrtimer, timer);
+
+        if (hrtimer_is_hres_active(timer)) {
+                tasklet_hi_schedule(&ttimer->tasklet);
+                return HRTIMER_NORESTART;
+        }
+        return ttimer->function(timer);
+}
+
+/*
+ * Helper function which calls the hrtimer callback from
+ * tasklet/softirq context
+ */
+static void __tasklet_hrtimer_trampoline(unsigned long data)
+{
+        struct tasklet_hrtimer *ttimer = (void *)data;
+        enum hrtimer_restart restart;
+
+        restart = ttimer->function(&ttimer->timer);
+        if (restart != HRTIMER_NORESTART)
+                hrtimer_restart(&ttimer->timer);
+}
+
+/**
+ * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
+ * @ttimer:      tasklet_hrtimer which is initialized
+ * @function:    hrtimer callback funtion which gets called from softirq context
+ * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
+ * @mode:        hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
+ */
+void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
+                          enum hrtimer_restart (*function)(struct hrtimer *),
+                          clockid_t which_clock, enum hrtimer_mode mode)
+{
+        hrtimer_init(&ttimer->timer, which_clock, mode);
+        ttimer->timer.function = __hrtimer_tasklet_trampoline;
+        tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
+                     (unsigned long)ttimer);
+        ttimer->function = function;
+}
+EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
+
+/*
+ * Remote softirq bits
+ */
+
 DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
 EXPORT_PER_CPU_SYMBOL(softirq_work_list);
 
@@ -647,7 +721,7 @@ static int ksoftirqd(void * __bind_cpu)
                         preempt_enable_no_resched();
                         cond_resched();
                         preempt_disable();
-                        rcu_qsctr_inc((long)__bind_cpu);
+                        rcu_sched_qs((long)__bind_cpu);
                 }
                 preempt_enable();
                 set_current_state(TASK_INTERRUPTIBLE);
diff --git a/kernel/spinlock.c b/kernel/spinlock.c
index 7932653c4ebd..5ddab730cb2f 100644
--- a/kernel/spinlock.c
+++ b/kernel/spinlock.c
@@ -21,44 +21,29 @@
 #include <linux/debug_locks.h>
 #include <linux/module.h>
 
+#ifndef _spin_trylock
 int __lockfunc _spin_trylock(spinlock_t *lock)
 {
-        preempt_disable();
-        if (_raw_spin_trylock(lock)) {
-                spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
-                return 1;
-        }
-        
-        preempt_enable();
-        return 0;
+        return __spin_trylock(lock);
 }
 EXPORT_SYMBOL(_spin_trylock);
+#endif
 
+#ifndef _read_trylock
 int __lockfunc _read_trylock(rwlock_t *lock)
 {
-        preempt_disable();
-        if (_raw_read_trylock(lock)) {
-                rwlock_acquire_read(&lock->dep_map, 0, 1, _RET_IP_);
-                return 1;
-        }
-
-        preempt_enable();
-        return 0;
+        return __read_trylock(lock);
 }
 EXPORT_SYMBOL(_read_trylock);
+#endif
 
+#ifndef _write_trylock
 int __lockfunc _write_trylock(rwlock_t *lock)
 {
-        preempt_disable();
-        if (_raw_write_trylock(lock)) {
-                rwlock_acquire(&lock->dep_map, 0, 1, _RET_IP_);
-                return 1;
-        }
-
-        preempt_enable();
-        return 0;
+        return __write_trylock(lock);
 }
 EXPORT_SYMBOL(_write_trylock);
+#endif
 
 /*
  * If lockdep is enabled then we use the non-preemption spin-ops
@@ -67,132 +52,101 @@ EXPORT_SYMBOL(_write_trylock);
  */
 #if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)
 
+#ifndef _read_lock
 void __lockfunc _read_lock(rwlock_t *lock)
 {
-        preempt_disable();
-        rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
-        LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
+        __read_lock(lock);
 }
 EXPORT_SYMBOL(_read_lock);
+#endif
 
+#ifndef _spin_lock_irqsave
 unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
 {
-        unsigned long flags;
-
-        local_irq_save(flags);
-        preempt_disable();
-        spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
-        /*
-         * On lockdep we dont want the hand-coded irq-enable of
-         * _raw_spin_lock_flags() code, because lockdep assumes
-         * that interrupts are not re-enabled during lock-acquire:
-         */
-#ifdef CONFIG_LOCKDEP
-        LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
-#else
-        _raw_spin_lock_flags(lock, &flags);
-#endif
-        return flags;
+        return __spin_lock_irqsave(lock);
 }
 EXPORT_SYMBOL(_spin_lock_irqsave);
+#endif
 
+#ifndef _spin_lock_irq
 void __lockfunc _spin_lock_irq(spinlock_t *lock)
 {
-        local_irq_disable();
-        preempt_disable();
-        spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
-        LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
+        __spin_lock_irq(lock);
 }
 EXPORT_SYMBOL(_spin_lock_irq);
+#endif
 
+#ifndef _spin_lock_bh
 void __lockfunc _spin_lock_bh(spinlock_t *lock)
 {
-        local_bh_disable();
-        preempt_disable();
-        spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
-        LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
+        __spin_lock_bh(lock);
 }
 EXPORT_SYMBOL(_spin_lock_bh);
+#endif
 
+#ifndef _read_lock_irqsave
 unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)
 {
-        unsigned long flags;
-
-        local_irq_save(flags);
-        preempt_disable();
-        rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
-        LOCK_CONTENDED_FLAGS(lock, _raw_read_trylock, _raw_read_lock,
-                             _raw_read_lock_flags, &flags);
-        return flags;
+        return __read_lock_irqsave(lock);
 }
 EXPORT_SYMBOL(_read_lock_irqsave);
+#endif
 
+#ifndef _read_lock_irq
 void __lockfunc _read_lock_irq(rwlock_t *lock)
 {
-        local_irq_disable();
-        preempt_disable();
-        rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
-        LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
+        __read_lock_irq(lock);
 }
 EXPORT_SYMBOL(_read_lock_irq);
+#endif
 
+#ifndef _read_lock_bh
 void __lockfunc _read_lock_bh(rwlock_t *lock)
 {
-        local_bh_disable();
-        preempt_disable();
-        rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
-        LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
+        __read_lock_bh(lock);
 }
 EXPORT_SYMBOL(_read_lock_bh);
+#endif
 
+#ifndef _write_lock_irqsave
 unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)
 {
-        unsigned long flags;
-
-        local_irq_save(flags);
-        preempt_disable();
-        rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
-        LOCK_CONTENDED_FLAGS(lock, _raw_write_trylock, _raw_write_lock,
-                             _raw_write_lock_flags, &flags);
-        return flags;
+        return __write_lock_irqsave(lock);
 }
 EXPORT_SYMBOL(_write_lock_irqsave);
+#endif
 
+#ifndef _write_lock_irq
 void __lockfunc _write_lock_irq(rwlock_t *lock)
 {
-        local_irq_disable();
-        preempt_disable();
-        rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
-        LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
+        __write_lock_irq(lock);
 }
 EXPORT_SYMBOL(_write_lock_irq);
+#endif
 
+#ifndef _write_lock_bh
 void __lockfunc _write_lock_bh(rwlock_t *lock)
 {
-        local_bh_disable();
-        preempt_disable();
-        rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
-        LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
+        __write_lock_bh(lock);
 }
 EXPORT_SYMBOL(_write_lock_bh);
+#endif
 
+#ifndef _spin_lock
 void __lockfunc _spin_lock(spinlock_t *lock)
 {
-        preempt_disable();
-        spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
-        LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
+        __spin_lock(lock);
 }
-
 EXPORT_SYMBOL(_spin_lock);
+#endif
 
+#ifndef _write_lock
 void __lockfunc _write_lock(rwlock_t *lock)
 {
-        preempt_disable();
-        rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
-        LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
+        __write_lock(lock);
 }
-
 EXPORT_SYMBOL(_write_lock);
+#endif
 
 #else /* CONFIG_PREEMPT: */
 
@@ -318,125 +272,109 @@ EXPORT_SYMBOL(_spin_lock_nest_lock);
 
 #endif
 
+#ifndef _spin_unlock
 void __lockfunc _spin_unlock(spinlock_t *lock)
 {
-        spin_release(&lock->dep_map, 1, _RET_IP_);
-        _raw_spin_unlock(lock);
-        preempt_enable();
+        __spin_unlock(lock);
 }
 EXPORT_SYMBOL(_spin_unlock);
+#endif
 
+#ifndef _write_unlock
 void __lockfunc _write_unlock(rwlock_t *lock)
 {
-        rwlock_release(&lock->dep_map, 1, _RET_IP_);
-        _raw_write_unlock(lock);
-        preempt_enable();
+        __write_unlock(lock);
 }
 EXPORT_SYMBOL(_write_unlock);
+#endif
 
+#ifndef _read_unlock
 void __lockfunc _read_unlock(rwlock_t *lock)
 {
-        rwlock_release(&lock->dep_map, 1, _RET_IP_);
-        _raw_read_unlock(lock);
-        preempt_enable();
+        __read_unlock(lock);
 }
 EXPORT_SYMBOL(_read_unlock);
+#endif
 
+#ifndef _spin_unlock_irqrestore
 void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
 {
-        spin_release(&lock->dep_map, 1, _RET_IP_);
-        _raw_spin_unlock(lock);
-        local_irq_restore(flags);
-        preempt_enable();
+        __spin_unlock_irqrestore(lock, flags);
 }
 EXPORT_SYMBOL(_spin_unlock_irqrestore);
+#endif
 
+#ifndef _spin_unlock_irq
 void __lockfunc _spin_unlock_irq(spinlock_t *lock)
 {
-        spin_release(&lock->dep_map, 1, _RET_IP_);
-        _raw_spin_unlock(lock);
-        local_irq_enable();
-        preempt_enable();
+        __spin_unlock_irq(lock);
 }
 EXPORT_SYMBOL(_spin_unlock_irq);
+#endif
 
+#ifndef _spin_unlock_bh
 void __lockfunc _spin_unlock_bh(spinlock_t *lock)
 {
-        spin_release(&lock->dep_map, 1, _RET_IP_);
-        _raw_spin_unlock(lock);
-        preempt_enable_no_resched();
-        local_bh_enable_ip((unsigned long)__builtin_return_address(0));
+        __spin_unlock_bh(lock);
 }
 EXPORT_SYMBOL(_spin_unlock_bh);
+#endif
 
+#ifndef _read_unlock_irqrestore
 void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
 {
-        rwlock_release(&lock->dep_map, 1, _RET_IP_);
-        _raw_read_unlock(lock);
-        local_irq_restore(flags);
-        preempt_enable();
+        __read_unlock_irqrestore(lock, flags);
 }
 EXPORT_SYMBOL(_read_unlock_irqrestore);
+#endif
 
+#ifndef _read_unlock_irq
 void __lockfunc _read_unlock_irq(rwlock_t *lock)
 {
-        rwlock_release(&lock->dep_map, 1, _RET_IP_);
-        _raw_read_unlock(lock);
-        local_irq_enable();
-        preempt_enable();
+        __read_unlock_irq(lock);
 }
 EXPORT_SYMBOL(_read_unlock_irq);
+#endif
 
+#ifndef _read_unlock_bh
 void __lockfunc _read_unlock_bh(rwlock_t *lock)
 {
-        rwlock_release(&lock->dep_map, 1, _RET_IP_);
-        _raw_read_unlock(lock);
-        preempt_enable_no_resched();
-        local_bh_enable_ip((unsigned long)__builtin_return_address(0));
+        __read_unlock_bh(lock);
 }
 EXPORT_SYMBOL(_read_unlock_bh);
+#endif
 
+#ifndef _write_unlock_irqrestore
 void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
 {
-        rwlock_release(&lock->dep_map, 1, _RET_IP_);
-        _raw_write_unlock(lock);
-        local_irq_restore(flags);
-        preempt_enable();
+        __write_unlock_irqrestore(lock, flags);
 }
 EXPORT_SYMBOL(_write_unlock_irqrestore);
+#endif
 
+#ifndef _write_unlock_irq
 void __lockfunc _write_unlock_irq(rwlock_t *lock)
 {
-        rwlock_release(&lock->dep_map, 1, _RET_IP_);
-        _raw_write_unlock(lock);
-        local_irq_enable();
-        preempt_enable();
+        __write_unlock_irq(lock);
 }
 EXPORT_SYMBOL(_write_unlock_irq);
+#endif
 
+#ifndef _write_unlock_bh
 void __lockfunc _write_unlock_bh(rwlock_t *lock)
 {
-        rwlock_release(&lock->dep_map, 1, _RET_IP_);
-        _raw_write_unlock(lock);
-        preempt_enable_no_resched();
-        local_bh_enable_ip((unsigned long)__builtin_return_address(0));
+        __write_unlock_bh(lock);
 }
 EXPORT_SYMBOL(_write_unlock_bh);
+#endif
 
+#ifndef _spin_trylock_bh
 int __lockfunc _spin_trylock_bh(spinlock_t *lock)
 {
-        local_bh_disable();
-        preempt_disable();
-        if (_raw_spin_trylock(lock)) {
-                spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
-                return 1;
-        }
-
-        preempt_enable_no_resched();
-        local_bh_enable_ip((unsigned long)__builtin_return_address(0));
-        return 0;
+        return __spin_trylock_bh(lock);
 }
 EXPORT_SYMBOL(_spin_trylock_bh);
+#endif
 
 notrace int in_lock_functions(unsigned long addr)
 {
diff --git a/kernel/sys.c b/kernel/sys.c
index 438d99a38c87..b3f1097c76fa 100644
--- a/kernel/sys.c
+++ b/kernel/sys.c
@@ -1113,289 +1113,6 @@ out:
         return err;
 }
 
-/*
- * Supplementary group IDs
- */
-
-/* init to 2 - one for init_task, one to ensure it is never freed */
-struct group_info init_groups = { .usage = ATOMIC_INIT(2) };
-
-struct group_info *groups_alloc(int gidsetsize)
-{
-        struct group_info *group_info;
-        int nblocks;
-        int i;
-
-        nblocks = (gidsetsize + NGROUPS_PER_BLOCK - 1) / NGROUPS_PER_BLOCK;
-        /* Make sure we always allocate at least one indirect block pointer */
-        nblocks = nblocks ? : 1;
-        group_info = kmalloc(sizeof(*group_info) + nblocks*sizeof(gid_t *), GFP_USER);
-        if (!group_info)
-                return NULL;
-        group_info->ngroups = gidsetsize;
-        group_info->nblocks = nblocks;
-        atomic_set(&group_info->usage, 1);
-
-        if (gidsetsize <= NGROUPS_SMALL)
-                group_info->blocks[0] = group_info->small_block;
-        else {
-                for (i = 0; i < nblocks; i++) {
-                        gid_t *b;
-                        b = (void *)__get_free_page(GFP_USER);
-                        if (!b)
-                                goto out_undo_partial_alloc;
-                        group_info->blocks[i] = b;
-                }
-        }
-        return group_info;
-
-out_undo_partial_alloc:
-        while (--i >= 0) {
-                free_page((unsigned long)group_info->blocks[i]);
-        }
-        kfree(group_info);
-        return NULL;
-}
-
-EXPORT_SYMBOL(groups_alloc);
-
-void groups_free(struct group_info *group_info)
-{
-        if (group_info->blocks[0] != group_info->small_block) {
-                int i;
-                for (i = 0; i < group_info->nblocks; i++)
-                        free_page((unsigned long)group_info->blocks[i]);
-        }
-        kfree(group_info);
-}
-
-EXPORT_SYMBOL(groups_free);
-
-/* export the group_info to a user-space array */
-static int groups_to_user(gid_t __user *grouplist,
-                          const struct group_info *group_info)
-{
-        int i;
-        unsigned int count = group_info->ngroups;
-
-        for (i = 0; i < group_info->nblocks; i++) {
-                unsigned int cp_count = min(NGROUPS_PER_BLOCK, count);
-                unsigned int len = cp_count * sizeof(*grouplist);
-
-                if (copy_to_user(grouplist, group_info->blocks[i], len))
-                        return -EFAULT;
-
-                grouplist += NGROUPS_PER_BLOCK;
-                count -= cp_count;
-        }
-        return 0;
-}
-
-/* fill a group_info from a user-space array - it must be allocated already */
-static int groups_from_user(struct group_info *group_info,
-    gid_t __user *grouplist)
-{
-        int i;
-        unsigned int count = group_info->ngroups;
-
-        for (i = 0; i < group_info->nblocks; i++) {
-                unsigned int cp_count = min(NGROUPS_PER_BLOCK, count);
-                unsigned int len = cp_count * sizeof(*grouplist);
-
-                if (copy_from_user(group_info->blocks[i], grouplist, len))
-                        return -EFAULT;
-
-                grouplist += NGROUPS_PER_BLOCK;
-                count -= cp_count;
-        }
-        return 0;
-}
-
-/* a simple Shell sort */
-static void groups_sort(struct group_info *group_info)
-{
-        int base, max, stride;
-        int gidsetsize = group_info->ngroups;
-
-        for (stride = 1; stride < gidsetsize; stride = 3 * stride + 1)
-                ; /* nothing */
-        stride /= 3;
-
-        while (stride) {
-                max = gidsetsize - stride;
-                for (base = 0; base < max; base++) {
-                        int left = base;
-                        int right = left + stride;
-                        gid_t tmp = GROUP_AT(group_info, right);
-
-                        while (left >= 0 && GROUP_AT(group_info, left) > tmp) {
-                                GROUP_AT(group_info, right) =
-                                    GROUP_AT(group_info, left);
-                                right = left;
-                                left -= stride;
-                        }
-                        GROUP_AT(group_info, right) = tmp;
-                }
-                stride /= 3;
-        }
-}
-
-/* a simple bsearch */
-int groups_search(const struct group_info *group_info, gid_t grp)
-{
-        unsigned int left, right;
-
-        if (!group_info)
-                return 0;
-
-        left = 0;
-        right = group_info->ngroups;
-        while (left < right) {
-                unsigned int mid = (left+right)/2;
-                int cmp = grp - GROUP_AT(group_info, mid);
-                if (cmp > 0)
-                        left = mid + 1;
-                else if (cmp < 0)
-                        right = mid;
-                else
-                        return 1;
-        }
-        return 0;
-}
-
-/**
- * set_groups - Change a group subscription in a set of credentials
- * @new: The newly prepared set of credentials to alter
- * @group_info: The group list to install
- *
- * Validate a group subscription and, if valid, insert it into a set
- * of credentials.
- */
-int set_groups(struct cred *new, struct group_info *group_info)
-{
-        int retval;
-
-        retval = security_task_setgroups(group_info);
-        if (retval)
-                return retval;
-
-        put_group_info(new->group_info);
-        groups_sort(group_info);
-        get_group_info(group_info);
-        new->group_info = group_info;
-        return 0;
-}
-
-EXPORT_SYMBOL(set_groups);
-
-/**
- * set_current_groups - Change current's group subscription
- * @group_info: The group list to impose
- *
- * Validate a group subscription and, if valid, impose it upon current's task
- * security record.
- */
-int set_current_groups(struct group_info *group_info)
-{
-        struct cred *new;
-        int ret;
-
-        new = prepare_creds();
-        if (!new)
-                return -ENOMEM;
-
-        ret = set_groups(new, group_info);
-        if (ret < 0) {
-                abort_creds(new);
-                return ret;
-        }
-
-        return commit_creds(new);
-}
-
-EXPORT_SYMBOL(set_current_groups);
-
-SYSCALL_DEFINE2(getgroups, int, gidsetsize, gid_t __user *, grouplist)
-{
-        const struct cred *cred = current_cred();
-        int i;
-
-        if (gidsetsize < 0)
-                return -EINVAL;
-
-        /* no need to grab task_lock here; it cannot change */
-        i = cred->group_info->ngroups;
-        if (gidsetsize) {
-                if (i > gidsetsize) {
-                        i = -EINVAL;
-                        goto out;
-                }
-                if (groups_to_user(grouplist, cred->group_info)) {
-                        i = -EFAULT;
-                        goto out;
-                }
-        }
-out:
-        return i;
-}
-
-/*
- *      SMP: Our groups are copy-on-write. We can set them safely
- *      without another task interfering.
- */
- 
-SYSCALL_DEFINE2(setgroups, int, gidsetsize, gid_t __user *, grouplist)
-{
-        struct group_info *group_info;
-        int retval;
-
-        if (!capable(CAP_SETGID))
-                return -EPERM;
-        if ((unsigned)gidsetsize > NGROUPS_MAX)
-                return -EINVAL;
-
-        group_info = groups_alloc(gidsetsize);
-        if (!group_info)
-                return -ENOMEM;
-        retval = groups_from_user(group_info, grouplist);
-        if (retval) {
-                put_group_info(group_info);
-                return retval;
-        }
-
-        retval = set_current_groups(group_info);
-        put_group_info(group_info);
-
-        return retval;
-}
-
-/*
- * Check whether we're fsgid/egid or in the supplemental group..
- */
-int in_group_p(gid_t grp)
-{
-        const struct cred *cred = current_cred();
-        int retval = 1;
-
-        if (grp != cred->fsgid)
-                retval = groups_search(cred->group_info, grp);
-        return retval;
-}
-
-EXPORT_SYMBOL(in_group_p);
-
-int in_egroup_p(gid_t grp)
-{
-        const struct cred *cred = current_cred();
-        int retval = 1;
-
-        if (grp != cred->egid)
-                retval = groups_search(cred->group_info, grp);
-        return retval;
-}
-
-EXPORT_SYMBOL(in_egroup_p);
-
 DECLARE_RWSEM(uts_sem);
 
 SYSCALL_DEFINE1(newuname, struct new_utsname __user *, name)
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index 0e51a35a4486..1a631ba684a4 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -27,6 +27,7 @@
 #include <linux/security.h>
 #include <linux/ctype.h>
 #include <linux/utsname.h>
+#include <linux/kmemcheck.h>
 #include <linux/smp_lock.h>
 #include <linux/fs.h>
 #include <linux/init.h>
@@ -90,6 +91,9 @@ extern int sysctl_nr_trim_pages;
 #ifdef CONFIG_RCU_TORTURE_TEST
 extern int rcutorture_runnable;
 #endif /* #ifdef CONFIG_RCU_TORTURE_TEST */
+#ifdef CONFIG_BLOCK
+extern int blk_iopoll_enabled;
+#endif
 
 /* Constants used for minimum and  maximum */
 #ifdef CONFIG_DETECT_SOFTLOCKUP
@@ -244,6 +248,14 @@ static int max_wakeup_granularity_ns = NSEC_PER_SEC;	/* 1 second */
 #endif
 
 static struct ctl_table kern_table[] = {
+        {
+                .ctl_name       = CTL_UNNUMBERED,
+                .procname       = "sched_child_runs_first",
+                .data           = &sysctl_sched_child_runs_first,
+                .maxlen         = sizeof(unsigned int),
+                .mode           = 0644,
+                .proc_handler   = &proc_dointvec,
+        },
 #ifdef CONFIG_SCHED_DEBUG
         {
                 .ctl_name       = CTL_UNNUMBERED,
@@ -298,14 +310,6 @@ static struct ctl_table kern_table[] = {
         },
         {
                 .ctl_name       = CTL_UNNUMBERED,
-                .procname       = "sched_child_runs_first",
-                .data           = &sysctl_sched_child_runs_first,
-                .maxlen         = sizeof(unsigned int),
-                .mode           = 0644,
-                .proc_handler   = &proc_dointvec,
-        },
-        {
-                .ctl_name       = CTL_UNNUMBERED,
                 .procname       = "sched_features",
                 .data           = &sysctl_sched_features,
                 .maxlen         = sizeof(unsigned int),
@@ -330,11 +334,22 @@ static struct ctl_table kern_table[] = {
         },
         {
                 .ctl_name       = CTL_UNNUMBERED,
+                .procname       = "sched_time_avg",
+                .data           = &sysctl_sched_time_avg,
+                .maxlen         = sizeof(unsigned int),
+                .mode           = 0644,
+                .proc_handler   = &proc_dointvec,
+        },
+        {
+                .ctl_name       = CTL_UNNUMBERED,
                 .procname       = "timer_migration",
                 .data           = &sysctl_timer_migration,
                 .maxlen         = sizeof(unsigned int),
                 .mode           = 0644,
-                .proc_handler   = &proc_dointvec,
+                .proc_handler   = &proc_dointvec_minmax,
+                .strategy       = &sysctl_intvec,
+                .extra1         = &zero,
+                .extra2         = &one,
         },
 #endif
         {
@@ -743,6 +758,14 @@ static struct ctl_table kern_table[] = {
                 .proc_handler   = &proc_dointvec,
         },
         {
+                .ctl_name       = CTL_UNNUMBERED,
+                .procname       = "panic_on_io_nmi",
+                .data           = &panic_on_io_nmi,
+                .maxlen         = sizeof(int),
+                .mode           = 0644,
+                .proc_handler   = &proc_dointvec,
+        },
+        {
                 .ctl_name       = KERN_BOOTLOADER_TYPE,
                 .procname       = "bootloader_type",
                 .data           = &bootloader_type,
@@ -967,6 +990,26 @@ static struct ctl_table kern_table[] = {
                 .proc_handler   = &proc_dointvec,
         },
 #endif
+#ifdef CONFIG_KMEMCHECK
+        {
+                .ctl_name       = CTL_UNNUMBERED,
+                .procname       = "kmemcheck",
+                .data           = &kmemcheck_enabled,
+                .maxlen         = sizeof(int),
+                .mode           = 0644,
+                .proc_handler   = &proc_dointvec,
+        },
+#endif
+#ifdef CONFIG_BLOCK
+        {
+                .ctl_name       = CTL_UNNUMBERED,
+                .procname       = "blk_iopoll",
+                .data           = &blk_iopoll_enabled,
+                .maxlen         = sizeof(int),
+                .mode           = 0644,
+                .proc_handler   = &proc_dointvec,
+        },
+#endif
 /*
  * NOTE: do not add new entries to this table unless you have read
  * Documentation/sysctl/ctl_unnumbered.txt
@@ -1283,10 +1326,10 @@ static struct ctl_table vm_table[] = {
         {
                 .ctl_name       = CTL_UNNUMBERED,
                 .procname       = "mmap_min_addr",
-                .data           = &mmap_min_addr,
-                .maxlen         = sizeof(unsigned long),
+                .data           = &dac_mmap_min_addr,
+                .maxlen         = sizeof(unsigned long),
                 .mode           = 0644,
-                .proc_handler   = &proc_doulongvec_minmax,
+                .proc_handler   = &mmap_min_addr_handler,
         },
 #ifdef CONFIG_NUMA
         {
@@ -1325,7 +1368,6 @@ static struct ctl_table vm_table[] = {
                 .extra2         = &one,
         },
 #endif
-#ifdef CONFIG_UNEVICTABLE_LRU
         {
                 .ctl_name       = CTL_UNNUMBERED,
                 .procname       = "scan_unevictable_pages",
@@ -1334,7 +1376,6 @@ static struct ctl_table vm_table[] = {
                 .mode           = 0644,
                 .proc_handler   = &scan_unevictable_handler,
         },
-#endif
 /*
  * NOTE: do not add new entries to this table unless you have read
  * Documentation/sysctl/ctl_unnumbered.txt
@@ -2273,7 +2314,7 @@ static int __do_proc_dointvec(void *tbl_data, struct ctl_table *table,
                   void *data)
 {
 #define TMPBUFLEN 21
-        int *i, vleft, first=1, neg, val;
+        int *i, vleft, first = 1, neg;
         unsigned long lval;
         size_t left, len;
         
@@ -2326,8 +2367,6 @@ static int __do_proc_dointvec(void *tbl_data, struct ctl_table *table,
                         len = p-buf;
                         if ((len < left) && *p && !isspace(*p))
                                 break;
-                        if (neg)
-                                val = -val;
                         s += len;
                         left -= len;
 
diff --git a/kernel/taskstats.c b/kernel/taskstats.c
index 888adbcca30c..ea8384d3caa7 100644
--- a/kernel/taskstats.c
+++ b/kernel/taskstats.c
@@ -108,7 +108,7 @@ static int prepare_reply(struct genl_info *info, u8 cmd, struct sk_buff **skbp,
 /*
  * Send taskstats data in @skb to listener with nl_pid @pid
  */
-static int send_reply(struct sk_buff *skb, pid_t pid)
+static int send_reply(struct sk_buff *skb, struct genl_info *info)
 {
         struct genlmsghdr *genlhdr = nlmsg_data(nlmsg_hdr(skb));
         void *reply = genlmsg_data(genlhdr);
@@ -120,7 +120,7 @@ static int send_reply(struct sk_buff *skb, pid_t pid)
                 return rc;
         }
 
-        return genlmsg_unicast(skb, pid);
+        return genlmsg_reply(skb, info);
 }
 
 /*
@@ -150,7 +150,7 @@ static void send_cpu_listeners(struct sk_buff *skb,
                         if (!skb_next)
                                 break;
                 }
-                rc = genlmsg_unicast(skb_cur, s->pid);
+                rc = genlmsg_unicast(&init_net, skb_cur, s->pid);
                 if (rc == -ECONNREFUSED) {
                         s->valid = 0;
                         delcount++;
@@ -418,7 +418,7 @@ static int cgroupstats_user_cmd(struct sk_buff *skb, struct genl_info *info)
                 goto err;
         }
 
-        rc = send_reply(rep_skb, info->snd_pid);
+        rc = send_reply(rep_skb, info);
 
 err:
         fput_light(file, fput_needed);
@@ -487,7 +487,7 @@ free_return_rc:
         } else
                 goto err;
 
-        return send_reply(rep_skb, info->snd_pid);
+        return send_reply(rep_skb, info);
 err:
         nlmsg_free(rep_skb);
         return rc;
diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c
index 1ad6dd461119..620b58abdc32 100644
--- a/kernel/time/clockevents.c
+++ b/kernel/time/clockevents.c
@@ -137,11 +137,12 @@ int clockevents_program_event(struct clock_event_device *dev, ktime_t expires,
  */
 int clockevents_register_notifier(struct notifier_block *nb)
 {
+        unsigned long flags;
         int ret;
 
-        spin_lock(&clockevents_lock);
+        spin_lock_irqsave(&clockevents_lock, flags);
         ret = raw_notifier_chain_register(&clockevents_chain, nb);
-        spin_unlock(&clockevents_lock);
+        spin_unlock_irqrestore(&clockevents_lock, flags);
 
         return ret;
 }
@@ -178,16 +179,18 @@ static void clockevents_notify_released(void)
  */
 void clockevents_register_device(struct clock_event_device *dev)
 {
+        unsigned long flags;
+
         BUG_ON(dev->mode != CLOCK_EVT_MODE_UNUSED);
         BUG_ON(!dev->cpumask);
 
-        spin_lock(&clockevents_lock);
+        spin_lock_irqsave(&clockevents_lock, flags);
 
         list_add(&dev->list, &clockevent_devices);
         clockevents_do_notify(CLOCK_EVT_NOTIFY_ADD, dev);
         clockevents_notify_released();
 
-        spin_unlock(&clockevents_lock);
+        spin_unlock_irqrestore(&clockevents_lock, flags);
 }
 EXPORT_SYMBOL_GPL(clockevents_register_device);
 
@@ -235,8 +238,9 @@ void clockevents_exchange_device(struct clock_event_device *old,
 void clockevents_notify(unsigned long reason, void *arg)
 {
         struct list_head *node, *tmp;
+        unsigned long flags;
 
-        spin_lock(&clockevents_lock);
+        spin_lock_irqsave(&clockevents_lock, flags);
         clockevents_do_notify(reason, arg);
 
         switch (reason) {
@@ -251,18 +255,7 @@ void clockevents_notify(unsigned long reason, void *arg)
         default:
                 break;
         }
-        spin_unlock(&clockevents_lock);
+        spin_unlock_irqrestore(&clockevents_lock, flags);
 }
 EXPORT_SYMBOL_GPL(clockevents_notify);
-
-ktime_t clockevents_get_next_event(int cpu)
-{
-        struct tick_device *td;
-        struct clock_event_device *dev;
-
-        td = &per_cpu(tick_cpu_device, cpu);
-        dev = td->evtdev;
-
-        return dev->next_event;
-}
 #endif
diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c
index 592bf584d1d2..7466cb811251 100644
--- a/kernel/time/clocksource.c
+++ b/kernel/time/clocksource.c
@@ -513,7 +513,7 @@ static ssize_t sysfs_override_clocksource(struct sys_device *dev,
          * Check to make sure we don't switch to a non-highres capable
          * clocksource if the tick code is in oneshot mode (highres or nohz)
          */
-        if (tick_oneshot_mode_active() &&
+        if (tick_oneshot_mode_active() && ovr &&
             !(ovr->flags & CLOCK_SOURCE_VALID_FOR_HRES)) {
                 printk(KERN_WARNING "%s clocksource is not HRT compatible. "
                         "Cannot switch while in HRT/NOHZ mode\n", ovr->name);
diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c
index 877dbedc3118..c2ec25087a35 100644
--- a/kernel/time/tick-broadcast.c
+++ b/kernel/time/tick-broadcast.c
@@ -205,11 +205,11 @@ static void tick_handle_periodic_broadcast(struct clock_event_device *dev)
  * Powerstate information: The system enters/leaves a state, where
  * affected devices might stop
  */
-static void tick_do_broadcast_on_off(void *why)
+static void tick_do_broadcast_on_off(unsigned long *reason)
 {
         struct clock_event_device *bc, *dev;
         struct tick_device *td;
-        unsigned long flags, *reason = why;
+        unsigned long flags;
         int cpu, bc_stopped;
 
         spin_lock_irqsave(&tick_broadcast_lock, flags);
@@ -276,8 +276,7 @@ void tick_broadcast_on_off(unsigned long reason, int *oncpu)
                 printk(KERN_ERR "tick-broadcast: ignoring broadcast for "
                        "offline CPU #%d\n", *oncpu);
         else
-                smp_call_function_single(*oncpu, tick_do_broadcast_on_off,
-                                         &reason, 1);
+                tick_do_broadcast_on_off(&reason);
 }
 
 /*
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 2aff39c6f10c..e0f59a21c061 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -222,6 +222,15 @@ void tick_nohz_stop_sched_tick(int inidle)
 
         cpu = smp_processor_id();
         ts = &per_cpu(tick_cpu_sched, cpu);
+
+        /*
+         * Call to tick_nohz_start_idle stops the last_update_time from being
+         * updated. Thus, it must not be called in the event we are called from
+         * irq_exit() with the prior state different than idle.
+         */
+        if (!inidle && !ts->inidle)
+                goto end;
+
         now = tick_nohz_start_idle(ts);
 
         /*
@@ -239,9 +248,6 @@ void tick_nohz_stop_sched_tick(int inidle)
         if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))
                 goto end;
 
-        if (!inidle && !ts->inidle)
-                goto end;
-
         ts->inidle = 1;
 
         if (need_resched())
diff --git a/kernel/time/timer_list.c b/kernel/time/timer_list.c
index a999b92a1277..fddd69d16e03 100644
--- a/kernel/time/timer_list.c
+++ b/kernel/time/timer_list.c
@@ -286,7 +286,7 @@ static int __init init_timer_list_procfs(void)
 {
         struct proc_dir_entry *pe;
 
-        pe = proc_create("timer_list", 0644, NULL, &timer_list_fops);
+        pe = proc_create("timer_list", 0444, NULL, &timer_list_fops);
         if (!pe)
                 return -ENOMEM;
         return 0;
diff --git a/kernel/time/timer_stats.c b/kernel/time/timer_stats.c
index c994530d166d..4cde8b9c716f 100644
--- a/kernel/time/timer_stats.c
+++ b/kernel/time/timer_stats.c
@@ -96,7 +96,7 @@ static DEFINE_MUTEX(show_mutex);
 /*
  * Collection status, active/inactive:
  */
-static int __read_mostly active;
+int __read_mostly timer_stats_active;
 
 /*
  * Beginning/end timestamps of measurement:
@@ -242,7 +242,7 @@ void timer_stats_update_stats(void *timer, pid_t pid, void *startf,
         struct entry *entry, input;
         unsigned long flags;
 
-        if (likely(!active))
+        if (likely(!timer_stats_active))
                 return;
 
         lock = &per_cpu(lookup_lock, raw_smp_processor_id());
@@ -254,7 +254,7 @@ void timer_stats_update_stats(void *timer, pid_t pid, void *startf,
         input.timer_flag = timer_flag;
 
         spin_lock_irqsave(lock, flags);
-        if (!active)
+        if (!timer_stats_active)
                 goto out_unlock;
 
         entry = tstat_lookup(&input, comm);
@@ -290,7 +290,7 @@ static int tstats_show(struct seq_file *m, void *v)
         /*
          * If still active then calculate up to now:
          */
-        if (active)
+        if (timer_stats_active)
                 time_stop = ktime_get();
 
         time = ktime_sub(time_stop, time_start);
@@ -368,18 +368,18 @@ static ssize_t tstats_write(struct file *file, const char __user *buf,
         mutex_lock(&show_mutex);
         switch (ctl[0]) {
         case '0':
-                if (active) {
-                        active = 0;
+                if (timer_stats_active) {
+                        timer_stats_active = 0;
                         time_stop = ktime_get();
                         sync_access();
                 }
                 break;
         case '1':
-                if (!active) {
+                if (!timer_stats_active) {
                         reset_entries();
                         time_start = ktime_get();
                         smp_mb();
-                        active = 1;
+                        timer_stats_active = 1;
                 }
                 break;
         default:
diff --git a/kernel/timer.c b/kernel/timer.c
index 54d3912f8cad..a3d25f415019 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -380,6 +380,8 @@ static void timer_stats_account_timer(struct timer_list *timer)
 {
         unsigned int flag = 0;
 
+        if (likely(!timer->start_site))
+                return;
         if (unlikely(tbase_get_deferrable(timer->base)))
                 flag |= TIMER_STATS_FLAG_DEFERRABLE;
 
@@ -712,7 +714,7 @@ int mod_timer(struct timer_list *timer, unsigned long expires)
          * networking code - if the timer is re-modified
          * to be the same thing then just return:
          */
-        if (timer->expires == expires && timer_pending(timer))
+        if (timer_pending(timer) && timer->expires == expires)
                 return 1;
 
         return __mod_timer(timer, expires, false, TIMER_NOT_PINNED);
@@ -1154,8 +1156,7 @@ void update_process_times(int user_tick)
         /* Note: this timer irq context must be accounted for as well. */
         account_process_tick(p, user_tick);
         run_local_timers();
-        if (rcu_pending(cpu))
-                rcu_check_callbacks(cpu, user_tick);
+        rcu_check_callbacks(cpu, user_tick);
         printk_tick();
         scheduler_tick();
         run_posix_cpu_timers(p);
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index 4a13e5a01ce3..e71634604400 100644
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -11,31 +11,48 @@ config NOP_TRACER
 
 config HAVE_FTRACE_NMI_ENTER
         bool
+        help
+          See Documentation/trace/ftrace-implementation.txt
 
 config HAVE_FUNCTION_TRACER
         bool
+        help
+          See Documentation/trace/ftrace-implementation.txt
 
 config HAVE_FUNCTION_GRAPH_TRACER
         bool
+        help
+          See Documentation/trace/ftrace-implementation.txt
+
+config HAVE_FUNCTION_GRAPH_FP_TEST
+        bool
+        help
+         An arch may pass in a unique value (frame pointer) to both the
+         entering and exiting of a function. On exit, the value is compared
+         and if it does not match, then it will panic the kernel.
 
 config HAVE_FUNCTION_TRACE_MCOUNT_TEST
         bool
         help
-         This gets selected when the arch tests the function_trace_stop
-         variable at the mcount call site. Otherwise, this variable
-         is tested by the called function.
+          See Documentation/trace/ftrace-implementation.txt
 
 config HAVE_DYNAMIC_FTRACE
         bool
+        help
+          See Documentation/trace/ftrace-implementation.txt
 
 config HAVE_FTRACE_MCOUNT_RECORD
         bool
+        help
+          See Documentation/trace/ftrace-implementation.txt
 
 config HAVE_HW_BRANCH_TRACER
         bool
 
-config HAVE_FTRACE_SYSCALLS
+config HAVE_SYSCALL_TRACEPOINTS
         bool
+        help
+          See Documentation/trace/ftrace-implementation.txt
 
 config TRACER_MAX_TRACE
         bool
@@ -53,9 +70,14 @@ config EVENT_TRACING
         bool
 
 config CONTEXT_SWITCH_TRACER
-        select MARKERS
         bool
 
+config RING_BUFFER_ALLOW_SWAP
+        bool
+        help
+         Allow the use of ring_buffer_swap_cpu.
+         Adds a very slight overhead to tracing when enabled.
+
 # All tracer options should select GENERIC_TRACER. For those options that are
 # enabled by all tracers (context switch and event tracer) they select TRACING.
 # This allows those options to appear when no other tracer is selected. But the
@@ -121,6 +143,7 @@ config FUNCTION_GRAPH_TRACER
         bool "Kernel Function Graph Tracer"
         depends on HAVE_FUNCTION_GRAPH_TRACER
         depends on FUNCTION_TRACER
+        depends on !X86_32 || !CC_OPTIMIZE_FOR_SIZE
         default y
         help
           Enable the kernel to trace a function at both its return
@@ -139,6 +162,7 @@ config IRQSOFF_TRACER
         select TRACE_IRQFLAGS
         select GENERIC_TRACER
         select TRACER_MAX_TRACE
+        select RING_BUFFER_ALLOW_SWAP
         help
           This option measures the time spent in irqs-off critical
           sections, with microsecond accuracy.
@@ -147,7 +171,7 @@ config IRQSOFF_TRACER
           disabled by default and can be runtime (re-)started
           via:
 
-              echo 0 > /debugfs/tracing/tracing_max_latency
+              echo 0 > /sys/kernel/debug/tracing/tracing_max_latency
 
           (Note that kernel size and overhead increases with this option
           enabled. This option and the preempt-off timing option can be
@@ -160,6 +184,7 @@ config PREEMPT_TRACER
         depends on PREEMPT
         select GENERIC_TRACER
         select TRACER_MAX_TRACE
+        select RING_BUFFER_ALLOW_SWAP
         help
           This option measures the time spent in preemption off critical
           sections, with microsecond accuracy.
@@ -168,7 +193,7 @@ config PREEMPT_TRACER
           disabled by default and can be runtime (re-)started
           via:
 
-              echo 0 > /debugfs/tracing/tracing_max_latency
+              echo 0 > /sys/kernel/debug/tracing/tracing_max_latency
 
           (Note that kernel size and overhead increases with this option
           enabled. This option and the irqs-off timing option can be
@@ -203,7 +228,7 @@ config ENABLE_DEFAULT_TRACERS
 
 config FTRACE_SYSCALLS
         bool "Trace syscalls"
-        depends on HAVE_FTRACE_SYSCALLS
+        depends on HAVE_SYSCALL_TRACEPOINTS
         select GENERIC_TRACER
         select KALLSYMS
         help
@@ -218,13 +243,13 @@ config BOOT_TRACER
           the timings of the initcalls and traces key events and the identity
           of tasks that can cause boot delays, such as context-switches.
 
-          Its aim is to be parsed by the /scripts/bootgraph.pl tool to
+          Its aim is to be parsed by the scripts/bootgraph.pl tool to
           produce pretty graphics about boot inefficiencies, giving a visual
           representation of the delays during initcalls - but the raw
           /debug/tracing/trace text output is readable too.
 
-          You must pass in ftrace=initcall to the kernel command line
-          to enable this on bootup.
+          You must pass in initcall_debug and ftrace=initcall to the kernel
+          command line to enable this on bootup.
 
 config TRACE_BRANCH_PROFILING
         bool
@@ -261,7 +286,7 @@ config PROFILE_ANNOTATED_BRANCHES
           This tracer profiles all the the likely and unlikely macros
           in the kernel. It will display the results in:
 
-          /debugfs/tracing/profile_annotated_branch
+          /sys/kernel/debug/tracing/profile_annotated_branch
 
           Note: this will add a significant overhead, only turn this
           on if you need to profile the system's use of these macros.
@@ -274,7 +299,7 @@ config PROFILE_ALL_BRANCHES
           taken in the kernel is recorded whether it hit or miss.
           The results will be displayed in:
 
-          /debugfs/tracing/profile_branch
+          /sys/kernel/debug/tracing/profile_branch
 
           This option also enables the likely/unlikely profiler.
 
@@ -323,7 +348,7 @@ config STACK_TRACER
         select KALLSYMS
         help
           This special tracer records the maximum stack footprint of the
-          kernel and displays it in debugfs/tracing/stack_trace.
+          kernel and displays it in /sys/kernel/debug/tracing/stack_trace.
 
           This tracer works by hooking into every function call that the
           kernel executes, and keeping a maximum stack depth value and
@@ -454,6 +479,18 @@ config FTRACE_STARTUP_TEST
           functioning properly. It will do tests on all the configured
           tracers of ftrace.
 
+config EVENT_TRACE_TEST_SYSCALLS
+        bool "Run selftest on syscall events"
+        depends on FTRACE_STARTUP_TEST
+        help
+         This option will also enable testing every syscall event.
+         It only enables the event and disables it and runs various loads
+         with the event enabled. This adds a bit more time for kernel boot
+         up since it runs this on every system call defined.
+
+         TBD - enable a way to actually call the syscalls as we test their
+               events
+
 config MMIOTRACE
         bool "Memory mapped IO tracing"
         depends on HAVE_MMIOTRACE_SUPPORT && PCI
diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c
index 39af8af6fc30..3eb159c277c8 100644
--- a/kernel/trace/blktrace.c
+++ b/kernel/trace/blktrace.c
@@ -22,6 +22,7 @@
 #include <linux/init.h>
 #include <linux/mutex.h>
 #include <linux/debugfs.h>
+#include <linux/smp_lock.h>
 #include <linux/time.h>
 #include <linux/uaccess.h>
 
@@ -64,13 +65,15 @@ static void trace_note(struct blk_trace *bt, pid_t pid, int action,
 {
         struct blk_io_trace *t;
         struct ring_buffer_event *event = NULL;
+        struct ring_buffer *buffer = NULL;
         int pc = 0;
         int cpu = smp_processor_id();
         bool blk_tracer = blk_tracer_enabled;
 
         if (blk_tracer) {
+                buffer = blk_tr->buffer;
                 pc = preempt_count();
-                event = trace_buffer_lock_reserve(blk_tr, TRACE_BLK,
+                event = trace_buffer_lock_reserve(buffer, TRACE_BLK,
                                                   sizeof(*t) + len,
                                                   0, pc);
                 if (!event)
@@ -95,7 +98,7 @@ record_it:
                 memcpy((void *) t + sizeof(*t), data, len);
 
                 if (blk_tracer)
-                        trace_buffer_unlock_commit(blk_tr, event, 0, pc);
+                        trace_buffer_unlock_commit(buffer, event, 0, pc);
         }
 }
 
@@ -178,6 +181,7 @@ static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes,
 {
         struct task_struct *tsk = current;
         struct ring_buffer_event *event = NULL;
+        struct ring_buffer *buffer = NULL;
         struct blk_io_trace *t;
         unsigned long flags = 0;
         unsigned long *sequence;
@@ -203,8 +207,9 @@ static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes,
         if (blk_tracer) {
                 tracing_record_cmdline(current);
 
+                buffer = blk_tr->buffer;
                 pc = preempt_count();
-                event = trace_buffer_lock_reserve(blk_tr, TRACE_BLK,
+                event = trace_buffer_lock_reserve(buffer, TRACE_BLK,
                                                   sizeof(*t) + pdu_len,
                                                   0, pc);
                 if (!event)
@@ -251,7 +256,7 @@ record_it:
                         memcpy((void *) t + sizeof(*t), pdu_data, pdu_len);
 
                 if (blk_tracer) {
-                        trace_buffer_unlock_commit(blk_tr, event, 0, pc);
+                        trace_buffer_unlock_commit(buffer, event, 0, pc);
                         return;
                 }
         }
@@ -266,8 +271,8 @@ static void blk_trace_free(struct blk_trace *bt)
 {
         debugfs_remove(bt->msg_file);
         debugfs_remove(bt->dropped_file);
-        debugfs_remove(bt->dir);
         relay_close(bt->rchan);
+        debugfs_remove(bt->dir);
         free_percpu(bt->sequence);
         free_percpu(bt->msg_data);
         kfree(bt);
@@ -377,18 +382,8 @@ static int blk_subbuf_start_callback(struct rchan_buf *buf, void *subbuf,
 
 static int blk_remove_buf_file_callback(struct dentry *dentry)
 {
-        struct dentry *parent = dentry->d_parent;
         debugfs_remove(dentry);
 
-        /*
-        * this will fail for all but the last file, but that is ok. what we
-        * care about is the top level buts->name directory going away, when
-        * the last trace file is gone. Then we don't have to rmdir() that
-        * manually on trace stop, so it nicely solves the issue with
-        * force killing of running traces.
-        */
-
-        debugfs_remove(parent);
         return 0;
 }
 
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index bb60732ade0c..cc615f84751b 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -291,7 +291,9 @@ function_stat_next(void *v, int idx)
         pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
 
  again:
-        rec++;
+        if (idx != 0)
+                rec++;
+
         if ((void *)rec >= (void *)&pg->records[pg->index]) {
                 pg = pg->next;
                 if (!pg)
@@ -766,7 +768,7 @@ static struct tracer_stat function_stats __initdata = {
         .stat_show      = function_stat_show
 };
 
-static void ftrace_profile_debugfs(struct dentry *d_tracer)
+static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
 {
         struct ftrace_profile_stat *stat;
         struct dentry *entry;
@@ -784,7 +786,6 @@ static void ftrace_profile_debugfs(struct dentry *d_tracer)
                          * The files created are permanent, if something happens
                          * we still do not free memory.
                          */
-                        kfree(stat);
                         WARN(1,
                              "Could not allocate stat file for cpu %d\n",
                              cpu);
@@ -811,7 +812,7 @@ static void ftrace_profile_debugfs(struct dentry *d_tracer)
 }
 
 #else /* CONFIG_FUNCTION_PROFILER */
-static void ftrace_profile_debugfs(struct dentry *d_tracer)
+static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
 {
 }
 #endif /* CONFIG_FUNCTION_PROFILER */
@@ -1015,71 +1016,35 @@ static int
 __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
 {
         unsigned long ftrace_addr;
-        unsigned long ip, fl;
+        unsigned long flag = 0UL;
 
         ftrace_addr = (unsigned long)FTRACE_ADDR;
 
-        ip = rec->ip;
-
         /*
-         * If this record is not to be traced and
-         * it is not enabled then do nothing.
+         * If this record is not to be traced or we want to disable it,
+         * then disable it.
          *
-         * If this record is not to be traced and
-         * it is enabled then disable it.
+         * If we want to enable it and filtering is off, then enable it.
          *
+         * If we want to enable it and filtering is on, enable it only if
+         * it's filtered
          */
-        if (rec->flags & FTRACE_FL_NOTRACE) {
-                if (rec->flags & FTRACE_FL_ENABLED)
-                        rec->flags &= ~FTRACE_FL_ENABLED;
-                else
-                        return 0;
-
-        } else if (ftrace_filtered && enable) {
-                /*
-                 * Filtering is on:
-                 */
-
-                fl = rec->flags & (FTRACE_FL_FILTER | FTRACE_FL_ENABLED);
-
-                /* Record is filtered and enabled, do nothing */
-                if (fl == (FTRACE_FL_FILTER | FTRACE_FL_ENABLED))
-                        return 0;
-
-                /* Record is not filtered or enabled, do nothing */
-                if (!fl)
-                        return 0;
-
-                /* Record is not filtered but enabled, disable it */
-                if (fl == FTRACE_FL_ENABLED)
-                        rec->flags &= ~FTRACE_FL_ENABLED;
-                else
-                /* Otherwise record is filtered but not enabled, enable it */
-                        rec->flags |= FTRACE_FL_ENABLED;
-        } else {
-                /* Disable or not filtered */
-
-                if (enable) {
-                        /* if record is enabled, do nothing */
-                        if (rec->flags & FTRACE_FL_ENABLED)
-                                return 0;
-
-                        rec->flags |= FTRACE_FL_ENABLED;
-
-                } else {
+        if (enable && !(rec->flags & FTRACE_FL_NOTRACE)) {
+                if (!ftrace_filtered || (rec->flags & FTRACE_FL_FILTER))
+                        flag = FTRACE_FL_ENABLED;
+        }
 
-                        /* if record is not enabled, do nothing */
-                        if (!(rec->flags & FTRACE_FL_ENABLED))
-                                return 0;
+        /* If the state of this record hasn't changed, then do nothing */
+        if ((rec->flags & FTRACE_FL_ENABLED) == flag)
+                return 0;
 
-                        rec->flags &= ~FTRACE_FL_ENABLED;
-                }
+        if (flag) {
+                rec->flags |= FTRACE_FL_ENABLED;
+                return ftrace_make_call(rec, ftrace_addr);
         }
 
-        if (rec->flags & FTRACE_FL_ENABLED)
-                return ftrace_make_call(rec, ftrace_addr);
-        else
-                return ftrace_make_nop(NULL, rec, ftrace_addr);
+        rec->flags &= ~FTRACE_FL_ENABLED;
+        return ftrace_make_nop(NULL, rec, ftrace_addr);
 }
 
 static void ftrace_replace_code(int enable)
@@ -1224,6 +1189,13 @@ static void ftrace_shutdown(int command)
                 return;
 
         ftrace_start_up--;
+        /*
+         * Just warn in case of unbalance, no need to kill ftrace, it's not
+         * critical but the ftrace_call callers may be never nopped again after
+         * further ftrace uses.
+         */
+        WARN_ON_ONCE(ftrace_start_up < 0);
+
         if (!ftrace_start_up)
                 command |= FTRACE_DISABLE_CALLS;
 
@@ -1351,11 +1323,10 @@ static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
 
 enum {
         FTRACE_ITER_FILTER      = (1 << 0),
-        FTRACE_ITER_CONT        = (1 << 1),
-        FTRACE_ITER_NOTRACE     = (1 << 2),
-        FTRACE_ITER_FAILURES    = (1 << 3),
-        FTRACE_ITER_PRINTALL    = (1 << 4),
-        FTRACE_ITER_HASH        = (1 << 5),
+        FTRACE_ITER_NOTRACE     = (1 << 1),
+        FTRACE_ITER_FAILURES    = (1 << 2),
+        FTRACE_ITER_PRINTALL    = (1 << 3),
+        FTRACE_ITER_HASH        = (1 << 4),
 };
 
 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
@@ -1365,9 +1336,7 @@ struct ftrace_iterator {
         int                     hidx;
         int                     idx;
         unsigned                flags;
-        unsigned char           buffer[FTRACE_BUFF_MAX+1];
-        unsigned                buffer_idx;
-        unsigned                filtered;
+        struct trace_parser     parser;
 };
 
 static void *
@@ -1410,28 +1379,33 @@ static void *t_hash_start(struct seq_file *m, loff_t *pos)
 {
         struct ftrace_iterator *iter = m->private;
         void *p = NULL;
+        loff_t l;
+
+        if (!(iter->flags & FTRACE_ITER_HASH))
+                *pos = 0;
 
         iter->flags |= FTRACE_ITER_HASH;
 
-        return t_hash_next(m, p, pos);
+        iter->hidx = 0;
+        for (l = 0; l <= *pos; ) {
+                p = t_hash_next(m, p, &l);
+                if (!p)
+                        break;
+        }
+        return p;
 }
 
 static int t_hash_show(struct seq_file *m, void *v)
 {
         struct ftrace_func_probe *rec;
         struct hlist_node *hnd = v;
-        char str[KSYM_SYMBOL_LEN];
 
         rec = hlist_entry(hnd, struct ftrace_func_probe, node);
 
         if (rec->ops->print)
                 return rec->ops->print(m, rec->ip, rec->ops, rec->data);
 
-        kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
-        seq_printf(m, "%s:", str);
-
-        kallsyms_lookup((unsigned long)rec->ops->func, NULL, NULL, NULL, str);
-        seq_printf(m, "%s", str);
+        seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func);
 
         if (rec->data)
                 seq_printf(m, ":%p", rec->data);
@@ -1460,8 +1434,6 @@ t_next(struct seq_file *m, void *v, loff_t *pos)
                         iter->pg = iter->pg->next;
                         iter->idx = 0;
                         goto retry;
-                } else {
-                        iter->idx = -1;
                 }
         } else {
                 rec = &iter->pg->records[iter->idx++];
@@ -1490,6 +1462,7 @@ static void *t_start(struct seq_file *m, loff_t *pos)
 {
         struct ftrace_iterator *iter = m->private;
         void *p = NULL;
+        loff_t l;
 
         mutex_lock(&ftrace_lock);
         /*
@@ -1501,23 +1474,21 @@ static void *t_start(struct seq_file *m, loff_t *pos)
                 if (*pos > 0)
                         return t_hash_start(m, pos);
                 iter->flags |= FTRACE_ITER_PRINTALL;
-                (*pos)++;
                 return iter;
         }
 
         if (iter->flags & FTRACE_ITER_HASH)
                 return t_hash_start(m, pos);
 
-        if (*pos > 0) {
-                if (iter->idx < 0)
-                        return p;
-                (*pos)--;
-                iter->idx--;
+        iter->pg = ftrace_pages_start;
+        iter->idx = 0;
+        for (l = 0; l <= *pos; ) {
+                p = t_next(m, p, &l);
+                if (!p)
+                        break;
         }
 
-        p = t_next(m, p, pos);
-
-        if (!p)
+        if (!p && iter->flags & FTRACE_ITER_FILTER)
                 return t_hash_start(m, pos);
 
         return p;
@@ -1532,7 +1503,6 @@ static int t_show(struct seq_file *m, void *v)
 {
         struct ftrace_iterator *iter = m->private;
         struct dyn_ftrace *rec = v;
-        char str[KSYM_SYMBOL_LEN];
 
         if (iter->flags & FTRACE_ITER_HASH)
                 return t_hash_show(m, v);
@@ -1545,9 +1515,7 @@ static int t_show(struct seq_file *m, void *v)
         if (!rec)
                 return 0;
 
-        kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
-
-        seq_printf(m, "%s\n", str);
+        seq_printf(m, "%ps\n", (void *)rec->ip);
 
         return 0;
 }
@@ -1586,17 +1554,6 @@ ftrace_avail_open(struct inode *inode, struct file *file)
         return ret;
 }
 
-int ftrace_avail_release(struct inode *inode, struct file *file)
-{
-        struct seq_file *m = (struct seq_file *)file->private_data;
-        struct ftrace_iterator *iter = m->private;
-
-        seq_release(inode, file);
-        kfree(iter);
-
-        return 0;
-}
-
 static int
 ftrace_failures_open(struct inode *inode, struct file *file)
 {
@@ -1645,9 +1602,14 @@ ftrace_regex_open(struct inode *inode, struct file *file, int enable)
         if (!iter)
                 return -ENOMEM;
 
+        if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
+                kfree(iter);
+                return -ENOMEM;
+        }
+
         mutex_lock(&ftrace_regex_lock);
         if ((file->f_mode & FMODE_WRITE) &&
-            !(file->f_flags & O_APPEND))
+            (file->f_flags & O_TRUNC))
                 ftrace_filter_reset(enable);
 
         if (file->f_mode & FMODE_READ) {
@@ -2100,9 +2062,9 @@ __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
         int i, len = 0;
         char *search;
 
-        if (glob && (strcmp(glob, "*") || !strlen(glob)))
+        if (glob && (strcmp(glob, "*") == 0 || !strlen(glob)))
                 glob = NULL;
-        else {
+        else if (glob) {
                 int not;
 
                 type = ftrace_setup_glob(glob, strlen(glob), &search, &not);
@@ -2237,9 +2199,8 @@ ftrace_regex_write(struct file *file, const char __user *ubuf,
                    size_t cnt, loff_t *ppos, int enable)
 {
         struct ftrace_iterator *iter;
-        char ch;
-        size_t read = 0;
-        ssize_t ret;
+        struct trace_parser *parser;
+        ssize_t ret, read;
 
         if (!cnt || cnt < 0)
                 return 0;
@@ -2252,68 +2213,23 @@ ftrace_regex_write(struct file *file, const char __user *ubuf,
         } else
                 iter = file->private_data;
 
-        if (!*ppos) {
-                iter->flags &= ~FTRACE_ITER_CONT;
-                iter->buffer_idx = 0;
-        }
-
-        ret = get_user(ch, ubuf++);
-        if (ret)
-                goto out;
-        read++;
-        cnt--;
-
-        if (!(iter->flags & ~FTRACE_ITER_CONT)) {
-                /* skip white space */
-                while (cnt && isspace(ch)) {
-                        ret = get_user(ch, ubuf++);
-                        if (ret)
-                                goto out;
-                        read++;
-                        cnt--;
-                }
+        parser = &iter->parser;
+        read = trace_get_user(parser, ubuf, cnt, ppos);
 
-                if (isspace(ch)) {
-                        file->f_pos += read;
-                        ret = read;
-                        goto out;
-                }
-
-                iter->buffer_idx = 0;
-        }
-
-        while (cnt && !isspace(ch)) {
-                if (iter->buffer_idx < FTRACE_BUFF_MAX)
-                        iter->buffer[iter->buffer_idx++] = ch;
-                else {
-                        ret = -EINVAL;
-                        goto out;
-                }
-                ret = get_user(ch, ubuf++);
+        if (trace_parser_loaded(parser) &&
+            !trace_parser_cont(parser)) {
+                ret = ftrace_process_regex(parser->buffer,
+                                           parser->idx, enable);
                 if (ret)
                         goto out;
-                read++;
-                cnt--;
-        }
 
-        if (isspace(ch)) {
-                iter->filtered++;
-                iter->buffer[iter->buffer_idx] = 0;
-                ret = ftrace_process_regex(iter->buffer,
-                                           iter->buffer_idx, enable);
-                if (ret)
-                        goto out;
-                iter->buffer_idx = 0;
-        } else
-                iter->flags |= FTRACE_ITER_CONT;
-
-
-        file->f_pos += read;
+                trace_parser_clear(parser);
+        }
 
         ret = read;
- out:
-        mutex_unlock(&ftrace_regex_lock);
 
+        mutex_unlock(&ftrace_regex_lock);
+out:
         return ret;
 }
 
@@ -2418,6 +2334,7 @@ ftrace_regex_release(struct inode *inode, struct file *file, int enable)
 {
         struct seq_file *m = (struct seq_file *)file->private_data;
         struct ftrace_iterator *iter;
+        struct trace_parser *parser;
 
         mutex_lock(&ftrace_regex_lock);
         if (file->f_mode & FMODE_READ) {
@@ -2427,10 +2344,10 @@ ftrace_regex_release(struct inode *inode, struct file *file, int enable)
         } else
                 iter = file->private_data;
 
-        if (iter->buffer_idx) {
-                iter->filtered++;
-                iter->buffer[iter->buffer_idx] = 0;
-                ftrace_match_records(iter->buffer, iter->buffer_idx, enable);
+        parser = &iter->parser;
+        if (trace_parser_loaded(parser)) {
+                parser->buffer[parser->idx] = 0;
+                ftrace_match_records(parser->buffer, parser->idx, enable);
         }
 
         mutex_lock(&ftrace_lock);
@@ -2438,7 +2355,9 @@ ftrace_regex_release(struct inode *inode, struct file *file, int enable)
                 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
         mutex_unlock(&ftrace_lock);
 
+        trace_parser_put(parser);
         kfree(iter);
+
         mutex_unlock(&ftrace_regex_lock);
         return 0;
 }
@@ -2459,14 +2378,14 @@ static const struct file_operations ftrace_avail_fops = {
         .open = ftrace_avail_open,
         .read = seq_read,
         .llseek = seq_lseek,
-        .release = ftrace_avail_release,
+        .release = seq_release_private,
 };
 
 static const struct file_operations ftrace_failures_fops = {
         .open = ftrace_failures_open,
         .read = seq_read,
         .llseek = seq_lseek,
-        .release = ftrace_avail_release,
+        .release = seq_release_private,
 };
 
 static const struct file_operations ftrace_filter_fops = {
@@ -2493,32 +2412,31 @@ int ftrace_graph_count;
 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
 
 static void *
-g_next(struct seq_file *m, void *v, loff_t *pos)
+__g_next(struct seq_file *m, loff_t *pos)
 {
         unsigned long *array = m->private;
-        int index = *pos;
-
-        (*pos)++;
 
-        if (index >= ftrace_graph_count)
+        if (*pos >= ftrace_graph_count)
                 return NULL;
+        return &array[*pos];
+}
 
-        return &array[index];
+static void *
+g_next(struct seq_file *m, void *v, loff_t *pos)
+{
+        (*pos)++;
+        return __g_next(m, pos);
 }
 
 static void *g_start(struct seq_file *m, loff_t *pos)
 {
-        void *p = NULL;
-
         mutex_lock(&graph_lock);
 
         /* Nothing, tell g_show to print all functions are enabled */
         if (!ftrace_graph_count && !*pos)
                 return (void *)1;
 
-        p = g_next(m, p, pos);
-
-        return p;
+        return __g_next(m, pos);
 }
 
 static void g_stop(struct seq_file *m, void *p)
@@ -2529,7 +2447,6 @@ static void g_stop(struct seq_file *m, void *p)
 static int g_show(struct seq_file *m, void *v)
 {
         unsigned long *ptr = v;
-        char str[KSYM_SYMBOL_LEN];
 
         if (!ptr)
                 return 0;
@@ -2539,9 +2456,7 @@ static int g_show(struct seq_file *m, void *v)
                 return 0;
         }
 
-        kallsyms_lookup(*ptr, NULL, NULL, NULL, str);
-
-        seq_printf(m, "%s\n", str);
+        seq_printf(m, "%ps\n", (void *)*ptr);
 
         return 0;
 }
@@ -2563,7 +2478,7 @@ ftrace_graph_open(struct inode *inode, struct file *file)
 
         mutex_lock(&graph_lock);
         if ((file->f_mode & FMODE_WRITE) &&
-            !(file->f_flags & O_APPEND)) {
+            (file->f_flags & O_TRUNC)) {
                 ftrace_graph_count = 0;
                 memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
         }
@@ -2582,6 +2497,14 @@ ftrace_graph_open(struct inode *inode, struct file *file)
 }
 
 static int
+ftrace_graph_release(struct inode *inode, struct file *file)
+{
+        if (file->f_mode & FMODE_READ)
+                seq_release(inode, file);
+        return 0;
+}
+
+static int
 ftrace_set_func(unsigned long *array, int *idx, char *buffer)
 {
         struct dyn_ftrace *rec;
@@ -2636,12 +2559,10 @@ static ssize_t
 ftrace_graph_write(struct file *file, const char __user *ubuf,
                    size_t cnt, loff_t *ppos)
 {
-        unsigned char buffer[FTRACE_BUFF_MAX+1];
+        struct trace_parser parser;
         unsigned long *array;
         size_t read = 0;
         ssize_t ret;
-        int index = 0;
-        char ch;
 
         if (!cnt || cnt < 0)
                 return 0;
@@ -2659,60 +2580,36 @@ ftrace_graph_write(struct file *file, const char __user *ubuf,
         } else
                 array = file->private_data;
 
-        ret = get_user(ch, ubuf++);
-        if (ret)
+        if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) {
+                ret = -ENOMEM;
                 goto out;
-        read++;
-        cnt--;
-
-        /* skip white space */
-        while (cnt && isspace(ch)) {
-                ret = get_user(ch, ubuf++);
-                if (ret)
-                        goto out;
-                read++;
-                cnt--;
         }
 
-        if (isspace(ch)) {
-                *ppos += read;
-                ret = read;
-                goto out;
-        }
+        read = trace_get_user(&parser, ubuf, cnt, ppos);
 
-        while (cnt && !isspace(ch)) {
-                if (index < FTRACE_BUFF_MAX)
-                        buffer[index++] = ch;
-                else {
-                        ret = -EINVAL;
-                        goto out;
-                }
-                ret = get_user(ch, ubuf++);
+        if (trace_parser_loaded((&parser))) {
+                parser.buffer[parser.idx] = 0;
+
+                /* we allow only one expression at a time */
+                ret = ftrace_set_func(array, &ftrace_graph_count,
+                                        parser.buffer);
                 if (ret)
                         goto out;
-                read++;
-                cnt--;
         }
-        buffer[index] = 0;
-
-        /* we allow only one expression at a time */
-        ret = ftrace_set_func(array, &ftrace_graph_count, buffer);
-        if (ret)
-                goto out;
-
-        file->f_pos += read;
 
         ret = read;
  out:
+        trace_parser_put(&parser);
         mutex_unlock(&graph_lock);
 
         return ret;
 }
 
 static const struct file_operations ftrace_graph_fops = {
-        .open = ftrace_graph_open,
-        .read = seq_read,
-        .write = ftrace_graph_write,
+        .open           = ftrace_graph_open,
+        .read           = seq_read,
+        .write          = ftrace_graph_write,
+        .release        = ftrace_graph_release,
 };
 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 
@@ -3145,10 +3042,10 @@ ftrace_enable_sysctl(struct ctl_table *table, int write,
 
         ret  = proc_dointvec(table, write, file, buffer, lenp, ppos);
 
-        if (ret || !write || (last_ftrace_enabled == ftrace_enabled))
+        if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
                 goto out;
 
-        last_ftrace_enabled = ftrace_enabled;
+        last_ftrace_enabled = !!ftrace_enabled;
 
         if (ftrace_enabled) {
 
diff --git a/kernel/trace/kmemtrace.c b/kernel/trace/kmemtrace.c
index 86cdf671d7e2..81b1645c8549 100644
--- a/kernel/trace/kmemtrace.c
+++ b/kernel/trace/kmemtrace.c
@@ -183,11 +183,9 @@ static void kmemtrace_stop_probes(void)
 
 static int kmem_trace_init(struct trace_array *tr)
 {
-        int cpu;
         kmemtrace_array = tr;
 
-        for_each_cpu_mask(cpu, cpu_possible_map)
-                tracing_reset(tr, cpu);
+        tracing_reset_online_cpus(tr);
 
         kmemtrace_start_probes();
 
@@ -239,12 +237,52 @@ struct kmemtrace_user_event_alloc {
 };
 
 static enum print_line_t
-kmemtrace_print_alloc_user(struct trace_iterator *iter,
-                           struct kmemtrace_alloc_entry *entry)
+kmemtrace_print_alloc(struct trace_iterator *iter, int flags)
 {
-        struct kmemtrace_user_event_alloc *ev_alloc;
         struct trace_seq *s = &iter->seq;
+        struct kmemtrace_alloc_entry *entry;
+        int ret;
+
+        trace_assign_type(entry, iter->ent);
+
+        ret = trace_seq_printf(s, "type_id %d call_site %pF ptr %lu "
+            "bytes_req %lu bytes_alloc %lu gfp_flags %lu node %d\n",
+            entry->type_id, (void *)entry->call_site, (unsigned long)entry->ptr,
+            (unsigned long)entry->bytes_req, (unsigned long)entry->bytes_alloc,
+            (unsigned long)entry->gfp_flags, entry->node);
+
+        if (!ret)
+                return TRACE_TYPE_PARTIAL_LINE;
+        return TRACE_TYPE_HANDLED;
+}
+
+static enum print_line_t
+kmemtrace_print_free(struct trace_iterator *iter, int flags)
+{
+        struct trace_seq *s = &iter->seq;
+        struct kmemtrace_free_entry *entry;
+        int ret;
+
+        trace_assign_type(entry, iter->ent);
+
+        ret = trace_seq_printf(s, "type_id %d call_site %pF ptr %lu\n",
+                               entry->type_id, (void *)entry->call_site,
+                               (unsigned long)entry->ptr);
+
+        if (!ret)
+                return TRACE_TYPE_PARTIAL_LINE;
+        return TRACE_TYPE_HANDLED;
+}
+
+static enum print_line_t
+kmemtrace_print_alloc_user(struct trace_iterator *iter, int flags)
+{
+        struct trace_seq *s = &iter->seq;
+        struct kmemtrace_alloc_entry *entry;
         struct kmemtrace_user_event *ev;
+        struct kmemtrace_user_event_alloc *ev_alloc;
+
+        trace_assign_type(entry, iter->ent);
 
         ev = trace_seq_reserve(s, sizeof(*ev));
         if (!ev)
@@ -271,12 +309,14 @@ kmemtrace_print_alloc_user(struct trace_iterator *iter,
 }
 
 static enum print_line_t
-kmemtrace_print_free_user(struct trace_iterator *iter,
-                          struct kmemtrace_free_entry *entry)
+kmemtrace_print_free_user(struct trace_iterator *iter, int flags)
 {
         struct trace_seq *s = &iter->seq;
+        struct kmemtrace_free_entry *entry;
         struct kmemtrace_user_event *ev;
 
+        trace_assign_type(entry, iter->ent);
+
         ev = trace_seq_reserve(s, sizeof(*ev));
         if (!ev)
                 return TRACE_TYPE_PARTIAL_LINE;
@@ -294,12 +334,14 @@ kmemtrace_print_free_user(struct trace_iterator *iter,
 
 /* The two other following provide a more minimalistic output */
 static enum print_line_t
-kmemtrace_print_alloc_compress(struct trace_iterator *iter,
-                                        struct kmemtrace_alloc_entry *entry)
+kmemtrace_print_alloc_compress(struct trace_iterator *iter)
 {
+        struct kmemtrace_alloc_entry *entry;
         struct trace_seq *s = &iter->seq;
         int ret;
 
+        trace_assign_type(entry, iter->ent);
+
         /* Alloc entry */
         ret = trace_seq_printf(s, "  +      ");
         if (!ret)
@@ -345,29 +387,24 @@ kmemtrace_print_alloc_compress(struct trace_iterator *iter,
         if (!ret)
                 return TRACE_TYPE_PARTIAL_LINE;
 
-        /* Node */
-        ret = trace_seq_printf(s, "%4d   ", entry->node);
-        if (!ret)
-                return TRACE_TYPE_PARTIAL_LINE;
-
-        /* Call site */
-        ret = seq_print_ip_sym(s, entry->call_site, 0);
+        /* Node and call site*/
+        ret = trace_seq_printf(s, "%4d   %pf\n", entry->node,
+                                                 (void *)entry->call_site);
         if (!ret)
                 return TRACE_TYPE_PARTIAL_LINE;
 
-        if (!trace_seq_printf(s, "\n"))
-                return TRACE_TYPE_PARTIAL_LINE;
-
         return TRACE_TYPE_HANDLED;
 }
 
 static enum print_line_t
-kmemtrace_print_free_compress(struct trace_iterator *iter,
-                              struct kmemtrace_free_entry *entry)
+kmemtrace_print_free_compress(struct trace_iterator *iter)
 {
+        struct kmemtrace_free_entry *entry;
         struct trace_seq *s = &iter->seq;
         int ret;
 
+        trace_assign_type(entry, iter->ent);
+
         /* Free entry */
         ret = trace_seq_printf(s, "  -      ");
         if (!ret)
@@ -401,19 +438,11 @@ kmemtrace_print_free_compress(struct trace_iterator *iter,
         if (!ret)
                 return TRACE_TYPE_PARTIAL_LINE;
 
-        /* Skip node */
-        ret = trace_seq_printf(s, "       ");
+        /* Skip node and print call site*/
+        ret = trace_seq_printf(s, "       %pf\n", (void *)entry->call_site);
         if (!ret)
                 return TRACE_TYPE_PARTIAL_LINE;
 
-        /* Call site */
-        ret = seq_print_ip_sym(s, entry->call_site, 0);
-        if (!ret)
-                return TRACE_TYPE_PARTIAL_LINE;
-
-        if (!trace_seq_printf(s, "\n"))
-                return TRACE_TYPE_PARTIAL_LINE;
-
         return TRACE_TYPE_HANDLED;
 }
 
@@ -421,32 +450,31 @@ static enum print_line_t kmemtrace_print_line(struct trace_iterator *iter)
 {
         struct trace_entry *entry = iter->ent;
 
-        switch (entry->type) {
-        case TRACE_KMEM_ALLOC: {
-                struct kmemtrace_alloc_entry *field;
-
-                trace_assign_type(field, entry);
-                if (kmem_tracer_flags.val & TRACE_KMEM_OPT_MINIMAL)
-                        return kmemtrace_print_alloc_compress(iter, field);
-                else
-                        return kmemtrace_print_alloc_user(iter, field);
-        }
-
-        case TRACE_KMEM_FREE: {
-                struct kmemtrace_free_entry *field;
-
-                trace_assign_type(field, entry);
-                if (kmem_tracer_flags.val & TRACE_KMEM_OPT_MINIMAL)
-                        return kmemtrace_print_free_compress(iter, field);
-                else
-                        return kmemtrace_print_free_user(iter, field);
-        }
+        if (!(kmem_tracer_flags.val & TRACE_KMEM_OPT_MINIMAL))
+                return TRACE_TYPE_UNHANDLED;
 
+        switch (entry->type) {
+        case TRACE_KMEM_ALLOC:
+                return kmemtrace_print_alloc_compress(iter);
+        case TRACE_KMEM_FREE:
+                return kmemtrace_print_free_compress(iter);
         default:
                 return TRACE_TYPE_UNHANDLED;
         }
 }
 
+static struct trace_event kmem_trace_alloc = {
+        .type                   = TRACE_KMEM_ALLOC,
+        .trace                  = kmemtrace_print_alloc,
+        .binary                 = kmemtrace_print_alloc_user,
+};
+
+static struct trace_event kmem_trace_free = {
+        .type                   = TRACE_KMEM_FREE,
+        .trace                  = kmemtrace_print_free,
+        .binary                 = kmemtrace_print_free_user,
+};
+
 static struct tracer kmem_tracer __read_mostly = {
         .name                   = "kmemtrace",
         .init                   = kmem_trace_init,
@@ -463,6 +491,21 @@ void kmemtrace_init(void)
 
 static int __init init_kmem_tracer(void)
 {
-        return register_tracer(&kmem_tracer);
+        if (!register_ftrace_event(&kmem_trace_alloc)) {
+                pr_warning("Warning: could not register kmem events\n");
+                return 1;
+        }
+
+        if (!register_ftrace_event(&kmem_trace_free)) {
+                pr_warning("Warning: could not register kmem events\n");
+                return 1;
+        }
+
+        if (!register_tracer(&kmem_tracer)) {
+                pr_warning("Warning: could not register the kmem tracer\n");
+                return 1;
+        }
+
+        return 0;
 }
 device_initcall(init_kmem_tracer);
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index 2e642b2b7253..6eef38923b07 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -10,6 +10,7 @@
 #include <linux/debugfs.h>
 #include <linux/uaccess.h>
 #include <linux/hardirq.h>
+#include <linux/kmemcheck.h>
 #include <linux/module.h>
 #include <linux/percpu.h>
 #include <linux/mutex.h>
@@ -205,6 +206,7 @@ EXPORT_SYMBOL_GPL(tracing_is_on);
 #define RB_EVNT_HDR_SIZE (offsetof(struct ring_buffer_event, array))
 #define RB_ALIGNMENT            4U
 #define RB_MAX_SMALL_DATA       (RB_ALIGNMENT * RINGBUF_TYPE_DATA_TYPE_LEN_MAX)
+#define RB_EVNT_MIN_SIZE        8U      /* two 32bit words */
 
 /* define RINGBUF_TYPE_DATA for 'case RINGBUF_TYPE_DATA:' */
 #define RINGBUF_TYPE_DATA 0 ... RINGBUF_TYPE_DATA_TYPE_LEN_MAX
@@ -216,17 +218,12 @@ enum {
 
 static inline int rb_null_event(struct ring_buffer_event *event)
 {
-        return event->type_len == RINGBUF_TYPE_PADDING
-                        && event->time_delta == 0;
-}
-
-static inline int rb_discarded_event(struct ring_buffer_event *event)
-{
-        return event->type_len == RINGBUF_TYPE_PADDING && event->time_delta;
+        return event->type_len == RINGBUF_TYPE_PADDING && !event->time_delta;
 }
 
 static void rb_event_set_padding(struct ring_buffer_event *event)
 {
+        /* padding has a NULL time_delta */
         event->type_len = RINGBUF_TYPE_PADDING;
         event->time_delta = 0;
 }
@@ -320,6 +317,14 @@ struct buffer_data_page {
         unsigned char    data[];        /* data of buffer page */
 };
 
+/*
+ * Note, the buffer_page list must be first. The buffer pages
+ * are allocated in cache lines, which means that each buffer
+ * page will be at the beginning of a cache line, and thus
+ * the least significant bits will be zero. We use this to
+ * add flags in the list struct pointers, to make the ring buffer
+ * lockless.
+ */
 struct buffer_page {
         struct list_head list;          /* list of buffer pages */
         local_t          write;         /* index for next write */
@@ -328,6 +333,21 @@ struct buffer_page {
         struct buffer_data_page *page;  /* Actual data page */
 };
 
+/*
+ * The buffer page counters, write and entries, must be reset
+ * atomically when crossing page boundaries. To synchronize this
+ * update, two counters are inserted into the number. One is
+ * the actual counter for the write position or count on the page.
+ *
+ * The other is a counter of updaters. Before an update happens
+ * the update partition of the counter is incremented. This will
+ * allow the updater to update the counter atomically.
+ *
+ * The counter is 20 bits, and the state data is 12.
+ */
+#define RB_WRITE_MASK           0xfffff
+#define RB_WRITE_INTCNT         (1 << 20)
+
 static void rb_init_page(struct buffer_data_page *bpage)
 {
         local_set(&bpage->commit, 0);
@@ -401,19 +421,20 @@ int ring_buffer_print_page_header(struct trace_seq *s)
 struct ring_buffer_per_cpu {
         int                             cpu;
         struct ring_buffer              *buffer;
-        spinlock_t                      reader_lock; /* serialize readers */
+        spinlock_t                      reader_lock;    /* serialize readers */
         raw_spinlock_t                  lock;
         struct lock_class_key           lock_key;
-        struct list_head                pages;
+        struct list_head                *pages;
         struct buffer_page              *head_page;     /* read from head */
         struct buffer_page              *tail_page;     /* write to tail */
         struct buffer_page              *commit_page;   /* committed pages */
         struct buffer_page              *reader_page;
-        unsigned long                   nmi_dropped;
-        unsigned long                   commit_overrun;
-        unsigned long                   overrun;
-        unsigned long                   read;
+        local_t                         commit_overrun;
+        local_t                         overrun;
         local_t                         entries;
+        local_t                         committing;
+        local_t                         commits;
+        unsigned long                   read;
         u64                             write_stamp;
         u64                             read_stamp;
         atomic_t                        record_disabled;
@@ -446,14 +467,19 @@ struct ring_buffer_iter {
 };
 
 /* buffer may be either ring_buffer or ring_buffer_per_cpu */
-#define RB_WARN_ON(buffer, cond)                                \
-        ({                                                      \
-                int _____ret = unlikely(cond);                  \
-                if (_____ret) {                                 \
-                        atomic_inc(&buffer->record_disabled);   \
-                        WARN_ON(1);                             \
-                }                                               \
-                _____ret;                                       \
+#define RB_WARN_ON(b, cond)                                             \
+        ({                                                              \
+                int _____ret = unlikely(cond);                          \
+                if (_____ret) {                                         \
+                        if (__same_type(*(b), struct ring_buffer_per_cpu)) { \
+                                struct ring_buffer_per_cpu *__b =       \
+                                        (void *)b;                      \
+                                atomic_inc(&__b->buffer->record_disabled); \
+                        } else                                          \
+                                atomic_inc(&b->record_disabled);        \
+                        WARN_ON(1);                                     \
+                }                                                       \
+                _____ret;                                               \
         })
 
 /* Up this if you want to test the TIME_EXTENTS and normalization */
@@ -485,6 +511,390 @@ void ring_buffer_normalize_time_stamp(struct ring_buffer *buffer,
 }
 EXPORT_SYMBOL_GPL(ring_buffer_normalize_time_stamp);
 
+/*
+ * Making the ring buffer lockless makes things tricky.
+ * Although writes only happen on the CPU that they are on,
+ * and they only need to worry about interrupts. Reads can
+ * happen on any CPU.
+ *
+ * The reader page is always off the ring buffer, but when the
+ * reader finishes with a page, it needs to swap its page with
+ * a new one from the buffer. The reader needs to take from
+ * the head (writes go to the tail). But if a writer is in overwrite
+ * mode and wraps, it must push the head page forward.
+ *
+ * Here lies the problem.
+ *
+ * The reader must be careful to replace only the head page, and
+ * not another one. As described at the top of the file in the
+ * ASCII art, the reader sets its old page to point to the next
+ * page after head. It then sets the page after head to point to
+ * the old reader page. But if the writer moves the head page
+ * during this operation, the reader could end up with the tail.
+ *
+ * We use cmpxchg to help prevent this race. We also do something
+ * special with the page before head. We set the LSB to 1.
+ *
+ * When the writer must push the page forward, it will clear the
+ * bit that points to the head page, move the head, and then set
+ * the bit that points to the new head page.
+ *
+ * We also don't want an interrupt coming in and moving the head
+ * page on another writer. Thus we use the second LSB to catch
+ * that too. Thus:
+ *
+ * head->list->prev->next        bit 1          bit 0
+ *                              -------        -------
+ * Normal page                     0              0
+ * Points to head page             0              1
+ * New head page                   1              0
+ *
+ * Note we can not trust the prev pointer of the head page, because:
+ *
+ * +----+       +-----+        +-----+
+ * |    |------>|  T  |---X--->|  N  |
+ * |    |<------|     |        |     |
+ * +----+       +-----+        +-----+
+ *   ^                           ^ |
+ *   |          +-----+          | |
+ *   +----------|  R  |----------+ |
+ *              |     |<-----------+
+ *              +-----+
+ *
+ * Key:  ---X-->  HEAD flag set in pointer
+ *         T      Tail page
+ *         R      Reader page
+ *         N      Next page
+ *
+ * (see __rb_reserve_next() to see where this happens)
+ *
+ *  What the above shows is that the reader just swapped out
+ *  the reader page with a page in the buffer, but before it
+ *  could make the new header point back to the new page added
+ *  it was preempted by a writer. The writer moved forward onto
+ *  the new page added by the reader and is about to move forward
+ *  again.
+ *
+ *  You can see, it is legitimate for the previous pointer of
+ *  the head (or any page) not to point back to itself. But only
+ *  temporarially.
+ */
+
+#define RB_PAGE_NORMAL          0UL
+#define RB_PAGE_HEAD            1UL
+#define RB_PAGE_UPDATE          2UL
+
+
+#define RB_FLAG_MASK            3UL
+
+/* PAGE_MOVED is not part of the mask */
+#define RB_PAGE_MOVED           4UL
+
+/*
+ * rb_list_head - remove any bit
+ */
+static struct list_head *rb_list_head(struct list_head *list)
+{
+        unsigned long val = (unsigned long)list;
+
+        return (struct list_head *)(val & ~RB_FLAG_MASK);
+}
+
+/*
+ * rb_is_head_page - test if the give page is the head page
+ *
+ * Because the reader may move the head_page pointer, we can
+ * not trust what the head page is (it may be pointing to
+ * the reader page). But if the next page is a header page,
+ * its flags will be non zero.
+ */
+static int inline
+rb_is_head_page(struct ring_buffer_per_cpu *cpu_buffer,
+                struct buffer_page *page, struct list_head *list)
+{
+        unsigned long val;
+
+        val = (unsigned long)list->next;
+
+        if ((val & ~RB_FLAG_MASK) != (unsigned long)&page->list)
+                return RB_PAGE_MOVED;
+
+        return val & RB_FLAG_MASK;
+}
+
+/*
+ * rb_is_reader_page
+ *
+ * The unique thing about the reader page, is that, if the
+ * writer is ever on it, the previous pointer never points
+ * back to the reader page.
+ */
+static int rb_is_reader_page(struct buffer_page *page)
+{
+        struct list_head *list = page->list.prev;
+
+        return rb_list_head(list->next) != &page->list;
+}
+
+/*
+ * rb_set_list_to_head - set a list_head to be pointing to head.
+ */
+static void rb_set_list_to_head(struct ring_buffer_per_cpu *cpu_buffer,
+                                struct list_head *list)
+{
+        unsigned long *ptr;
+
+        ptr = (unsigned long *)&list->next;
+        *ptr |= RB_PAGE_HEAD;
+        *ptr &= ~RB_PAGE_UPDATE;
+}
+
+/*
+ * rb_head_page_activate - sets up head page
+ */
+static void rb_head_page_activate(struct ring_buffer_per_cpu *cpu_buffer)
+{
+        struct buffer_page *head;
+
+        head = cpu_buffer->head_page;
+        if (!head)
+                return;
+
+        /*
+         * Set the previous list pointer to have the HEAD flag.
+         */
+        rb_set_list_to_head(cpu_buffer, head->list.prev);
+}
+
+static void rb_list_head_clear(struct list_head *list)
+{
+        unsigned long *ptr = (unsigned long *)&list->next;
+
+        *ptr &= ~RB_FLAG_MASK;
+}
+
+/*
+ * rb_head_page_dactivate - clears head page ptr (for free list)
+ */
+static void
+rb_head_page_deactivate(struct ring_buffer_per_cpu *cpu_buffer)
+{
+        struct list_head *hd;
+
+        /* Go through the whole list and clear any pointers found. */
+        rb_list_head_clear(cpu_buffer->pages);
+
+        list_for_each(hd, cpu_buffer->pages)
+                rb_list_head_clear(hd);
+}
+
+static int rb_head_page_set(struct ring_buffer_per_cpu *cpu_buffer,
+                            struct buffer_page *head,
+                            struct buffer_page *prev,
+                            int old_flag, int new_flag)
+{
+        struct list_head *list;
+        unsigned long val = (unsigned long)&head->list;
+        unsigned long ret;
+
+        list = &prev->list;
+
+        val &= ~RB_FLAG_MASK;
+
+        ret = cmpxchg((unsigned long *)&list->next,
+                      val | old_flag, val | new_flag);
+
+        /* check if the reader took the page */
+        if ((ret & ~RB_FLAG_MASK) != val)
+                return RB_PAGE_MOVED;
+
+        return ret & RB_FLAG_MASK;
+}
+
+static int rb_head_page_set_update(struct ring_buffer_per_cpu *cpu_buffer,
+                                   struct buffer_page *head,
+                                   struct buffer_page *prev,
+                                   int old_flag)
+{
+        return rb_head_page_set(cpu_buffer, head, prev,
+                                old_flag, RB_PAGE_UPDATE);
+}
+
+static int rb_head_page_set_head(struct ring_buffer_per_cpu *cpu_buffer,
+                                 struct buffer_page *head,
+                                 struct buffer_page *prev,
+                                 int old_flag)
+{
+        return rb_head_page_set(cpu_buffer, head, prev,
+                                old_flag, RB_PAGE_HEAD);
+}
+
+static int rb_head_page_set_normal(struct ring_buffer_per_cpu *cpu_buffer,
+                                   struct buffer_page *head,
+                                   struct buffer_page *prev,
+                                   int old_flag)
+{
+        return rb_head_page_set(cpu_buffer, head, prev,
+                                old_flag, RB_PAGE_NORMAL);
+}
+
+static inline void rb_inc_page(struct ring_buffer_per_cpu *cpu_buffer,
+                               struct buffer_page **bpage)
+{
+        struct list_head *p = rb_list_head((*bpage)->list.next);
+
+        *bpage = list_entry(p, struct buffer_page, list);
+}
+
+static struct buffer_page *
+rb_set_head_page(struct ring_buffer_per_cpu *cpu_buffer)
+{
+        struct buffer_page *head;
+        struct buffer_page *page;
+        struct list_head *list;
+        int i;
+
+        if (RB_WARN_ON(cpu_buffer, !cpu_buffer->head_page))
+                return NULL;
+
+        /* sanity check */
+        list = cpu_buffer->pages;
+        if (RB_WARN_ON(cpu_buffer, rb_list_head(list->prev->next) != list))
+                return NULL;
+
+        page = head = cpu_buffer->head_page;
+        /*
+         * It is possible that the writer moves the header behind
+         * where we started, and we miss in one loop.
+         * A second loop should grab the header, but we'll do
+         * three loops just because I'm paranoid.
+         */
+        for (i = 0; i < 3; i++) {
+                do {
+                        if (rb_is_head_page(cpu_buffer, page, page->list.prev)) {
+                                cpu_buffer->head_page = page;
+                                return page;
+                        }
+                        rb_inc_page(cpu_buffer, &page);
+                } while (page != head);
+        }
+
+        RB_WARN_ON(cpu_buffer, 1);
+
+        return NULL;
+}
+
+static int rb_head_page_replace(struct buffer_page *old,
+                                struct buffer_page *new)
+{
+        unsigned long *ptr = (unsigned long *)&old->list.prev->next;
+        unsigned long val;
+        unsigned long ret;
+
+        val = *ptr & ~RB_FLAG_MASK;
+        val |= RB_PAGE_HEAD;
+
+        ret = cmpxchg(ptr, val, (unsigned long)&new->list);
+
+        return ret == val;
+}
+
+/*
+ * rb_tail_page_update - move the tail page forward
+ *
+ * Returns 1 if moved tail page, 0 if someone else did.
+ */
+static int rb_tail_page_update(struct ring_buffer_per_cpu *cpu_buffer,
+                               struct buffer_page *tail_page,
+                               struct buffer_page *next_page)
+{
+        struct buffer_page *old_tail;
+        unsigned long old_entries;
+        unsigned long old_write;
+        int ret = 0;
+
+        /*
+         * The tail page now needs to be moved forward.
+         *
+         * We need to reset the tail page, but without messing
+         * with possible erasing of data brought in by interrupts
+         * that have moved the tail page and are currently on it.
+         *
+         * We add a counter to the write field to denote this.
+         */
+        old_write = local_add_return(RB_WRITE_INTCNT, &next_page->write);
+        old_entries = local_add_return(RB_WRITE_INTCNT, &next_page->entries);
+
+        /*
+         * Just make sure we have seen our old_write and synchronize
+         * with any interrupts that come in.
+         */
+        barrier();
+
+        /*
+         * If the tail page is still the same as what we think
+         * it is, then it is up to us to update the tail
+         * pointer.
+         */
+        if (tail_page == cpu_buffer->tail_page) {
+                /* Zero the write counter */
+                unsigned long val = old_write & ~RB_WRITE_MASK;
+                unsigned long eval = old_entries & ~RB_WRITE_MASK;
+
+                /*
+                 * This will only succeed if an interrupt did
+                 * not come in and change it. In which case, we
+                 * do not want to modify it.
+                 *
+                 * We add (void) to let the compiler know that we do not care
+                 * about the return value of these functions. We use the
+                 * cmpxchg to only update if an interrupt did not already
+                 * do it for us. If the cmpxchg fails, we don't care.
+                 */
+                (void)local_cmpxchg(&next_page->write, old_write, val);
+                (void)local_cmpxchg(&next_page->entries, old_entries, eval);
+
+                /*
+                 * No need to worry about races with clearing out the commit.
+                 * it only can increment when a commit takes place. But that
+                 * only happens in the outer most nested commit.
+                 */
+                local_set(&next_page->page->commit, 0);
+
+                old_tail = cmpxchg(&cpu_buffer->tail_page,
+                                   tail_page, next_page);
+
+                if (old_tail == tail_page)
+                        ret = 1;
+        }
+
+        return ret;
+}
+
+static int rb_check_bpage(struct ring_buffer_per_cpu *cpu_buffer,
+                          struct buffer_page *bpage)
+{
+        unsigned long val = (unsigned long)bpage;
+
+        if (RB_WARN_ON(cpu_buffer, val & RB_FLAG_MASK))
+                return 1;
+
+        return 0;
+}
+
+/**
+ * rb_check_list - make sure a pointer to a list has the last bits zero
+ */
+static int rb_check_list(struct ring_buffer_per_cpu *cpu_buffer,
+                         struct list_head *list)
+{
+        if (RB_WARN_ON(cpu_buffer, rb_list_head(list->prev) != list->prev))
+                return 1;
+        if (RB_WARN_ON(cpu_buffer, rb_list_head(list->next) != list->next))
+                return 1;
+        return 0;
+}
+
 /**
  * check_pages - integrity check of buffer pages
  * @cpu_buffer: CPU buffer with pages to test
@@ -494,14 +904,19 @@ EXPORT_SYMBOL_GPL(ring_buffer_normalize_time_stamp);
  */
 static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer)
 {
-        struct list_head *head = &cpu_buffer->pages;
+        struct list_head *head = cpu_buffer->pages;
         struct buffer_page *bpage, *tmp;
 
+        rb_head_page_deactivate(cpu_buffer);
+
         if (RB_WARN_ON(cpu_buffer, head->next->prev != head))
                 return -1;
         if (RB_WARN_ON(cpu_buffer, head->prev->next != head))
                 return -1;
 
+        if (rb_check_list(cpu_buffer, head))
+                return -1;
+
         list_for_each_entry_safe(bpage, tmp, head, list) {
                 if (RB_WARN_ON(cpu_buffer,
                                bpage->list.next->prev != &bpage->list))
@@ -509,25 +924,33 @@ static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer)
                 if (RB_WARN_ON(cpu_buffer,
                                bpage->list.prev->next != &bpage->list))
                         return -1;
+                if (rb_check_list(cpu_buffer, &bpage->list))
+                        return -1;
         }
 
+        rb_head_page_activate(cpu_buffer);
+
         return 0;
 }
 
 static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
                              unsigned nr_pages)
 {
-        struct list_head *head = &cpu_buffer->pages;
         struct buffer_page *bpage, *tmp;
         unsigned long addr;
         LIST_HEAD(pages);
         unsigned i;
 
+        WARN_ON(!nr_pages);
+
         for (i = 0; i < nr_pages; i++) {
                 bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),
                                     GFP_KERNEL, cpu_to_node(cpu_buffer->cpu));
                 if (!bpage)
                         goto free_pages;
+
+                rb_check_bpage(cpu_buffer, bpage);
+
                 list_add(&bpage->list, &pages);
 
                 addr = __get_free_page(GFP_KERNEL);
@@ -537,7 +960,13 @@ static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
                 rb_init_page(bpage->page);
         }
 
-        list_splice(&pages, head);
+        /*
+         * The ring buffer page list is a circular list that does not
+         * start and end with a list head. All page list items point to
+         * other pages.
+         */
+        cpu_buffer->pages = pages.next;
+        list_del(&pages);
 
         rb_check_pages(cpu_buffer);
 
@@ -569,13 +998,14 @@ rb_allocate_cpu_buffer(struct ring_buffer *buffer, int cpu)
         spin_lock_init(&cpu_buffer->reader_lock);
         lockdep_set_class(&cpu_buffer->reader_lock, buffer->reader_lock_key);
         cpu_buffer->lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
-        INIT_LIST_HEAD(&cpu_buffer->pages);
 
         bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),
                             GFP_KERNEL, cpu_to_node(cpu));
         if (!bpage)
                 goto fail_free_buffer;
 
+        rb_check_bpage(cpu_buffer, bpage);
+
         cpu_buffer->reader_page = bpage;
         addr = __get_free_page(GFP_KERNEL);
         if (!addr)
@@ -590,9 +1020,11 @@ rb_allocate_cpu_buffer(struct ring_buffer *buffer, int cpu)
                 goto fail_free_reader;
 
         cpu_buffer->head_page
-                = list_entry(cpu_buffer->pages.next, struct buffer_page, list);
+                = list_entry(cpu_buffer->pages, struct buffer_page, list);
         cpu_buffer->tail_page = cpu_buffer->commit_page = cpu_buffer->head_page;
 
+        rb_head_page_activate(cpu_buffer);
+
         return cpu_buffer;
 
  fail_free_reader:
@@ -605,24 +1037,25 @@ rb_allocate_cpu_buffer(struct ring_buffer *buffer, int cpu)
 
 static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer)
 {
-        struct list_head *head = &cpu_buffer->pages;
+        struct list_head *head = cpu_buffer->pages;
         struct buffer_page *bpage, *tmp;
 
         free_buffer_page(cpu_buffer->reader_page);
 
-        list_for_each_entry_safe(bpage, tmp, head, list) {
-                list_del_init(&bpage->list);
+        rb_head_page_deactivate(cpu_buffer);
+
+        if (head) {
+                list_for_each_entry_safe(bpage, tmp, head, list) {
+                        list_del_init(&bpage->list);
+                        free_buffer_page(bpage);
+                }
+                bpage = list_entry(head, struct buffer_page, list);
                 free_buffer_page(bpage);
         }
+
         kfree(cpu_buffer);
 }
 
-/*
- * Causes compile errors if the struct buffer_page gets bigger
- * than the struct page.
- */
-extern int ring_buffer_page_too_big(void);
-
 #ifdef CONFIG_HOTPLUG_CPU
 static int rb_cpu_notify(struct notifier_block *self,
                          unsigned long action, void *hcpu);
@@ -645,11 +1078,6 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags,
         int bsize;
         int cpu;
 
-        /* Paranoid! Optimizes out when all is well */
-        if (sizeof(struct buffer_page) > sizeof(struct page))
-                ring_buffer_page_too_big();
-
-
         /* keep it in its own cache line */
         buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()),
                          GFP_KERNEL);
@@ -665,8 +1093,8 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags,
         buffer->reader_lock_key = key;
 
         /* need at least two pages */
-        if (buffer->pages == 1)
-                buffer->pages++;
+        if (buffer->pages < 2)
+                buffer->pages = 2;
 
         /*
          * In case of non-hotplug cpu, if the ring-buffer is allocated
@@ -742,6 +1170,7 @@ ring_buffer_free(struct ring_buffer *buffer)
 
         put_online_cpus();
 
+        kfree(buffer->buffers);
         free_cpumask_var(buffer->cpumask);
 
         kfree(buffer);
@@ -766,15 +1195,17 @@ rb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned nr_pages)
         atomic_inc(&cpu_buffer->record_disabled);
         synchronize_sched();
 
+        rb_head_page_deactivate(cpu_buffer);
+
         for (i = 0; i < nr_pages; i++) {
-                if (RB_WARN_ON(cpu_buffer, list_empty(&cpu_buffer->pages)))
+                if (RB_WARN_ON(cpu_buffer, list_empty(cpu_buffer->pages)))
                         return;
-                p = cpu_buffer->pages.next;
+                p = cpu_buffer->pages->next;
                 bpage = list_entry(p, struct buffer_page, list);
                 list_del_init(&bpage->list);
                 free_buffer_page(bpage);
         }
-        if (RB_WARN_ON(cpu_buffer, list_empty(&cpu_buffer->pages)))
+        if (RB_WARN_ON(cpu_buffer, list_empty(cpu_buffer->pages)))
                 return;
 
         rb_reset_cpu(cpu_buffer);
@@ -796,15 +1227,19 @@ rb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer,
         atomic_inc(&cpu_buffer->record_disabled);
         synchronize_sched();
 
+        spin_lock_irq(&cpu_buffer->reader_lock);
+        rb_head_page_deactivate(cpu_buffer);
+
         for (i = 0; i < nr_pages; i++) {
                 if (RB_WARN_ON(cpu_buffer, list_empty(pages)))
                         return;
                 p = pages->next;
                 bpage = list_entry(p, struct buffer_page, list);
                 list_del_init(&bpage->list);
-                list_add_tail(&bpage->list, &cpu_buffer->pages);
+                list_add_tail(&bpage->list, cpu_buffer->pages);
         }
         rb_reset_cpu(cpu_buffer);
+        spin_unlock_irq(&cpu_buffer->reader_lock);
 
         rb_check_pages(cpu_buffer);
 
@@ -955,21 +1390,14 @@ rb_reader_event(struct ring_buffer_per_cpu *cpu_buffer)
 }
 
 static inline struct ring_buffer_event *
-rb_head_event(struct ring_buffer_per_cpu *cpu_buffer)
-{
-        return __rb_page_index(cpu_buffer->head_page,
-                               cpu_buffer->head_page->read);
-}
-
-static inline struct ring_buffer_event *
 rb_iter_head_event(struct ring_buffer_iter *iter)
 {
         return __rb_page_index(iter->head_page, iter->head);
 }
 
-static inline unsigned rb_page_write(struct buffer_page *bpage)
+static inline unsigned long rb_page_write(struct buffer_page *bpage)
 {
-        return local_read(&bpage->write);
+        return local_read(&bpage->write) & RB_WRITE_MASK;
 }
 
 static inline unsigned rb_page_commit(struct buffer_page *bpage)
@@ -977,6 +1405,11 @@ static inline unsigned rb_page_commit(struct buffer_page *bpage)
         return local_read(&bpage->page->commit);
 }
 
+static inline unsigned long rb_page_entries(struct buffer_page *bpage)
+{
+        return local_read(&bpage->entries) & RB_WRITE_MASK;
+}
+
 /* Size is determined by what has been commited */
 static inline unsigned rb_page_size(struct buffer_page *bpage)
 {
@@ -989,33 +1422,17 @@ rb_commit_index(struct ring_buffer_per_cpu *cpu_buffer)
         return rb_page_commit(cpu_buffer->commit_page);
 }
 
-static inline unsigned rb_head_size(struct ring_buffer_per_cpu *cpu_buffer)
-{
-        return rb_page_commit(cpu_buffer->head_page);
-}
-
-static inline void rb_inc_page(struct ring_buffer_per_cpu *cpu_buffer,
-                               struct buffer_page **bpage)
-{
-        struct list_head *p = (*bpage)->list.next;
-
-        if (p == &cpu_buffer->pages)
-                p = p->next;
-
-        *bpage = list_entry(p, struct buffer_page, list);
-}
-
 static inline unsigned
 rb_event_index(struct ring_buffer_event *event)
 {
         unsigned long addr = (unsigned long)event;
 
-        return (addr & ~PAGE_MASK) - (PAGE_SIZE - BUF_PAGE_SIZE);
+        return (addr & ~PAGE_MASK) - BUF_PAGE_HDR_SIZE;
 }
 
 static inline int
-rb_is_commit(struct ring_buffer_per_cpu *cpu_buffer,
-             struct ring_buffer_event *event)
+rb_event_is_commit(struct ring_buffer_per_cpu *cpu_buffer,
+                   struct ring_buffer_event *event)
 {
         unsigned long addr = (unsigned long)event;
         unsigned long index;
@@ -1028,33 +1445,10 @@ rb_is_commit(struct ring_buffer_per_cpu *cpu_buffer,
 }
 
 static void
-rb_set_commit_event(struct ring_buffer_per_cpu *cpu_buffer,
-                    struct ring_buffer_event *event)
-{
-        unsigned long addr = (unsigned long)event;
-        unsigned long index;
-
-        index = rb_event_index(event);
-        addr &= PAGE_MASK;
-
-        while (cpu_buffer->commit_page->page != (void *)addr) {
-                if (RB_WARN_ON(cpu_buffer,
-                          cpu_buffer->commit_page == cpu_buffer->tail_page))
-                        return;
-                cpu_buffer->commit_page->page->commit =
-                        cpu_buffer->commit_page->write;
-                rb_inc_page(cpu_buffer, &cpu_buffer->commit_page);
-                cpu_buffer->write_stamp =
-                        cpu_buffer->commit_page->page->time_stamp;
-        }
-
-        /* Now set the commit to the event's index */
-        local_set(&cpu_buffer->commit_page->page->commit, index);
-}
-
-static void
 rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer)
 {
+        unsigned long max_count;
+
         /*
          * We only race with interrupts and NMIs on this CPU.
          * If we own the commit event, then we can commit
@@ -1064,9 +1458,16 @@ rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer)
          * assign the commit to the tail.
          */
  again:
+        max_count = cpu_buffer->buffer->pages * 100;
+
         while (cpu_buffer->commit_page != cpu_buffer->tail_page) {
-                cpu_buffer->commit_page->page->commit =
-                        cpu_buffer->commit_page->write;
+                if (RB_WARN_ON(cpu_buffer, !(--max_count)))
+                        return;
+                if (RB_WARN_ON(cpu_buffer,
+                               rb_is_reader_page(cpu_buffer->tail_page)))
+                        return;
+                local_set(&cpu_buffer->commit_page->page->commit,
+                          rb_page_write(cpu_buffer->commit_page));
                 rb_inc_page(cpu_buffer, &cpu_buffer->commit_page);
                 cpu_buffer->write_stamp =
                         cpu_buffer->commit_page->page->time_stamp;
@@ -1075,8 +1476,12 @@ rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer)
         }
         while (rb_commit_index(cpu_buffer) !=
                rb_page_write(cpu_buffer->commit_page)) {
-                cpu_buffer->commit_page->page->commit =
-                        cpu_buffer->commit_page->write;
+
+                local_set(&cpu_buffer->commit_page->page->commit,
+                          rb_page_write(cpu_buffer->commit_page));
+                RB_WARN_ON(cpu_buffer,
+                           local_read(&cpu_buffer->commit_page->page->commit) &
+                           ~RB_WRITE_MASK);
                 barrier();
         }
 
@@ -1109,7 +1514,7 @@ static void rb_inc_iter(struct ring_buffer_iter *iter)
          * to the head page instead of next.
          */
         if (iter->head_page == cpu_buffer->reader_page)
-                iter->head_page = cpu_buffer->head_page;
+                iter->head_page = rb_set_head_page(cpu_buffer);
         else
                 rb_inc_page(cpu_buffer, &iter->head_page);
 
@@ -1153,6 +1558,163 @@ rb_update_event(struct ring_buffer_event *event,
         }
 }
 
+/*
+ * rb_handle_head_page - writer hit the head page
+ *
+ * Returns: +1 to retry page
+ *           0 to continue
+ *          -1 on error
+ */
+static int
+rb_handle_head_page(struct ring_buffer_per_cpu *cpu_buffer,
+                    struct buffer_page *tail_page,
+                    struct buffer_page *next_page)
+{
+        struct buffer_page *new_head;
+        int entries;
+        int type;
+        int ret;
+
+        entries = rb_page_entries(next_page);
+
+        /*
+         * The hard part is here. We need to move the head
+         * forward, and protect against both readers on
+         * other CPUs and writers coming in via interrupts.
+         */
+        type = rb_head_page_set_update(cpu_buffer, next_page, tail_page,
+                                       RB_PAGE_HEAD);
+
+        /*
+         * type can be one of four:
+         *  NORMAL - an interrupt already moved it for us
+         *  HEAD   - we are the first to get here.
+         *  UPDATE - we are the interrupt interrupting
+         *           a current move.
+         *  MOVED  - a reader on another CPU moved the next
+         *           pointer to its reader page. Give up
+         *           and try again.
+         */
+
+        switch (type) {
+        case RB_PAGE_HEAD:
+                /*
+                 * We changed the head to UPDATE, thus
+                 * it is our responsibility to update
+                 * the counters.
+                 */
+                local_add(entries, &cpu_buffer->overrun);
+
+                /*
+                 * The entries will be zeroed out when we move the
+                 * tail page.
+                 */
+
+                /* still more to do */
+                break;
+
+        case RB_PAGE_UPDATE:
+                /*
+                 * This is an interrupt that interrupt the
+                 * previous update. Still more to do.
+                 */
+                break;
+        case RB_PAGE_NORMAL:
+                /*
+                 * An interrupt came in before the update
+                 * and processed this for us.
+                 * Nothing left to do.
+                 */
+                return 1;
+        case RB_PAGE_MOVED:
+                /*
+                 * The reader is on another CPU and just did
+                 * a swap with our next_page.
+                 * Try again.
+                 */
+                return 1;
+        default:
+                RB_WARN_ON(cpu_buffer, 1); /* WTF??? */
+                return -1;
+        }
+
+        /*
+         * Now that we are here, the old head pointer is
+         * set to UPDATE. This will keep the reader from
+         * swapping the head page with the reader page.
+         * The reader (on another CPU) will spin till
+         * we are finished.
+         *
+         * We just need to protect against interrupts
+         * doing the job. We will set the next pointer
+         * to HEAD. After that, we set the old pointer
+         * to NORMAL, but only if it was HEAD before.
+         * otherwise we are an interrupt, and only
+         * want the outer most commit to reset it.
+         */
+        new_head = next_page;
+        rb_inc_page(cpu_buffer, &new_head);
+
+        ret = rb_head_page_set_head(cpu_buffer, new_head, next_page,
+                                    RB_PAGE_NORMAL);
+
+        /*
+         * Valid returns are:
+         *  HEAD   - an interrupt came in and already set it.
+         *  NORMAL - One of two things:
+         *            1) We really set it.
+         *            2) A bunch of interrupts came in and moved
+         *               the page forward again.
+         */
+        switch (ret) {
+        case RB_PAGE_HEAD:
+        case RB_PAGE_NORMAL:
+                /* OK */
+                break;
+        default:
+                RB_WARN_ON(cpu_buffer, 1);
+                return -1;
+        }
+
+        /*
+         * It is possible that an interrupt came in,
+         * set the head up, then more interrupts came in
+         * and moved it again. When we get back here,
+         * the page would have been set to NORMAL but we
+         * just set it back to HEAD.
+         *
+         * How do you detect this? Well, if that happened
+         * the tail page would have moved.
+         */
+        if (ret == RB_PAGE_NORMAL) {
+                /*
+                 * If the tail had moved passed next, then we need
+                 * to reset the pointer.
+                 */
+                if (cpu_buffer->tail_page != tail_page &&
+                    cpu_buffer->tail_page != next_page)
+                        rb_head_page_set_normal(cpu_buffer, new_head,
+                                                next_page,
+                                                RB_PAGE_HEAD);
+        }
+
+        /*
+         * If this was the outer most commit (the one that
+         * changed the original pointer from HEAD to UPDATE),
+         * then it is up to us to reset it to NORMAL.
+         */
+        if (type == RB_PAGE_HEAD) {
+                ret = rb_head_page_set_normal(cpu_buffer, next_page,
+                                              tail_page,
+                                              RB_PAGE_UPDATE);
+                if (RB_WARN_ON(cpu_buffer,
+                               ret != RB_PAGE_UPDATE))
+                        return -1;
+        }
+
+        return 0;
+}
+
 static unsigned rb_calculate_event_length(unsigned length)
 {
         struct ring_buffer_event event; /* Used only for sizeof array */
@@ -1170,6 +1732,57 @@ static unsigned rb_calculate_event_length(unsigned length)
         return length;
 }
 
+static inline void
+rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer,
+              struct buffer_page *tail_page,
+              unsigned long tail, unsigned long length)
+{
+        struct ring_buffer_event *event;
+
+        /*
+         * Only the event that crossed the page boundary
+         * must fill the old tail_page with padding.
+         */
+        if (tail >= BUF_PAGE_SIZE) {
+                local_sub(length, &tail_page->write);
+                return;
+        }
+
+        event = __rb_page_index(tail_page, tail);
+        kmemcheck_annotate_bitfield(event, bitfield);
+
+        /*
+         * If this event is bigger than the minimum size, then
+         * we need to be careful that we don't subtract the
+         * write counter enough to allow another writer to slip
+         * in on this page.
+         * We put in a discarded commit instead, to make sure
+         * that this space is not used again.
+         *
+         * If we are less than the minimum size, we don't need to
+         * worry about it.
+         */
+        if (tail > (BUF_PAGE_SIZE - RB_EVNT_MIN_SIZE)) {
+                /* No room for any events */
+
+                /* Mark the rest of the page with padding */
+                rb_event_set_padding(event);
+
+                /* Set the write back to the previous setting */
+                local_sub(length, &tail_page->write);
+                return;
+        }
+
+        /* Put in a discarded event */
+        event->array[0] = (BUF_PAGE_SIZE - tail) - RB_EVNT_HDR_SIZE;
+        event->type_len = RINGBUF_TYPE_PADDING;
+        /* time delta must be non zero */
+        event->time_delta = 1;
+
+        /* Set write to end of buffer */
+        length = (tail + length) - BUF_PAGE_SIZE;
+        local_sub(length, &tail_page->write);
+}
 
 static struct ring_buffer_event *
 rb_move_tail(struct ring_buffer_per_cpu *cpu_buffer,
@@ -1177,127 +1790,101 @@ rb_move_tail(struct ring_buffer_per_cpu *cpu_buffer,
              struct buffer_page *commit_page,
              struct buffer_page *tail_page, u64 *ts)
 {
-        struct buffer_page *next_page, *head_page, *reader_page;
         struct ring_buffer *buffer = cpu_buffer->buffer;
-        struct ring_buffer_event *event;
-        bool lock_taken = false;
-        unsigned long flags;
+        struct buffer_page *next_page;
+        int ret;
 
         next_page = tail_page;
 
-        local_irq_save(flags);
-        /*
-         * Since the write to the buffer is still not
-         * fully lockless, we must be careful with NMIs.
-         * The locks in the writers are taken when a write
-         * crosses to a new page. The locks protect against
-         * races with the readers (this will soon be fixed
-         * with a lockless solution).
-         *
-         * Because we can not protect against NMIs, and we
-         * want to keep traces reentrant, we need to manage
-         * what happens when we are in an NMI.
-         *
-         * NMIs can happen after we take the lock.
-         * If we are in an NMI, only take the lock
-         * if it is not already taken. Otherwise
-         * simply fail.
-         */
-        if (unlikely(in_nmi())) {
-                if (!__raw_spin_trylock(&cpu_buffer->lock)) {
-                        cpu_buffer->nmi_dropped++;
-                        goto out_reset;
-                }
-        } else
-                __raw_spin_lock(&cpu_buffer->lock);
-
-        lock_taken = true;
-
         rb_inc_page(cpu_buffer, &next_page);
 
-        head_page = cpu_buffer->head_page;
-        reader_page = cpu_buffer->reader_page;
-
-        /* we grabbed the lock before incrementing */
-        if (RB_WARN_ON(cpu_buffer, next_page == reader_page))
-                goto out_reset;
-
         /*
          * If for some reason, we had an interrupt storm that made
          * it all the way around the buffer, bail, and warn
          * about it.
          */
         if (unlikely(next_page == commit_page)) {
-                cpu_buffer->commit_overrun++;
+                local_inc(&cpu_buffer->commit_overrun);
                 goto out_reset;
         }
 
-        if (next_page == head_page) {
-                if (!(buffer->flags & RB_FL_OVERWRITE))
-                        goto out_reset;
-
-                /* tail_page has not moved yet? */
-                if (tail_page == cpu_buffer->tail_page) {
-                        /* count overflows */
-                        cpu_buffer->overrun +=
-                                local_read(&head_page->entries);
-
-                        rb_inc_page(cpu_buffer, &head_page);
-                        cpu_buffer->head_page = head_page;
-                        cpu_buffer->head_page->read = 0;
-                }
-        }
-
         /*
-         * If the tail page is still the same as what we think
-         * it is, then it is up to us to update the tail
-         * pointer.
+         * This is where the fun begins!
+         *
+         * We are fighting against races between a reader that
+         * could be on another CPU trying to swap its reader
+         * page with the buffer head.
+         *
+         * We are also fighting against interrupts coming in and
+         * moving the head or tail on us as well.
+         *
+         * If the next page is the head page then we have filled
+         * the buffer, unless the commit page is still on the
+         * reader page.
          */
-        if (tail_page == cpu_buffer->tail_page) {
-                local_set(&next_page->write, 0);
-                local_set(&next_page->entries, 0);
-                local_set(&next_page->page->commit, 0);
-                cpu_buffer->tail_page = next_page;
+        if (rb_is_head_page(cpu_buffer, next_page, &tail_page->list)) {
 
-                /* reread the time stamp */
-                *ts = rb_time_stamp(buffer, cpu_buffer->cpu);
-                cpu_buffer->tail_page->page->time_stamp = *ts;
+                /*
+                 * If the commit is not on the reader page, then
+                 * move the header page.
+                 */
+                if (!rb_is_reader_page(cpu_buffer->commit_page)) {
+                        /*
+                         * If we are not in overwrite mode,
+                         * this is easy, just stop here.
+                         */
+                        if (!(buffer->flags & RB_FL_OVERWRITE))
+                                goto out_reset;
+
+                        ret = rb_handle_head_page(cpu_buffer,
+                                                  tail_page,
+                                                  next_page);
+                        if (ret < 0)
+                                goto out_reset;
+                        if (ret)
+                                goto out_again;
+                } else {
+                        /*
+                         * We need to be careful here too. The
+                         * commit page could still be on the reader
+                         * page. We could have a small buffer, and
+                         * have filled up the buffer with events
+                         * from interrupts and such, and wrapped.
+                         *
+                         * Note, if the tail page is also the on the
+                         * reader_page, we let it move out.
+                         */
+                        if (unlikely((cpu_buffer->commit_page !=
+                                      cpu_buffer->tail_page) &&
+                                     (cpu_buffer->commit_page ==
+                                      cpu_buffer->reader_page))) {
+                                local_inc(&cpu_buffer->commit_overrun);
+                                goto out_reset;
+                        }
+                }
         }
 
-        /*
-         * The actual tail page has moved forward.
-         */
-        if (tail < BUF_PAGE_SIZE) {
-                /* Mark the rest of the page with padding */
-                event = __rb_page_index(tail_page, tail);
-                rb_event_set_padding(event);
+        ret = rb_tail_page_update(cpu_buffer, tail_page, next_page);
+        if (ret) {
+                /*
+                 * Nested commits always have zero deltas, so
+                 * just reread the time stamp
+                 */
+                *ts = rb_time_stamp(buffer, cpu_buffer->cpu);
+                next_page->page->time_stamp = *ts;
         }
 
-        /* Set the write back to the previous setting */
-        local_sub(length, &tail_page->write);
+ out_again:
 
-        /*
-         * If this was a commit entry that failed,
-         * increment that too
-         */
-        if (tail_page == cpu_buffer->commit_page &&
-            tail == rb_commit_index(cpu_buffer)) {
-                rb_set_commit_to_write(cpu_buffer);
-        }
-
-        __raw_spin_unlock(&cpu_buffer->lock);
-        local_irq_restore(flags);
+        rb_reset_tail(cpu_buffer, tail_page, tail, length);
 
         /* fail and let the caller try again */
         return ERR_PTR(-EAGAIN);
 
  out_reset:
         /* reset write */
-        local_sub(length, &tail_page->write);
+        rb_reset_tail(cpu_buffer, tail_page, tail, length);
 
-        if (likely(lock_taken))
-                __raw_spin_unlock(&cpu_buffer->lock);
-        local_irq_restore(flags);
         return NULL;
 }
 
@@ -1314,6 +1901,9 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
         barrier();
         tail_page = cpu_buffer->tail_page;
         write = local_add_return(length, &tail_page->write);
+
+        /* set write to only the index of the write */
+        write &= RB_WRITE_MASK;
         tail = write - length;
 
         /* See if we shot pass the end of this buffer page */
@@ -1323,10 +1913,8 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
 
         /* We reserved something on the buffer */
 
-        if (RB_WARN_ON(cpu_buffer, write > BUF_PAGE_SIZE))
-                return NULL;
-
         event = __rb_page_index(tail_page, tail);
+        kmemcheck_annotate_bitfield(event, bitfield);
         rb_update_event(event, type, length);
 
         /* The passed in type is zero for DATA */
@@ -1334,11 +1922,11 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
                 local_inc(&tail_page->entries);
 
         /*
-         * If this is a commit and the tail is zero, then update
-         * this page's time stamp.
+         * If this is the first commit on the page, then update
+         * its timestamp.
          */
-        if (!tail && rb_is_commit(cpu_buffer, event))
-                cpu_buffer->commit_page->page->time_stamp = *ts;
+        if (!tail)
+                tail_page->page->time_stamp = *ts;
 
         return event;
 }
@@ -1360,12 +1948,16 @@ rb_try_to_discard(struct ring_buffer_per_cpu *cpu_buffer,
         bpage = cpu_buffer->tail_page;
 
         if (bpage->page == (void *)addr && rb_page_write(bpage) == old_index) {
+                unsigned long write_mask =
+                        local_read(&bpage->write) & ~RB_WRITE_MASK;
                 /*
                  * This is on the tail page. It is possible that
                  * a write could come in and move the tail page
                  * and write to the next page. That is fine
                  * because we just shorten what is on this page.
                  */
+                old_index += write_mask;
+                new_index += write_mask;
                 index = local_cmpxchg(&bpage->write, old_index, new_index);
                 if (index == old_index)
                         return 1;
@@ -1407,16 +1999,16 @@ rb_add_time_stamp(struct ring_buffer_per_cpu *cpu_buffer,
                 return -EAGAIN;
 
         /* Only a commited time event can update the write stamp */
-        if (rb_is_commit(cpu_buffer, event)) {
+        if (rb_event_is_commit(cpu_buffer, event)) {
                 /*
-                 * If this is the first on the page, then we need to
-                 * update the page itself, and just put in a zero.
+                 * If this is the first on the page, then it was
+                 * updated with the page itself. Try to discard it
+                 * and if we can't just make it zero.
                  */
                 if (rb_event_index(event)) {
                         event->time_delta = *delta & TS_MASK;
                         event->array[0] = *delta >> TS_SHIFT;
                 } else {
-                        cpu_buffer->commit_page->page->time_stamp = *ts;
                         /* try to discard, since we do not need this */
                         if (!rb_try_to_discard(cpu_buffer, event)) {
                                 /* nope, just zero it */
@@ -1442,8 +2034,47 @@ rb_add_time_stamp(struct ring_buffer_per_cpu *cpu_buffer,
         return ret;
 }
 
+static void rb_start_commit(struct ring_buffer_per_cpu *cpu_buffer)
+{
+        local_inc(&cpu_buffer->committing);
+        local_inc(&cpu_buffer->commits);
+}
+
+static void rb_end_commit(struct ring_buffer_per_cpu *cpu_buffer)
+{
+        unsigned long commits;
+
+        if (RB_WARN_ON(cpu_buffer,
+                       !local_read(&cpu_buffer->committing)))
+                return;
+
+ again:
+        commits = local_read(&cpu_buffer->commits);
+        /* synchronize with interrupts */
+        barrier();
+        if (local_read(&cpu_buffer->committing) == 1)
+                rb_set_commit_to_write(cpu_buffer);
+
+        local_dec(&cpu_buffer->committing);
+
+        /* synchronize with interrupts */
+        barrier();
+
+        /*
+         * Need to account for interrupts coming in between the
+         * updating of the commit page and the clearing of the
+         * committing counter.
+         */
+        if (unlikely(local_read(&cpu_buffer->commits) != commits) &&
+            !local_read(&cpu_buffer->committing)) {
+                local_inc(&cpu_buffer->committing);
+                goto again;
+        }
+}
+
 static struct ring_buffer_event *
-rb_reserve_next_event(struct ring_buffer_per_cpu *cpu_buffer,
+rb_reserve_next_event(struct ring_buffer *buffer,
+                      struct ring_buffer_per_cpu *cpu_buffer,
                       unsigned long length)
 {
         struct ring_buffer_event *event;
@@ -1451,6 +2082,23 @@ rb_reserve_next_event(struct ring_buffer_per_cpu *cpu_buffer,
         int commit = 0;
         int nr_loops = 0;
 
+        rb_start_commit(cpu_buffer);
+
+#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
+        /*
+         * Due to the ability to swap a cpu buffer from a buffer
+         * it is possible it was swapped before we committed.
+         * (committing stops a swap). We check for it here and
+         * if it happened, we have to fail the write.
+         */
+        barrier();
+        if (unlikely(ACCESS_ONCE(cpu_buffer->buffer) != buffer)) {
+                local_dec(&cpu_buffer->committing);
+                local_dec(&cpu_buffer->commits);
+                return NULL;
+        }
+#endif
+
         length = rb_calculate_event_length(length);
  again:
         /*
@@ -1463,7 +2111,7 @@ rb_reserve_next_event(struct ring_buffer_per_cpu *cpu_buffer,
          * Bail!
          */
         if (RB_WARN_ON(cpu_buffer, ++nr_loops > 1000))
-                return NULL;
+                goto out_fail;
 
         ts = rb_time_stamp(cpu_buffer->buffer, cpu_buffer->cpu);
 
@@ -1494,7 +2142,7 @@ rb_reserve_next_event(struct ring_buffer_per_cpu *cpu_buffer,
 
                         commit = rb_add_time_stamp(cpu_buffer, &ts, &delta);
                         if (commit == -EBUSY)
-                                return NULL;
+                                goto out_fail;
 
                         if (commit == -EAGAIN)
                                 goto again;
@@ -1508,30 +2156,23 @@ rb_reserve_next_event(struct ring_buffer_per_cpu *cpu_buffer,
         if (unlikely(PTR_ERR(event) == -EAGAIN))
                 goto again;
 
-        if (!event) {
-                if (unlikely(commit))
-                        /*
-                         * Ouch! We needed a timestamp and it was commited. But
-                         * we didn't get our event reserved.
-                         */
-                        rb_set_commit_to_write(cpu_buffer);
-                return NULL;
-        }
+        if (!event)
+                goto out_fail;
 
-        /*
-         * If the timestamp was commited, make the commit our entry
-         * now so that we will update it when needed.
-         */
-        if (unlikely(commit))
-                rb_set_commit_event(cpu_buffer, event);
-        else if (!rb_is_commit(cpu_buffer, event))
+        if (!rb_event_is_commit(cpu_buffer, event))
                 delta = 0;
 
         event->time_delta = delta;
 
         return event;
+
+ out_fail:
+        rb_end_commit(cpu_buffer);
+        return NULL;
 }
 
+#ifdef CONFIG_TRACING
+
 #define TRACE_RECURSIVE_DEPTH 16
 
 static int trace_recursive_lock(void)
@@ -1562,6 +2203,13 @@ static void trace_recursive_unlock(void)
         current->trace_recursion--;
 }
 
+#else
+
+#define trace_recursive_lock()          (0)
+#define trace_recursive_unlock()        do { } while (0)
+
+#endif
+
 static DEFINE_PER_CPU(int, rb_need_resched);
 
 /**
@@ -1611,7 +2259,7 @@ ring_buffer_lock_reserve(struct ring_buffer *buffer, unsigned long length)
         if (length > BUF_MAX_DATA_SIZE)
                 goto out;
 
-        event = rb_reserve_next_event(cpu_buffer, length);
+        event = rb_reserve_next_event(buffer, cpu_buffer, length);
         if (!event)
                 goto out;
 
@@ -1634,18 +2282,24 @@ ring_buffer_lock_reserve(struct ring_buffer *buffer, unsigned long length)
 }
 EXPORT_SYMBOL_GPL(ring_buffer_lock_reserve);
 
+static void
+rb_update_write_stamp(struct ring_buffer_per_cpu *cpu_buffer,
+                      struct ring_buffer_event *event)
+{
+        /*
+         * The event first in the commit queue updates the
+         * time stamp.
+         */
+        if (rb_event_is_commit(cpu_buffer, event))
+                cpu_buffer->write_stamp += event->time_delta;
+}
+
 static void rb_commit(struct ring_buffer_per_cpu *cpu_buffer,
                       struct ring_buffer_event *event)
 {
         local_inc(&cpu_buffer->entries);
-
-        /* Only process further if we own the commit */
-        if (!rb_is_commit(cpu_buffer, event))
-                return;
-
-        cpu_buffer->write_stamp += event->time_delta;
-
-        rb_set_commit_to_write(cpu_buffer);
+        rb_update_write_stamp(cpu_buffer, event);
+        rb_end_commit(cpu_buffer);
 }
 
 /**
@@ -1691,32 +2345,57 @@ static inline void rb_event_discard(struct ring_buffer_event *event)
                 event->time_delta = 1;
 }
 
-/**
- * ring_buffer_event_discard - discard any event in the ring buffer
- * @event: the event to discard
- *
- * Sometimes a event that is in the ring buffer needs to be ignored.
- * This function lets the user discard an event in the ring buffer
- * and then that event will not be read later.
- *
- * Note, it is up to the user to be careful with this, and protect
- * against races. If the user discards an event that has been consumed
- * it is possible that it could corrupt the ring buffer.
+/*
+ * Decrement the entries to the page that an event is on.
+ * The event does not even need to exist, only the pointer
+ * to the page it is on. This may only be called before the commit
+ * takes place.
  */
-void ring_buffer_event_discard(struct ring_buffer_event *event)
+static inline void
+rb_decrement_entry(struct ring_buffer_per_cpu *cpu_buffer,
+                   struct ring_buffer_event *event)
 {
-        rb_event_discard(event);
+        unsigned long addr = (unsigned long)event;
+        struct buffer_page *bpage = cpu_buffer->commit_page;
+        struct buffer_page *start;
+
+        addr &= PAGE_MASK;
+
+        /* Do the likely case first */
+        if (likely(bpage->page == (void *)addr)) {
+                local_dec(&bpage->entries);
+                return;
+        }
+
+        /*
+         * Because the commit page may be on the reader page we
+         * start with the next page and check the end loop there.
+         */
+        rb_inc_page(cpu_buffer, &bpage);
+        start = bpage;
+        do {
+                if (bpage->page == (void *)addr) {
+                        local_dec(&bpage->entries);
+                        return;
+                }
+                rb_inc_page(cpu_buffer, &bpage);
+        } while (bpage != start);
+
+        /* commit not part of this buffer?? */
+        RB_WARN_ON(cpu_buffer, 1);
 }
-EXPORT_SYMBOL_GPL(ring_buffer_event_discard);
 
 /**
  * ring_buffer_commit_discard - discard an event that has not been committed
  * @buffer: the ring buffer
  * @event: non committed event to discard
  *
- * This is similar to ring_buffer_event_discard but must only be
- * performed on an event that has not been committed yet. The difference
- * is that this will also try to free the event from the ring buffer
+ * Sometimes an event that is in the ring buffer needs to be ignored.
+ * This function lets the user discard an event in the ring buffer
+ * and then that event will not be read later.
+ *
+ * This function only works if it is called before the the item has been
+ * committed. It will try to free the event from the ring buffer
  * if another event has not been added behind it.
  *
  * If another event has been added behind it, it will set the event
@@ -1734,32 +2413,27 @@ void ring_buffer_discard_commit(struct ring_buffer *buffer,
         /* The event is discarded regardless */
         rb_event_discard(event);
 
+        cpu = smp_processor_id();
+        cpu_buffer = buffer->buffers[cpu];
+
         /*
          * This must only be called if the event has not been
          * committed yet. Thus we can assume that preemption
          * is still disabled.
          */
-        RB_WARN_ON(buffer, preemptible());
-
-        cpu = smp_processor_id();
-        cpu_buffer = buffer->buffers[cpu];
+        RB_WARN_ON(buffer, !local_read(&cpu_buffer->committing));
 
-        if (!rb_try_to_discard(cpu_buffer, event))
+        rb_decrement_entry(cpu_buffer, event);
+        if (rb_try_to_discard(cpu_buffer, event))
                 goto out;
 
         /*
          * The commit is still visible by the reader, so we
-         * must increment entries.
+         * must still update the timestamp.
          */
-        local_inc(&cpu_buffer->entries);
+        rb_update_write_stamp(cpu_buffer, event);
  out:
-        /*
-         * If a write came in and pushed the tail page
-         * we still need to update the commit pointer
-         * if we were the commit.
-         */
-        if (rb_is_commit(cpu_buffer, event))
-                rb_set_commit_to_write(cpu_buffer);
+        rb_end_commit(cpu_buffer);
 
         trace_recursive_unlock();
 
@@ -1818,7 +2492,7 @@ int ring_buffer_write(struct ring_buffer *buffer,
         if (length > BUF_MAX_DATA_SIZE)
                 goto out;
 
-        event = rb_reserve_next_event(cpu_buffer, length);
+        event = rb_reserve_next_event(buffer, cpu_buffer, length);
         if (!event)
                 goto out;
 
@@ -1839,9 +2513,13 @@ EXPORT_SYMBOL_GPL(ring_buffer_write);
 static int rb_per_cpu_empty(struct ring_buffer_per_cpu *cpu_buffer)
 {
         struct buffer_page *reader = cpu_buffer->reader_page;
-        struct buffer_page *head = cpu_buffer->head_page;
+        struct buffer_page *head = rb_set_head_page(cpu_buffer);
         struct buffer_page *commit = cpu_buffer->commit_page;
 
+        /* In case of error, head will be NULL */
+        if (unlikely(!head))
+                return 1;
+
         return reader->read == rb_page_commit(reader) &&
                 (commit == reader ||
                  (commit == head &&
@@ -1932,7 +2610,7 @@ unsigned long ring_buffer_entries_cpu(struct ring_buffer *buffer, int cpu)
                 return 0;
 
         cpu_buffer = buffer->buffers[cpu];
-        ret = (local_read(&cpu_buffer->entries) - cpu_buffer->overrun)
+        ret = (local_read(&cpu_buffer->entries) - local_read(&cpu_buffer->overrun))
                 - cpu_buffer->read;
 
         return ret;
@@ -1953,33 +2631,13 @@ unsigned long ring_buffer_overrun_cpu(struct ring_buffer *buffer, int cpu)
                 return 0;
 
         cpu_buffer = buffer->buffers[cpu];
-        ret = cpu_buffer->overrun;
+        ret = local_read(&cpu_buffer->overrun);
 
         return ret;
 }
 EXPORT_SYMBOL_GPL(ring_buffer_overrun_cpu);
 
 /**
- * ring_buffer_nmi_dropped_cpu - get the number of nmis that were dropped
- * @buffer: The ring buffer
- * @cpu: The per CPU buffer to get the number of overruns from
- */
-unsigned long ring_buffer_nmi_dropped_cpu(struct ring_buffer *buffer, int cpu)
-{
-        struct ring_buffer_per_cpu *cpu_buffer;
-        unsigned long ret;
-
-        if (!cpumask_test_cpu(cpu, buffer->cpumask))
-                return 0;
-
-        cpu_buffer = buffer->buffers[cpu];
-        ret = cpu_buffer->nmi_dropped;
-
-        return ret;
-}
-EXPORT_SYMBOL_GPL(ring_buffer_nmi_dropped_cpu);
-
-/**
  * ring_buffer_commit_overrun_cpu - get the number of overruns caused by commits
  * @buffer: The ring buffer
  * @cpu: The per CPU buffer to get the number of overruns from
@@ -1994,7 +2652,7 @@ ring_buffer_commit_overrun_cpu(struct ring_buffer *buffer, int cpu)
                 return 0;
 
         cpu_buffer = buffer->buffers[cpu];
-        ret = cpu_buffer->commit_overrun;
+        ret = local_read(&cpu_buffer->commit_overrun);
 
         return ret;
 }
@@ -2017,7 +2675,7 @@ unsigned long ring_buffer_entries(struct ring_buffer *buffer)
         for_each_buffer_cpu(buffer, cpu) {
                 cpu_buffer = buffer->buffers[cpu];
                 entries += (local_read(&cpu_buffer->entries) -
-                            cpu_buffer->overrun) - cpu_buffer->read;
+                            local_read(&cpu_buffer->overrun)) - cpu_buffer->read;
         }
 
         return entries;
@@ -2040,7 +2698,7 @@ unsigned long ring_buffer_overruns(struct ring_buffer *buffer)
         /* if you care about this being correct, lock the buffer */
         for_each_buffer_cpu(buffer, cpu) {
                 cpu_buffer = buffer->buffers[cpu];
-                overruns += cpu_buffer->overrun;
+                overruns += local_read(&cpu_buffer->overrun);
         }
 
         return overruns;
@@ -2053,8 +2711,10 @@ static void rb_iter_reset(struct ring_buffer_iter *iter)
 
         /* Iterator usage is expected to have record disabled */
         if (list_empty(&cpu_buffer->reader_page->list)) {
-                iter->head_page = cpu_buffer->head_page;
-                iter->head = cpu_buffer->head_page->read;
+                iter->head_page = rb_set_head_page(cpu_buffer);
+                if (unlikely(!iter->head_page))
+                        return;
+                iter->head = iter->head_page->read;
         } else {
                 iter->head_page = cpu_buffer->reader_page;
                 iter->head = cpu_buffer->reader_page->read;
@@ -2171,6 +2831,7 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
         struct buffer_page *reader = NULL;
         unsigned long flags;
         int nr_loops = 0;
+        int ret;
 
         local_irq_save(flags);
         __raw_spin_lock(&cpu_buffer->lock);
@@ -2204,30 +2865,56 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
                 goto out;
 
         /*
-         * Splice the empty reader page into the list around the head.
          * Reset the reader page to size zero.
          */
+        local_set(&cpu_buffer->reader_page->write, 0);
+        local_set(&cpu_buffer->reader_page->entries, 0);
+        local_set(&cpu_buffer->reader_page->page->commit, 0);
 
-        reader = cpu_buffer->head_page;
+ spin:
+        /*
+         * Splice the empty reader page into the list around the head.
+         */
+        reader = rb_set_head_page(cpu_buffer);
         cpu_buffer->reader_page->list.next = reader->list.next;
         cpu_buffer->reader_page->list.prev = reader->list.prev;
 
-        local_set(&cpu_buffer->reader_page->write, 0);
-        local_set(&cpu_buffer->reader_page->entries, 0);
-        local_set(&cpu_buffer->reader_page->page->commit, 0);
+        /*
+         * cpu_buffer->pages just needs to point to the buffer, it
+         *  has no specific buffer page to point to. Lets move it out
+         *  of our way so we don't accidently swap it.
+         */
+        cpu_buffer->pages = reader->list.prev;
 
-        /* Make the reader page now replace the head */
-        reader->list.prev->next = &cpu_buffer->reader_page->list;
-        reader->list.next->prev = &cpu_buffer->reader_page->list;
+        /* The reader page will be pointing to the new head */
+        rb_set_list_to_head(cpu_buffer, &cpu_buffer->reader_page->list);
 
         /*
-         * If the tail is on the reader, then we must set the head
-         * to the inserted page, otherwise we set it one before.
+         * Here's the tricky part.
+         *
+         * We need to move the pointer past the header page.
+         * But we can only do that if a writer is not currently
+         * moving it. The page before the header page has the
+         * flag bit '1' set if it is pointing to the page we want.
+         * but if the writer is in the process of moving it
+         * than it will be '2' or already moved '0'.
          */
-        cpu_buffer->head_page = cpu_buffer->reader_page;
 
-        if (cpu_buffer->commit_page != reader)
-                rb_inc_page(cpu_buffer, &cpu_buffer->head_page);
+        ret = rb_head_page_replace(reader, cpu_buffer->reader_page);
+
+        /*
+         * If we did not convert it, then we must try again.
+         */
+        if (!ret)
+                goto spin;
+
+        /*
+         * Yeah! We succeeded in replacing the page.
+         *
+         * Now make the new head point back to the reader page.
+         */
+        reader->list.next->prev = &cpu_buffer->reader_page->list;
+        rb_inc_page(cpu_buffer, &cpu_buffer->head_page);
 
         /* Finally update the reader page to the new head */
         cpu_buffer->reader_page = reader;
@@ -2256,8 +2943,7 @@ static void rb_advance_reader(struct ring_buffer_per_cpu *cpu_buffer)
 
         event = rb_reader_event(cpu_buffer);
 
-        if (event->type_len <= RINGBUF_TYPE_DATA_TYPE_LEN_MAX
-                        || rb_discarded_event(event))
+        if (event->type_len <= RINGBUF_TYPE_DATA_TYPE_LEN_MAX)
                 cpu_buffer->read++;
 
         rb_update_read_stamp(cpu_buffer, event);
@@ -2311,15 +2997,12 @@ static void rb_advance_iter(struct ring_buffer_iter *iter)
 }
 
 static struct ring_buffer_event *
-rb_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts)
+rb_buffer_peek(struct ring_buffer_per_cpu *cpu_buffer, u64 *ts)
 {
-        struct ring_buffer_per_cpu *cpu_buffer;
         struct ring_buffer_event *event;
         struct buffer_page *reader;
         int nr_loops = 0;
 
-        cpu_buffer = buffer->buffers[cpu];
-
  again:
         /*
          * We repeat when a timestamp is encountered. It is possible
@@ -2348,7 +3031,6 @@ rb_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts)
                  * the box. Return the padding, and we will release
                  * the current locks, and try again.
                  */
-                rb_advance_reader(cpu_buffer);
                 return event;
 
         case RINGBUF_TYPE_TIME_EXTEND:
@@ -2364,7 +3046,7 @@ rb_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts)
         case RINGBUF_TYPE_DATA:
                 if (ts) {
                         *ts = cpu_buffer->read_stamp + event->time_delta;
-                        ring_buffer_normalize_time_stamp(buffer,
+                        ring_buffer_normalize_time_stamp(cpu_buffer->buffer,
                                                          cpu_buffer->cpu, ts);
                 }
                 return event;
@@ -2443,6 +3125,21 @@ rb_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
 }
 EXPORT_SYMBOL_GPL(ring_buffer_iter_peek);
 
+static inline int rb_ok_to_lock(void)
+{
+        /*
+         * If an NMI die dumps out the content of the ring buffer
+         * do not grab locks. We also permanently disable the ring
+         * buffer too. A one time deal is all you get from reading
+         * the ring buffer from an NMI.
+         */
+        if (likely(!in_nmi()))
+                return 1;
+
+        tracing_off_permanent();
+        return 0;
+}
+
 /**
  * ring_buffer_peek - peek at the next event to be read
  * @buffer: The ring buffer to read
@@ -2458,19 +3155,25 @@ ring_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts)
         struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
         struct ring_buffer_event *event;
         unsigned long flags;
+        int dolock;
 
         if (!cpumask_test_cpu(cpu, buffer->cpumask))
                 return NULL;
 
+        dolock = rb_ok_to_lock();
  again:
-        spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
-        event = rb_buffer_peek(buffer, cpu, ts);
-        spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
+        local_irq_save(flags);
+        if (dolock)
+                spin_lock(&cpu_buffer->reader_lock);
+        event = rb_buffer_peek(cpu_buffer, ts);
+        if (event && event->type_len == RINGBUF_TYPE_PADDING)
+                rb_advance_reader(cpu_buffer);
+        if (dolock)
+                spin_unlock(&cpu_buffer->reader_lock);
+        local_irq_restore(flags);
 
-        if (event && event->type_len == RINGBUF_TYPE_PADDING) {
-                cpu_relax();
+        if (event && event->type_len == RINGBUF_TYPE_PADDING)
                 goto again;
-        }
 
         return event;
 }
@@ -2495,10 +3198,8 @@ ring_buffer_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
         event = rb_iter_peek(iter, ts);
         spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
 
-        if (event && event->type_len == RINGBUF_TYPE_PADDING) {
-                cpu_relax();
+        if (event && event->type_len == RINGBUF_TYPE_PADDING)
                 goto again;
-        }
 
         return event;
 }
@@ -2517,6 +3218,9 @@ ring_buffer_consume(struct ring_buffer *buffer, int cpu, u64 *ts)
         struct ring_buffer_per_cpu *cpu_buffer;
         struct ring_buffer_event *event = NULL;
         unsigned long flags;
+        int dolock;
+
+        dolock = rb_ok_to_lock();
 
  again:
         /* might be called in atomic */
@@ -2526,24 +3230,23 @@ ring_buffer_consume(struct ring_buffer *buffer, int cpu, u64 *ts)
                 goto out;
 
         cpu_buffer = buffer->buffers[cpu];
-        spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
-
-        event = rb_buffer_peek(buffer, cpu, ts);
-        if (!event)
-                goto out_unlock;
+        local_irq_save(flags);
+        if (dolock)
+                spin_lock(&cpu_buffer->reader_lock);
 
-        rb_advance_reader(cpu_buffer);
+        event = rb_buffer_peek(cpu_buffer, ts);
+        if (event)
+                rb_advance_reader(cpu_buffer);
 
- out_unlock:
-        spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
+        if (dolock)
+                spin_unlock(&cpu_buffer->reader_lock);
+        local_irq_restore(flags);
 
  out:
         preempt_enable();
 
-        if (event && event->type_len == RINGBUF_TYPE_PADDING) {
-                cpu_relax();
+        if (event && event->type_len == RINGBUF_TYPE_PADDING)
                 goto again;
-        }
 
         return event;
 }
@@ -2623,21 +3326,19 @@ ring_buffer_read(struct ring_buffer_iter *iter, u64 *ts)
         struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
         unsigned long flags;
 
- again:
         spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
+ again:
         event = rb_iter_peek(iter, ts);
         if (!event)
                 goto out;
 
+        if (event->type_len == RINGBUF_TYPE_PADDING)
+                goto again;
+
         rb_advance_iter(iter);
  out:
         spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
 
-        if (event && event->type_len == RINGBUF_TYPE_PADDING) {
-                cpu_relax();
-                goto again;
-        }
-
         return event;
 }
 EXPORT_SYMBOL_GPL(ring_buffer_read);
@@ -2655,8 +3356,10 @@ EXPORT_SYMBOL_GPL(ring_buffer_size);
 static void
 rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer)
 {
+        rb_head_page_deactivate(cpu_buffer);
+
         cpu_buffer->head_page
-                = list_entry(cpu_buffer->pages.next, struct buffer_page, list);
+                = list_entry(cpu_buffer->pages, struct buffer_page, list);
         local_set(&cpu_buffer->head_page->write, 0);
         local_set(&cpu_buffer->head_page->entries, 0);
         local_set(&cpu_buffer->head_page->page->commit, 0);
@@ -2672,14 +3375,17 @@ rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer)
         local_set(&cpu_buffer->reader_page->page->commit, 0);
         cpu_buffer->reader_page->read = 0;
 
-        cpu_buffer->nmi_dropped = 0;
-        cpu_buffer->commit_overrun = 0;
-        cpu_buffer->overrun = 0;
-        cpu_buffer->read = 0;
+        local_set(&cpu_buffer->commit_overrun, 0);
+        local_set(&cpu_buffer->overrun, 0);
         local_set(&cpu_buffer->entries, 0);
+        local_set(&cpu_buffer->committing, 0);
+        local_set(&cpu_buffer->commits, 0);
+        cpu_buffer->read = 0;
 
         cpu_buffer->write_stamp = 0;
         cpu_buffer->read_stamp = 0;
+
+        rb_head_page_activate(cpu_buffer);
 }
 
 /**
@@ -2699,12 +3405,16 @@ void ring_buffer_reset_cpu(struct ring_buffer *buffer, int cpu)
 
         spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
 
+        if (RB_WARN_ON(cpu_buffer, local_read(&cpu_buffer->committing)))
+                goto out;
+
         __raw_spin_lock(&cpu_buffer->lock);
 
         rb_reset_cpu(cpu_buffer);
 
         __raw_spin_unlock(&cpu_buffer->lock);
 
+ out:
         spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
 
         atomic_dec(&cpu_buffer->record_disabled);
@@ -2731,12 +3441,25 @@ EXPORT_SYMBOL_GPL(ring_buffer_reset);
 int ring_buffer_empty(struct ring_buffer *buffer)
 {
         struct ring_buffer_per_cpu *cpu_buffer;
+        unsigned long flags;
+        int dolock;
         int cpu;
+        int ret;
+
+        dolock = rb_ok_to_lock();
 
         /* yes this is racy, but if you don't like the race, lock the buffer */
         for_each_buffer_cpu(buffer, cpu) {
                 cpu_buffer = buffer->buffers[cpu];
-                if (!rb_per_cpu_empty(cpu_buffer))
+                local_irq_save(flags);
+                if (dolock)
+                        spin_lock(&cpu_buffer->reader_lock);
+                ret = rb_per_cpu_empty(cpu_buffer);
+                if (dolock)
+                        spin_unlock(&cpu_buffer->reader_lock);
+                local_irq_restore(flags);
+
+                if (!ret)
                         return 0;
         }
 
@@ -2752,19 +3475,29 @@ EXPORT_SYMBOL_GPL(ring_buffer_empty);
 int ring_buffer_empty_cpu(struct ring_buffer *buffer, int cpu)
 {
         struct ring_buffer_per_cpu *cpu_buffer;
+        unsigned long flags;
+        int dolock;
         int ret;
 
         if (!cpumask_test_cpu(cpu, buffer->cpumask))
                 return 1;
 
+        dolock = rb_ok_to_lock();
+
         cpu_buffer = buffer->buffers[cpu];
+        local_irq_save(flags);
+        if (dolock)
+                spin_lock(&cpu_buffer->reader_lock);
         ret = rb_per_cpu_empty(cpu_buffer);
-
+        if (dolock)
+                spin_unlock(&cpu_buffer->reader_lock);
+        local_irq_restore(flags);
 
         return ret;
 }
 EXPORT_SYMBOL_GPL(ring_buffer_empty_cpu);
 
+#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
 /**
  * ring_buffer_swap_cpu - swap a CPU buffer between two ring buffers
  * @buffer_a: One buffer to swap with
@@ -2819,20 +3552,28 @@ int ring_buffer_swap_cpu(struct ring_buffer *buffer_a,
         atomic_inc(&cpu_buffer_a->record_disabled);
         atomic_inc(&cpu_buffer_b->record_disabled);
 
+        ret = -EBUSY;
+        if (local_read(&cpu_buffer_a->committing))
+                goto out_dec;
+        if (local_read(&cpu_buffer_b->committing))
+                goto out_dec;
+
         buffer_a->buffers[cpu] = cpu_buffer_b;
         buffer_b->buffers[cpu] = cpu_buffer_a;
 
         cpu_buffer_b->buffer = buffer_a;
         cpu_buffer_a->buffer = buffer_b;
 
+        ret = 0;
+
+out_dec:
         atomic_dec(&cpu_buffer_a->record_disabled);
         atomic_dec(&cpu_buffer_b->record_disabled);
-
-        ret = 0;
 out:
         return ret;
 }
 EXPORT_SYMBOL_GPL(ring_buffer_swap_cpu);
+#endif /* CONFIG_RING_BUFFER_ALLOW_SWAP */
 
 /**
  * ring_buffer_alloc_read_page - allocate a page to read from buffer
@@ -3005,7 +3746,7 @@ int ring_buffer_read_page(struct ring_buffer *buffer,
                 read = 0;
         } else {
                 /* update the entry counter */
-                cpu_buffer->read += local_read(&reader->entries);
+                cpu_buffer->read += rb_page_entries(reader);
 
                 /* swap the pages */
                 rb_init_page(bpage);
@@ -3026,6 +3767,7 @@ int ring_buffer_read_page(struct ring_buffer *buffer,
 }
 EXPORT_SYMBOL_GPL(ring_buffer_read_page);
 
+#ifdef CONFIG_TRACING
 static ssize_t
 rb_simple_read(struct file *filp, char __user *ubuf,
                size_t cnt, loff_t *ppos)
@@ -3093,6 +3835,7 @@ static __init int rb_init_debugfs(void)
 }
 
 fs_initcall(rb_init_debugfs);
+#endif
 
 #ifdef CONFIG_HOTPLUG_CPU
 static int rb_cpu_notify(struct notifier_block *self,
@@ -3105,7 +3848,7 @@ static int rb_cpu_notify(struct notifier_block *self,
         switch (action) {
         case CPU_UP_PREPARE:
         case CPU_UP_PREPARE_FROZEN:
-                if (cpu_isset(cpu, *buffer->cpumask))
+                if (cpumask_test_cpu(cpu, buffer->cpumask))
                         return NOTIFY_OK;
 
                 buffer->buffers[cpu] =
@@ -3116,7 +3859,7 @@ static int rb_cpu_notify(struct notifier_block *self,
                         return NOTIFY_OK;
                 }
                 smp_wmb();
-                cpu_set(cpu, *buffer->cpumask);
+                cpumask_set_cpu(cpu, buffer->cpumask);
                 break;
         case CPU_DOWN_PREPARE:
         case CPU_DOWN_PREPARE_FROZEN:
diff --git a/kernel/trace/ring_buffer_benchmark.c b/kernel/trace/ring_buffer_benchmark.c
index 8d68e149a8b3..573d3cc762c3 100644
--- a/kernel/trace/ring_buffer_benchmark.c
+++ b/kernel/trace/ring_buffer_benchmark.c
@@ -102,8 +102,10 @@ static enum event_status read_page(int cpu)
                         event = (void *)&rpage->data[i];
                         switch (event->type_len) {
                         case RINGBUF_TYPE_PADDING:
-                                /* We don't expect any padding */
-                                KILL_TEST();
+                                /* failed writes may be discarded events */
+                                if (!event->time_delta)
+                                        KILL_TEST();
+                                inc = event->array[0] + 4;
                                 break;
                         case RINGBUF_TYPE_TIME_EXTEND:
                                 inc = 8;
@@ -119,7 +121,7 @@ static enum event_status read_page(int cpu)
                                         KILL_TEST();
                                         break;
                                 }
-                                inc = event->array[0];
+                                inc = event->array[0] + 4;
                                 break;
                         default:
                                 entry = ring_buffer_event_data(event);
@@ -201,7 +203,7 @@ static void ring_buffer_producer(void)
          * Hammer the buffer for 10 secs (this may
          * make the system stall)
          */
-        pr_info("Starting ring buffer hammer\n");
+        trace_printk("Starting ring buffer hammer\n");
         do_gettimeofday(&start_tv);
         do {
                 struct ring_buffer_event *event;
@@ -237,7 +239,7 @@ static void ring_buffer_producer(void)
 #endif
 
         } while (end_tv.tv_sec < (start_tv.tv_sec + RUN_TIME) && !kill_test);
-        pr_info("End ring buffer hammer\n");
+        trace_printk("End ring buffer hammer\n");
 
         if (consumer) {
                 /* Init both completions here to avoid races */
@@ -260,49 +262,50 @@ static void ring_buffer_producer(void)
         overruns = ring_buffer_overruns(buffer);
 
         if (kill_test)
-                pr_info("ERROR!\n");
-        pr_info("Time:     %lld (usecs)\n", time);
-        pr_info("Overruns: %lld\n", overruns);
+                trace_printk("ERROR!\n");
+        trace_printk("Time:     %lld (usecs)\n", time);
+        trace_printk("Overruns: %lld\n", overruns);
         if (disable_reader)
-                pr_info("Read:     (reader disabled)\n");
+                trace_printk("Read:     (reader disabled)\n");
         else
-                pr_info("Read:     %ld  (by %s)\n", read,
+                trace_printk("Read:     %ld  (by %s)\n", read,
                         read_events ? "events" : "pages");
-        pr_info("Entries:  %lld\n", entries);
-        pr_info("Total:    %lld\n", entries + overruns + read);
-        pr_info("Missed:   %ld\n", missed);
-        pr_info("Hit:      %ld\n", hit);
+        trace_printk("Entries:  %lld\n", entries);
+        trace_printk("Total:    %lld\n", entries + overruns + read);
+        trace_printk("Missed:   %ld\n", missed);
+        trace_printk("Hit:      %ld\n", hit);
 
         /* Convert time from usecs to millisecs */
         do_div(time, USEC_PER_MSEC);
         if (time)
                 hit /= (long)time;
         else
-                pr_info("TIME IS ZERO??\n");
+                trace_printk("TIME IS ZERO??\n");
 
-        pr_info("Entries per millisec: %ld\n", hit);
+        trace_printk("Entries per millisec: %ld\n", hit);
 
         if (hit) {
                 /* Calculate the average time in nanosecs */
                 avg = NSEC_PER_MSEC / hit;
-                pr_info("%ld ns per entry\n", avg);
+                trace_printk("%ld ns per entry\n", avg);
         }
 
         if (missed) {
                 if (time)
                         missed /= (long)time;
 
-                pr_info("Total iterations per millisec: %ld\n", hit + missed);
+                trace_printk("Total iterations per millisec: %ld\n",
+                             hit + missed);
 
                 /* it is possible that hit + missed will overflow and be zero */
                 if (!(hit + missed)) {
-                        pr_info("hit + missed overflowed and totalled zero!\n");
+                        trace_printk("hit + missed overflowed and totalled zero!\n");
                         hit--; /* make it non zero */
                 }
 
                 /* Caculate the average time in nanosecs */
                 avg = NSEC_PER_MSEC / (hit + missed);
-                pr_info("%ld ns per entry\n", avg);
+                trace_printk("%ld ns per entry\n", avg);
         }
 }
 
@@ -353,7 +356,7 @@ static int ring_buffer_producer_thread(void *arg)
 
                 ring_buffer_producer();
 
-                pr_info("Sleeping for 10 secs\n");
+                trace_printk("Sleeping for 10 secs\n");
                 set_current_state(TASK_INTERRUPTIBLE);
                 schedule_timeout(HZ * SLEEP_TIME);
                 __set_current_state(TASK_RUNNING);
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 8acd9b81a5d7..fd52a19dd172 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -17,6 +17,7 @@
 #include <linux/writeback.h>
 #include <linux/kallsyms.h>
 #include <linux/seq_file.h>
+#include <linux/smp_lock.h>
 #include <linux/notifier.h>
 #include <linux/irqflags.h>
 #include <linux/debugfs.h>
@@ -42,14 +43,11 @@
 
 #define TRACE_BUFFER_FLAGS      (RB_FL_OVERWRITE)
 
-unsigned long __read_mostly     tracing_max_latency;
-unsigned long __read_mostly     tracing_thresh;
-
 /*
  * On boot up, the ring buffer is set to the minimum size, so that
  * we do not waste memory on systems that are not using tracing.
  */
-static int ring_buffer_expanded;
+int ring_buffer_expanded;
 
 /*
  * We need to change this state when a selftest is running.
@@ -63,7 +61,7 @@ static bool __read_mostly tracing_selftest_running;
 /*
  * If a tracer is running, we do not want to run SELFTEST.
  */
-static bool __read_mostly tracing_selftest_disabled;
+bool __read_mostly tracing_selftest_disabled;
 
 /* For tracers that don't implement custom flags */
 static struct tracer_opt dummy_tracer_opt[] = {
@@ -88,7 +86,7 @@ static int dummy_set_flag(u32 old_flags, u32 bit, int set)
  */
 static int tracing_disabled = 1;
 
-static DEFINE_PER_CPU(local_t, ftrace_cpu_disabled);
+DEFINE_PER_CPU(local_t, ftrace_cpu_disabled);
 
 static inline void ftrace_disable_cpu(void)
 {
@@ -171,10 +169,11 @@ static struct trace_array	global_trace;
 
 static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
 
-int filter_current_check_discard(struct ftrace_event_call *call, void *rec,
+int filter_current_check_discard(struct ring_buffer *buffer,
+                                 struct ftrace_event_call *call, void *rec,
                                  struct ring_buffer_event *event)
 {
-        return filter_check_discard(call, rec, global_trace.buffer, event);
+        return filter_check_discard(call, rec, buffer, event);
 }
 EXPORT_SYMBOL_GPL(filter_current_check_discard);
 
@@ -265,6 +264,9 @@ unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
         TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
         TRACE_ITER_GRAPH_TIME;
 
+static int trace_stop_count;
+static DEFINE_SPINLOCK(tracing_start_lock);
+
 /**
  * trace_wake_up - wake up tasks waiting for trace input
  *
@@ -284,13 +286,12 @@ void trace_wake_up(void)
 static int __init set_buf_size(char *str)
 {
         unsigned long buf_size;
-        int ret;
 
         if (!str)
                 return 0;
-        ret = strict_strtoul(str, 0, &buf_size);
+        buf_size = memparse(str, &str);
         /* nr_entries can not be zero */
-        if (ret < 0 || buf_size == 0)
+        if (buf_size == 0)
                 return 0;
         trace_buf_size = buf_size;
         return 1;
@@ -323,49 +324,125 @@ static const char *trace_options[] = {
         "printk-msg-only",
         "context-info",
         "latency-format",
-        "global-clock",
         "sleep-time",
         "graph-time",
         NULL
 };
 
+static struct {
+        u64 (*func)(void);
+        const char *name;
+} trace_clocks[] = {
+        { trace_clock_local,    "local" },
+        { trace_clock_global,   "global" },
+};
+
+int trace_clock_id;
+
 /*
- * ftrace_max_lock is used to protect the swapping of buffers
- * when taking a max snapshot. The buffers themselves are
- * protected by per_cpu spinlocks. But the action of the swap
- * needs its own lock.
- *
- * This is defined as a raw_spinlock_t in order to help
- * with performance when lockdep debugging is enabled.
+ * trace_parser_get_init - gets the buffer for trace parser
  */
-static raw_spinlock_t ftrace_max_lock =
-        (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
+int trace_parser_get_init(struct trace_parser *parser, int size)
+{
+        memset(parser, 0, sizeof(*parser));
+
+        parser->buffer = kmalloc(size, GFP_KERNEL);
+        if (!parser->buffer)
+                return 1;
+
+        parser->size = size;
+        return 0;
+}
 
 /*
- * Copy the new maximum trace into the separate maximum-trace
- * structure. (this way the maximum trace is permanently saved,
- * for later retrieval via /debugfs/tracing/latency_trace)
+ * trace_parser_put - frees the buffer for trace parser
  */
-static void
-__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
+void trace_parser_put(struct trace_parser *parser)
 {
-        struct trace_array_cpu *data = tr->data[cpu];
+        kfree(parser->buffer);
+}
 
-        max_tr.cpu = cpu;
-        max_tr.time_start = data->preempt_timestamp;
+/*
+ * trace_get_user - reads the user input string separated by  space
+ * (matched by isspace(ch))
+ *
+ * For each string found the 'struct trace_parser' is updated,
+ * and the function returns.
+ *
+ * Returns number of bytes read.
+ *
+ * See kernel/trace/trace.h for 'struct trace_parser' details.
+ */
+int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
+        size_t cnt, loff_t *ppos)
+{
+        char ch;
+        size_t read = 0;
+        ssize_t ret;
 
-        data = max_tr.data[cpu];
-        data->saved_latency = tracing_max_latency;
+        if (!*ppos)
+                trace_parser_clear(parser);
 
-        memcpy(data->comm, tsk->comm, TASK_COMM_LEN);
-        data->pid = tsk->pid;
-        data->uid = task_uid(tsk);
-        data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
-        data->policy = tsk->policy;
-        data->rt_priority = tsk->rt_priority;
+        ret = get_user(ch, ubuf++);
+        if (ret)
+                goto out;
 
-        /* record this tasks comm */
-        tracing_record_cmdline(tsk);
+        read++;
+        cnt--;
+
+        /*
+         * The parser is not finished with the last write,
+         * continue reading the user input without skipping spaces.
+         */
+        if (!parser->cont) {
+                /* skip white space */
+                while (cnt && isspace(ch)) {
+                        ret = get_user(ch, ubuf++);
+                        if (ret)
+                                goto out;
+                        read++;
+                        cnt--;
+                }
+
+                /* only spaces were written */
+                if (isspace(ch)) {
+                        *ppos += read;
+                        ret = read;
+                        goto out;
+                }
+
+                parser->idx = 0;
+        }
+
+        /* read the non-space input */
+        while (cnt && !isspace(ch)) {
+                if (parser->idx < parser->size)
+                        parser->buffer[parser->idx++] = ch;
+                else {
+                        ret = -EINVAL;
+                        goto out;
+                }
+                ret = get_user(ch, ubuf++);
+                if (ret)
+                        goto out;
+                read++;
+                cnt--;
+        }
+
+        /* We either got finished input or we have to wait for another call. */
+        if (isspace(ch)) {
+                parser->buffer[parser->idx] = 0;
+                parser->cont = false;
+        } else {
+                parser->cont = true;
+                parser->buffer[parser->idx++] = ch;
+        }
+
+        *ppos += read;
+        ret = read;
+
+out:
+        return ret;
 }
 
 ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
@@ -411,6 +488,56 @@ static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
         return cnt;
 }
 
+/*
+ * ftrace_max_lock is used to protect the swapping of buffers
+ * when taking a max snapshot. The buffers themselves are
+ * protected by per_cpu spinlocks. But the action of the swap
+ * needs its own lock.
+ *
+ * This is defined as a raw_spinlock_t in order to help
+ * with performance when lockdep debugging is enabled.
+ *
+ * It is also used in other places outside the update_max_tr
+ * so it needs to be defined outside of the
+ * CONFIG_TRACER_MAX_TRACE.
+ */
+static raw_spinlock_t ftrace_max_lock =
+        (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
+
+#ifdef CONFIG_TRACER_MAX_TRACE
+unsigned long __read_mostly     tracing_max_latency;
+unsigned long __read_mostly     tracing_thresh;
+
+/*
+ * Copy the new maximum trace into the separate maximum-trace
+ * structure. (this way the maximum trace is permanently saved,
+ * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
+ */
+static void
+__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
+{
+        struct trace_array_cpu *data = tr->data[cpu];
+        struct trace_array_cpu *max_data = tr->data[cpu];
+
+        max_tr.cpu = cpu;
+        max_tr.time_start = data->preempt_timestamp;
+
+        max_data = max_tr.data[cpu];
+        max_data->saved_latency = tracing_max_latency;
+        max_data->critical_start = data->critical_start;
+        max_data->critical_end = data->critical_end;
+
+        memcpy(data->comm, tsk->comm, TASK_COMM_LEN);
+        max_data->pid = tsk->pid;
+        max_data->uid = task_uid(tsk);
+        max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
+        max_data->policy = tsk->policy;
+        max_data->rt_priority = tsk->rt_priority;
+
+        /* record this tasks comm */
+        tracing_record_cmdline(tsk);
+}
+
 /**
  * update_max_tr - snapshot all trace buffers from global_trace to max_tr
  * @tr: tracer
@@ -425,16 +552,15 @@ update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
 {
         struct ring_buffer *buf = tr->buffer;
 
+        if (trace_stop_count)
+                return;
+
         WARN_ON_ONCE(!irqs_disabled());
         __raw_spin_lock(&ftrace_max_lock);
 
         tr->buffer = max_tr.buffer;
         max_tr.buffer = buf;
 
-        ftrace_disable_cpu();
-        ring_buffer_reset(tr->buffer);
-        ftrace_enable_cpu();
-
         __update_max_tr(tr, tsk, cpu);
         __raw_spin_unlock(&ftrace_max_lock);
 }
@@ -452,21 +578,35 @@ update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
 {
         int ret;
 
+        if (trace_stop_count)
+                return;
+
         WARN_ON_ONCE(!irqs_disabled());
         __raw_spin_lock(&ftrace_max_lock);
 
         ftrace_disable_cpu();
 
-        ring_buffer_reset(max_tr.buffer);
         ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
 
+        if (ret == -EBUSY) {
+                /*
+                 * We failed to swap the buffer due to a commit taking
+                 * place on this CPU. We fail to record, but we reset
+                 * the max trace buffer (no one writes directly to it)
+                 * and flag that it failed.
+                 */
+                trace_array_printk(&max_tr, _THIS_IP_,
+                        "Failed to swap buffers due to commit in progress\n");
+        }
+
         ftrace_enable_cpu();
 
-        WARN_ON_ONCE(ret && ret != -EAGAIN);
+        WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
 
         __update_max_tr(tr, tsk, cpu);
         __raw_spin_unlock(&ftrace_max_lock);
 }
+#endif /* CONFIG_TRACER_MAX_TRACE */
 
 /**
  * register_tracer - register a tracer with the ftrace system.
@@ -523,7 +663,6 @@ __acquires(kernel_lock)
         if (type->selftest && !tracing_selftest_disabled) {
                 struct tracer *saved_tracer = current_trace;
                 struct trace_array *tr = &global_trace;
-                int i;
 
                 /*
                  * Run a selftest on this tracer.
@@ -532,8 +671,7 @@ __acquires(kernel_lock)
                  * internal tracing to verify that everything is in order.
                  * If we fail, we do not register this tracer.
                  */
-                for_each_tracing_cpu(i)
-                        tracing_reset(tr, i);
+                tracing_reset_online_cpus(tr);
 
                 current_trace = type;
                 /* the test is responsible for initializing and enabling */
@@ -546,8 +684,7 @@ __acquires(kernel_lock)
                         goto out;
                 }
                 /* Only reset on passing, to avoid touching corrupted buffers */
-                for_each_tracing_cpu(i)
-                        tracing_reset(tr, i);
+                tracing_reset_online_cpus(tr);
 
                 printk(KERN_CONT "PASSED\n");
         }
@@ -622,21 +759,42 @@ void unregister_tracer(struct tracer *type)
         mutex_unlock(&trace_types_lock);
 }
 
-void tracing_reset(struct trace_array *tr, int cpu)
+static void __tracing_reset(struct trace_array *tr, int cpu)
 {
         ftrace_disable_cpu();
         ring_buffer_reset_cpu(tr->buffer, cpu);
         ftrace_enable_cpu();
 }
 
+void tracing_reset(struct trace_array *tr, int cpu)
+{
+        struct ring_buffer *buffer = tr->buffer;
+
+        ring_buffer_record_disable(buffer);
+
+        /* Make sure all commits have finished */
+        synchronize_sched();
+        __tracing_reset(tr, cpu);
+
+        ring_buffer_record_enable(buffer);
+}
+
 void tracing_reset_online_cpus(struct trace_array *tr)
 {
+        struct ring_buffer *buffer = tr->buffer;
         int cpu;
 
+        ring_buffer_record_disable(buffer);
+
+        /* Make sure all commits have finished */
+        synchronize_sched();
+
         tr->time_start = ftrace_now(tr->cpu);
 
         for_each_online_cpu(cpu)
-                tracing_reset(tr, cpu);
+                __tracing_reset(tr, cpu);
+
+        ring_buffer_record_enable(buffer);
 }
 
 void tracing_reset_current(int cpu)
@@ -667,8 +825,10 @@ static void trace_init_cmdlines(void)
         cmdline_idx = 0;
 }
 
-static int trace_stop_count;
-static DEFINE_SPINLOCK(tracing_start_lock);
+int is_tracing_stopped(void)
+{
+        return trace_stop_count;
+}
 
 /**
  * ftrace_off_permanent - disable all ftrace code permanently
@@ -837,7 +997,7 @@ tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
 
         entry->preempt_count            = pc & 0xff;
         entry->pid                      = (tsk) ? tsk->pid : 0;
-        entry->tgid                     = (tsk) ? tsk->tgid : 0;
+        entry->lock_depth               = (tsk) ? tsk->lock_depth : 0;
         entry->flags =
 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
                 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
@@ -848,15 +1008,17 @@ tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
                 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
                 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
 }
+EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
 
-struct ring_buffer_event *trace_buffer_lock_reserve(struct trace_array *tr,
-                                                    int type,
-                                                    unsigned long len,
-                                                    unsigned long flags, int pc)
+struct ring_buffer_event *
+trace_buffer_lock_reserve(struct ring_buffer *buffer,
+                          int type,
+                          unsigned long len,
+                          unsigned long flags, int pc)
 {
         struct ring_buffer_event *event;
 
-        event = ring_buffer_lock_reserve(tr->buffer, len);
+        event = ring_buffer_lock_reserve(buffer, len);
         if (event != NULL) {
                 struct trace_entry *ent = ring_buffer_event_data(event);
 
@@ -866,58 +1028,60 @@ struct ring_buffer_event *trace_buffer_lock_reserve(struct trace_array *tr,
 
         return event;
 }
-static void ftrace_trace_stack(struct trace_array *tr,
-                               unsigned long flags, int skip, int pc);
-static void ftrace_trace_userstack(struct trace_array *tr,
-                                   unsigned long flags, int pc);
 
-static inline void __trace_buffer_unlock_commit(struct trace_array *tr,
-                                        struct ring_buffer_event *event,
-                                        unsigned long flags, int pc,
-                                        int wake)
+static inline void
+__trace_buffer_unlock_commit(struct ring_buffer *buffer,
+                             struct ring_buffer_event *event,
+                             unsigned long flags, int pc,
+                             int wake)
 {
-        ring_buffer_unlock_commit(tr->buffer, event);
+        ring_buffer_unlock_commit(buffer, event);
 
-        ftrace_trace_stack(tr, flags, 6, pc);
-        ftrace_trace_userstack(tr, flags, pc);
+        ftrace_trace_stack(buffer, flags, 6, pc);
+        ftrace_trace_userstack(buffer, flags, pc);
 
         if (wake)
                 trace_wake_up();
 }
 
-void trace_buffer_unlock_commit(struct trace_array *tr,
-                                        struct ring_buffer_event *event,
-                                        unsigned long flags, int pc)
+void trace_buffer_unlock_commit(struct ring_buffer *buffer,
+                                struct ring_buffer_event *event,
+                                unsigned long flags, int pc)
 {
-        __trace_buffer_unlock_commit(tr, event, flags, pc, 1);
+        __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
 }
 
 struct ring_buffer_event *
-trace_current_buffer_lock_reserve(int type, unsigned long len,
+trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
+                                  int type, unsigned long len,
                                   unsigned long flags, int pc)
 {
-        return trace_buffer_lock_reserve(&global_trace,
+        *current_rb = global_trace.buffer;
+        return trace_buffer_lock_reserve(*current_rb,
                                          type, len, flags, pc);
 }
 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
 
-void trace_current_buffer_unlock_commit(struct ring_buffer_event *event,
+void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
+                                        struct ring_buffer_event *event,
                                         unsigned long flags, int pc)
 {
-        __trace_buffer_unlock_commit(&global_trace, event, flags, pc, 1);
+        __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
 }
 EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
 
-void trace_nowake_buffer_unlock_commit(struct ring_buffer_event *event,
-                                        unsigned long flags, int pc)
+void trace_nowake_buffer_unlock_commit(struct ring_buffer *buffer,
+                                       struct ring_buffer_event *event,
+                                       unsigned long flags, int pc)
 {
-        __trace_buffer_unlock_commit(&global_trace, event, flags, pc, 0);
+        __trace_buffer_unlock_commit(buffer, event, flags, pc, 0);
 }
 EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit);
 
-void trace_current_buffer_discard_commit(struct ring_buffer_event *event)
+void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
+                                         struct ring_buffer_event *event)
 {
-        ring_buffer_discard_commit(global_trace.buffer, event);
+        ring_buffer_discard_commit(buffer, event);
 }
 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
 
@@ -927,6 +1091,7 @@ trace_function(struct trace_array *tr,
                int pc)
 {
         struct ftrace_event_call *call = &event_function;
+        struct ring_buffer *buffer = tr->buffer;
         struct ring_buffer_event *event;
         struct ftrace_entry *entry;
 
@@ -934,7 +1099,7 @@ trace_function(struct trace_array *tr,
         if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
                 return;
 
-        event = trace_buffer_lock_reserve(tr, TRACE_FN, sizeof(*entry),
+        event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
                                           flags, pc);
         if (!event)
                 return;
@@ -942,58 +1107,10 @@ trace_function(struct trace_array *tr,
         entry->ip                       = ip;
         entry->parent_ip                = parent_ip;
 
-        if (!filter_check_discard(call, entry, tr->buffer, event))
-                ring_buffer_unlock_commit(tr->buffer, event);
+        if (!filter_check_discard(call, entry, buffer, event))
+                ring_buffer_unlock_commit(buffer, event);
 }
 
-#ifdef CONFIG_FUNCTION_GRAPH_TRACER
-static int __trace_graph_entry(struct trace_array *tr,
-                                struct ftrace_graph_ent *trace,
-                                unsigned long flags,
-                                int pc)
-{
-        struct ftrace_event_call *call = &event_funcgraph_entry;
-        struct ring_buffer_event *event;
-        struct ftrace_graph_ent_entry *entry;
-
-        if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
-                return 0;
-
-        event = trace_buffer_lock_reserve(&global_trace, TRACE_GRAPH_ENT,
-                                          sizeof(*entry), flags, pc);
-        if (!event)
-                return 0;
-        entry   = ring_buffer_event_data(event);
-        entry->graph_ent                        = *trace;
-        if (!filter_current_check_discard(call, entry, event))
-                ring_buffer_unlock_commit(global_trace.buffer, event);
-
-        return 1;
-}
-
-static void __trace_graph_return(struct trace_array *tr,
-                                struct ftrace_graph_ret *trace,
-                                unsigned long flags,
-                                int pc)
-{
-        struct ftrace_event_call *call = &event_funcgraph_exit;
-        struct ring_buffer_event *event;
-        struct ftrace_graph_ret_entry *entry;
-
-        if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
-                return;
-
-        event = trace_buffer_lock_reserve(&global_trace, TRACE_GRAPH_RET,
-                                          sizeof(*entry), flags, pc);
-        if (!event)
-                return;
-        entry   = ring_buffer_event_data(event);
-        entry->ret                              = *trace;
-        if (!filter_current_check_discard(call, entry, event))
-                ring_buffer_unlock_commit(global_trace.buffer, event);
-}
-#endif
-
 void
 ftrace(struct trace_array *tr, struct trace_array_cpu *data,
        unsigned long ip, unsigned long parent_ip, unsigned long flags,
@@ -1003,17 +1120,17 @@ ftrace(struct trace_array *tr, struct trace_array_cpu *data,
                 trace_function(tr, ip, parent_ip, flags, pc);
 }
 
-static void __ftrace_trace_stack(struct trace_array *tr,
+#ifdef CONFIG_STACKTRACE
+static void __ftrace_trace_stack(struct ring_buffer *buffer,
                                  unsigned long flags,
                                  int skip, int pc)
 {
-#ifdef CONFIG_STACKTRACE
         struct ftrace_event_call *call = &event_kernel_stack;
         struct ring_buffer_event *event;
         struct stack_entry *entry;
         struct stack_trace trace;
 
-        event = trace_buffer_lock_reserve(tr, TRACE_STACK,
+        event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
                                           sizeof(*entry), flags, pc);
         if (!event)
                 return;
@@ -1026,32 +1143,28 @@ static void __ftrace_trace_stack(struct trace_array *tr,
         trace.entries           = entry->caller;
 
         save_stack_trace(&trace);
-        if (!filter_check_discard(call, entry, tr->buffer, event))
-                ring_buffer_unlock_commit(tr->buffer, event);
-#endif
+        if (!filter_check_discard(call, entry, buffer, event))
+                ring_buffer_unlock_commit(buffer, event);
 }
 
-static void ftrace_trace_stack(struct trace_array *tr,
-                               unsigned long flags,
-                               int skip, int pc)
+void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
+                        int skip, int pc)
 {
         if (!(trace_flags & TRACE_ITER_STACKTRACE))
                 return;
 
-        __ftrace_trace_stack(tr, flags, skip, pc);
+        __ftrace_trace_stack(buffer, flags, skip, pc);
 }
 
-void __trace_stack(struct trace_array *tr,
-                   unsigned long flags,
-                   int skip, int pc)
+void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
+                   int pc)
 {
-        __ftrace_trace_stack(tr, flags, skip, pc);
+        __ftrace_trace_stack(tr->buffer, flags, skip, pc);
 }
 
-static void ftrace_trace_userstack(struct trace_array *tr,
-                                   unsigned long flags, int pc)
+void
+ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
 {
-#ifdef CONFIG_STACKTRACE
         struct ftrace_event_call *call = &event_user_stack;
         struct ring_buffer_event *event;
         struct userstack_entry *entry;
@@ -1060,12 +1173,13 @@ static void ftrace_trace_userstack(struct trace_array *tr,
         if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
                 return;
 
-        event = trace_buffer_lock_reserve(tr, TRACE_USER_STACK,
+        event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
                                           sizeof(*entry), flags, pc);
         if (!event)
                 return;
         entry   = ring_buffer_event_data(event);
 
+        entry->tgid             = current->tgid;
         memset(&entry->caller, 0, sizeof(entry->caller));
 
         trace.nr_entries        = 0;
@@ -1074,9 +1188,8 @@ static void ftrace_trace_userstack(struct trace_array *tr,
         trace.entries           = entry->caller;
 
         save_stack_trace_user(&trace);
-        if (!filter_check_discard(call, entry, tr->buffer, event))
-                ring_buffer_unlock_commit(tr->buffer, event);
-#endif
+        if (!filter_check_discard(call, entry, buffer, event))
+                ring_buffer_unlock_commit(buffer, event);
 }
 
 #ifdef UNUSED
@@ -1086,16 +1199,20 @@ static void __trace_userstack(struct trace_array *tr, unsigned long flags)
 }
 #endif /* UNUSED */
 
+#endif /* CONFIG_STACKTRACE */
+
 static void
 ftrace_trace_special(void *__tr,
                      unsigned long arg1, unsigned long arg2, unsigned long arg3,
                      int pc)
 {
+        struct ftrace_event_call *call = &event_special;
         struct ring_buffer_event *event;
         struct trace_array *tr = __tr;
+        struct ring_buffer *buffer = tr->buffer;
         struct special_entry *entry;
 
-        event = trace_buffer_lock_reserve(tr, TRACE_SPECIAL,
+        event = trace_buffer_lock_reserve(buffer, TRACE_SPECIAL,
                                           sizeof(*entry), 0, pc);
         if (!event)
                 return;
@@ -1103,7 +1220,9 @@ ftrace_trace_special(void *__tr,
         entry->arg1                     = arg1;
         entry->arg2                     = arg2;
         entry->arg3                     = arg3;
-        trace_buffer_unlock_commit(tr, event, 0, pc);
+
+        if (!filter_check_discard(call, entry, buffer, event))
+                trace_buffer_unlock_commit(buffer, event, 0, pc);
 }
 
 void
@@ -1114,62 +1233,6 @@ __trace_special(void *__tr, void *__data,
 }
 
 void
-tracing_sched_switch_trace(struct trace_array *tr,
-                           struct task_struct *prev,
-                           struct task_struct *next,
-                           unsigned long flags, int pc)
-{
-        struct ftrace_event_call *call = &event_context_switch;
-        struct ring_buffer_event *event;
-        struct ctx_switch_entry *entry;
-
-        event = trace_buffer_lock_reserve(tr, TRACE_CTX,
-                                          sizeof(*entry), flags, pc);
-        if (!event)
-                return;
-        entry   = ring_buffer_event_data(event);
-        entry->prev_pid                 = prev->pid;
-        entry->prev_prio                = prev->prio;
-        entry->prev_state               = prev->state;
-        entry->next_pid                 = next->pid;
-        entry->next_prio                = next->prio;
-        entry->next_state               = next->state;
-        entry->next_cpu = task_cpu(next);
-
-        if (!filter_check_discard(call, entry, tr->buffer, event))
-                trace_buffer_unlock_commit(tr, event, flags, pc);
-}
-
-void
-tracing_sched_wakeup_trace(struct trace_array *tr,
-                           struct task_struct *wakee,
-                           struct task_struct *curr,
-                           unsigned long flags, int pc)
-{
-        struct ftrace_event_call *call = &event_wakeup;
-        struct ring_buffer_event *event;
-        struct ctx_switch_entry *entry;
-
-        event = trace_buffer_lock_reserve(tr, TRACE_WAKE,
-                                          sizeof(*entry), flags, pc);
-        if (!event)
-                return;
-        entry   = ring_buffer_event_data(event);
-        entry->prev_pid                 = curr->pid;
-        entry->prev_prio                = curr->prio;
-        entry->prev_state               = curr->state;
-        entry->next_pid                 = wakee->pid;
-        entry->next_prio                = wakee->prio;
-        entry->next_state               = wakee->state;
-        entry->next_cpu                 = task_cpu(wakee);
-
-        if (!filter_check_discard(call, entry, tr->buffer, event))
-                ring_buffer_unlock_commit(tr->buffer, event);
-        ftrace_trace_stack(tr, flags, 6, pc);
-        ftrace_trace_userstack(tr, flags, pc);
-}
-
-void
 ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
 {
         struct trace_array *tr = &global_trace;
@@ -1193,68 +1256,6 @@ ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
         local_irq_restore(flags);
 }
 
-#ifdef CONFIG_FUNCTION_GRAPH_TRACER
-int trace_graph_entry(struct ftrace_graph_ent *trace)
-{
-        struct trace_array *tr = &global_trace;
-        struct trace_array_cpu *data;
-        unsigned long flags;
-        long disabled;
-        int ret;
-        int cpu;
-        int pc;
-
-        if (!ftrace_trace_task(current))
-                return 0;
-
-        if (!ftrace_graph_addr(trace->func))
-                return 0;
-
-        local_irq_save(flags);
-        cpu = raw_smp_processor_id();
-        data = tr->data[cpu];
-        disabled = atomic_inc_return(&data->disabled);
-        if (likely(disabled == 1)) {
-                pc = preempt_count();
-                ret = __trace_graph_entry(tr, trace, flags, pc);
-        } else {
-                ret = 0;
-        }
-        /* Only do the atomic if it is not already set */
-        if (!test_tsk_trace_graph(current))
-                set_tsk_trace_graph(current);
-
-        atomic_dec(&data->disabled);
-        local_irq_restore(flags);
-
-        return ret;
-}
-
-void trace_graph_return(struct ftrace_graph_ret *trace)
-{
-        struct trace_array *tr = &global_trace;
-        struct trace_array_cpu *data;
-        unsigned long flags;
-        long disabled;
-        int cpu;
-        int pc;
-
-        local_irq_save(flags);
-        cpu = raw_smp_processor_id();
-        data = tr->data[cpu];
-        disabled = atomic_inc_return(&data->disabled);
-        if (likely(disabled == 1)) {
-                pc = preempt_count();
-                __trace_graph_return(tr, trace, flags, pc);
-        }
-        if (!trace->depth)
-                clear_tsk_trace_graph(current);
-        atomic_dec(&data->disabled);
-        local_irq_restore(flags);
-}
-#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
-
-
 /**
  * trace_vbprintk - write binary msg to tracing buffer
  *
@@ -1267,6 +1268,7 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
 
         struct ftrace_event_call *call = &event_bprint;
         struct ring_buffer_event *event;
+        struct ring_buffer *buffer;
         struct trace_array *tr = &global_trace;
         struct trace_array_cpu *data;
         struct bprint_entry *entry;
@@ -1299,7 +1301,9 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
                 goto out_unlock;
 
         size = sizeof(*entry) + sizeof(u32) * len;
-        event = trace_buffer_lock_reserve(tr, TRACE_BPRINT, size, flags, pc);
+        buffer = tr->buffer;
+        event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
+                                          flags, pc);
         if (!event)
                 goto out_unlock;
         entry = ring_buffer_event_data(event);
@@ -1307,8 +1311,8 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
         entry->fmt                      = fmt;
 
         memcpy(entry->buf, trace_buf, sizeof(u32) * len);
-        if (!filter_check_discard(call, entry, tr->buffer, event))
-                ring_buffer_unlock_commit(tr->buffer, event);
+        if (!filter_check_discard(call, entry, buffer, event))
+                ring_buffer_unlock_commit(buffer, event);
 
 out_unlock:
         __raw_spin_unlock(&trace_buf_lock);
@@ -1323,14 +1327,30 @@ out:
 }
 EXPORT_SYMBOL_GPL(trace_vbprintk);
 
-int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
+int trace_array_printk(struct trace_array *tr,
+                       unsigned long ip, const char *fmt, ...)
+{
+        int ret;
+        va_list ap;
+
+        if (!(trace_flags & TRACE_ITER_PRINTK))
+                return 0;
+
+        va_start(ap, fmt);
+        ret = trace_array_vprintk(tr, ip, fmt, ap);
+        va_end(ap);
+        return ret;
+}
+
+int trace_array_vprintk(struct trace_array *tr,
+                        unsigned long ip, const char *fmt, va_list args)
 {
         static raw_spinlock_t trace_buf_lock = __RAW_SPIN_LOCK_UNLOCKED;
         static char trace_buf[TRACE_BUF_SIZE];
 
         struct ftrace_event_call *call = &event_print;
         struct ring_buffer_event *event;
-        struct trace_array *tr = &global_trace;
+        struct ring_buffer *buffer;
         struct trace_array_cpu *data;
         int cpu, len = 0, size, pc;
         struct print_entry *entry;
@@ -1358,7 +1378,9 @@ int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
         trace_buf[len] = 0;
 
         size = sizeof(*entry) + len + 1;
-        event = trace_buffer_lock_reserve(tr, TRACE_PRINT, size, irq_flags, pc);
+        buffer = tr->buffer;
+        event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
+                                          irq_flags, pc);
         if (!event)
                 goto out_unlock;
         entry = ring_buffer_event_data(event);
@@ -1366,8 +1388,8 @@ int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
 
         memcpy(&entry->buf, trace_buf, len);
         entry->buf[len] = 0;
-        if (!filter_check_discard(call, entry, tr->buffer, event))
-                ring_buffer_unlock_commit(tr->buffer, event);
+        if (!filter_check_discard(call, entry, buffer, event))
+                ring_buffer_unlock_commit(buffer, event);
 
  out_unlock:
         __raw_spin_unlock(&trace_buf_lock);
@@ -1379,6 +1401,11 @@ int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
 
         return len;
 }
+
+int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
+{
+        return trace_array_printk(&global_trace, ip, fmt, args);
+}
 EXPORT_SYMBOL_GPL(trace_vprintk);
 
 enum trace_file_type {
@@ -1518,6 +1545,37 @@ static void *s_next(struct seq_file *m, void *v, loff_t *pos)
         return ent;
 }
 
+static void tracing_iter_reset(struct trace_iterator *iter, int cpu)
+{
+        struct trace_array *tr = iter->tr;
+        struct ring_buffer_event *event;
+        struct ring_buffer_iter *buf_iter;
+        unsigned long entries = 0;
+        u64 ts;
+
+        tr->data[cpu]->skipped_entries = 0;
+
+        if (!iter->buffer_iter[cpu])
+                return;
+
+        buf_iter = iter->buffer_iter[cpu];
+        ring_buffer_iter_reset(buf_iter);
+
+        /*
+         * We could have the case with the max latency tracers
+         * that a reset never took place on a cpu. This is evident
+         * by the timestamp being before the start of the buffer.
+         */
+        while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
+                if (ts >= iter->tr->time_start)
+                        break;
+                entries++;
+                ring_buffer_read(buf_iter, NULL);
+        }
+
+        tr->data[cpu]->skipped_entries = entries;
+}
+
 /*
  * No necessary locking here. The worst thing which can
  * happen is loosing events consumed at the same time
@@ -1556,10 +1614,9 @@ static void *s_start(struct seq_file *m, loff_t *pos)
 
                 if (cpu_file == TRACE_PIPE_ALL_CPU) {
                         for_each_tracing_cpu(cpu)
-                                ring_buffer_iter_reset(iter->buffer_iter[cpu]);
+                                tracing_iter_reset(iter, cpu);
                 } else
-                        ring_buffer_iter_reset(iter->buffer_iter[cpu_file]);
-
+                        tracing_iter_reset(iter, cpu_file);
 
                 ftrace_enable_cpu();
 
@@ -1588,10 +1645,10 @@ static void print_lat_help_header(struct seq_file *m)
         seq_puts(m, "#                | / _----=> need-resched    \n");
         seq_puts(m, "#                || / _---=> hardirq/softirq \n");
         seq_puts(m, "#                ||| / _--=> preempt-depth   \n");
-        seq_puts(m, "#                |||| /                      \n");
-        seq_puts(m, "#                |||||     delay             \n");
-        seq_puts(m, "#  cmd     pid   ||||| time  |   caller      \n");
-        seq_puts(m, "#     \\   /      |||||   \\   |   /           \n");
+        seq_puts(m, "#                |||| /_--=> lock-depth       \n");
+        seq_puts(m, "#                |||||/     delay             \n");
+        seq_puts(m, "#  cmd     pid   |||||| time  |   caller      \n");
+        seq_puts(m, "#     \\   /      ||||||   \\   |   /           \n");
 }
 
 static void print_func_help_header(struct seq_file *m)
@@ -1608,16 +1665,32 @@ print_trace_header(struct seq_file *m, struct trace_iterator *iter)
         struct trace_array *tr = iter->tr;
         struct trace_array_cpu *data = tr->data[tr->cpu];
         struct tracer *type = current_trace;
-        unsigned long total;
-        unsigned long entries;
+        unsigned long entries = 0;
+        unsigned long total = 0;
+        unsigned long count;
         const char *name = "preemption";
+        int cpu;
 
         if (type)
                 name = type->name;
 
-        entries = ring_buffer_entries(iter->tr->buffer);
-        total = entries +
-                ring_buffer_overruns(iter->tr->buffer);
+
+        for_each_tracing_cpu(cpu) {
+                count = ring_buffer_entries_cpu(tr->buffer, cpu);
+                /*
+                 * If this buffer has skipped entries, then we hold all
+                 * entries for the trace and we need to ignore the
+                 * ones before the time stamp.
+                 */
+                if (tr->data[cpu]->skipped_entries) {
+                        count -= tr->data[cpu]->skipped_entries;
+                        /* total is the same as the entries */
+                        total += count;
+                } else
+                        total += count +
+                                ring_buffer_overrun_cpu(tr->buffer, cpu);
+                entries += count;
+        }
 
         seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
                    name, UTS_RELEASE);
@@ -1659,7 +1732,7 @@ print_trace_header(struct seq_file *m, struct trace_iterator *iter)
                 seq_puts(m, "\n#  => ended at:   ");
                 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
                 trace_print_seq(m, &iter->seq);
-                seq_puts(m, "#\n");
+                seq_puts(m, "\n#\n");
         }
 
         seq_puts(m, "#\n");
@@ -1678,6 +1751,9 @@ static void test_cpu_buff_start(struct trace_iterator *iter)
         if (cpumask_test_cpu(iter->cpu, iter->started))
                 return;
 
+        if (iter->tr->data[iter->cpu]->skipped_entries)
+                return;
+
         cpumask_set_cpu(iter->cpu, iter->started);
 
         /* Don't print started cpu buffer for the first entry of the trace */
@@ -1940,19 +2016,23 @@ __tracing_open(struct inode *inode, struct file *file)
         if (ring_buffer_overruns(iter->tr->buffer))
                 iter->iter_flags |= TRACE_FILE_ANNOTATE;
 
+        /* stop the trace while dumping */
+        tracing_stop();
+
         if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
                 for_each_tracing_cpu(cpu) {
 
                         iter->buffer_iter[cpu] =
                                 ring_buffer_read_start(iter->tr->buffer, cpu);
+                        tracing_iter_reset(iter, cpu);
                 }
         } else {
                 cpu = iter->cpu_file;
                 iter->buffer_iter[cpu] =
                                 ring_buffer_read_start(iter->tr->buffer, cpu);
+                tracing_iter_reset(iter, cpu);
         }
 
-        /* TODO stop tracer */
         ret = seq_open(file, &tracer_seq_ops);
         if (ret < 0) {
                 fail_ret = ERR_PTR(ret);
@@ -1962,9 +2042,6 @@ __tracing_open(struct inode *inode, struct file *file)
         m = file->private_data;
         m->private = iter;
 
-        /* stop the trace while dumping */
-        tracing_stop();
-
         mutex_unlock(&trace_types_lock);
 
         return iter;
@@ -1975,6 +2052,7 @@ __tracing_open(struct inode *inode, struct file *file)
                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
         }
         free_cpumask_var(iter->started);
+        tracing_start();
  fail:
         mutex_unlock(&trace_types_lock);
         kfree(iter->trace);
@@ -2031,7 +2109,7 @@ static int tracing_open(struct inode *inode, struct file *file)
 
         /* If this file was open for write, then erase contents */
         if ((file->f_mode & FMODE_WRITE) &&
-            !(file->f_flags & O_APPEND)) {
+            (file->f_flags & O_TRUNC)) {
                 long cpu = (long) inode->i_private;
 
                 if (cpu == TRACE_PIPE_ALL_CPU)
@@ -2053,25 +2131,23 @@ static int tracing_open(struct inode *inode, struct file *file)
 static void *
 t_next(struct seq_file *m, void *v, loff_t *pos)
 {
-        struct tracer *t = m->private;
+        struct tracer *t = v;
 
         (*pos)++;
 
         if (t)
                 t = t->next;
 
-        m->private = t;
-
         return t;
 }
 
 static void *t_start(struct seq_file *m, loff_t *pos)
 {
-        struct tracer *t = m->private;
+        struct tracer *t;
         loff_t l = 0;
 
         mutex_lock(&trace_types_lock);
-        for (; t && l < *pos; t = t_next(m, t, &l))
+        for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
                 ;
 
         return t;
@@ -2107,18 +2183,10 @@ static struct seq_operations show_traces_seq_ops = {
 
 static int show_traces_open(struct inode *inode, struct file *file)
 {
-        int ret;
-
         if (tracing_disabled)
                 return -ENODEV;
 
-        ret = seq_open(file, &show_traces_seq_ops);
-        if (!ret) {
-                struct seq_file *m = file->private_data;
-                m->private = trace_types;
-        }
-
-        return ret;
+        return seq_open(file, &show_traces_seq_ops);
 }
 
 static ssize_t
@@ -2191,11 +2259,12 @@ tracing_cpumask_write(struct file *filp, const char __user *ubuf,
         if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
                 return -ENOMEM;
 
-        mutex_lock(&tracing_cpumask_update_lock);
         err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
         if (err)
                 goto err_unlock;
 
+        mutex_lock(&tracing_cpumask_update_lock);
+
         local_irq_disable();
         __raw_spin_lock(&ftrace_max_lock);
         for_each_tracing_cpu(cpu) {
@@ -2223,8 +2292,7 @@ tracing_cpumask_write(struct file *filp, const char __user *ubuf,
         return count;
 
 err_unlock:
-        mutex_unlock(&tracing_cpumask_update_lock);
-        free_cpumask_var(tracing_cpumask);
+        free_cpumask_var(tracing_cpumask_new);
 
         return err;
 }
@@ -2266,8 +2334,8 @@ tracing_trace_options_read(struct file *filp, char __user *ubuf,
                 len += 3; /* "no" and newline */
         }
 
-        /* +2 for \n and \0 */
-        buf = kmalloc(len + 2, GFP_KERNEL);
+        /* +1 for \0 */
+        buf = kmalloc(len + 1, GFP_KERNEL);
         if (!buf) {
                 mutex_unlock(&trace_types_lock);
                 return -ENOMEM;
@@ -2290,7 +2358,7 @@ tracing_trace_options_read(struct file *filp, char __user *ubuf,
         }
         mutex_unlock(&trace_types_lock);
 
-        WARN_ON(r >= len + 2);
+        WARN_ON(r >= len + 1);
 
         r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
 
@@ -2301,23 +2369,23 @@ tracing_trace_options_read(struct file *filp, char __user *ubuf,
 /* Try to assign a tracer specific option */
 static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
 {
-        struct tracer_flags *trace_flags = trace->flags;
+        struct tracer_flags *tracer_flags = trace->flags;
         struct tracer_opt *opts = NULL;
         int ret = 0, i = 0;
         int len;
 
-        for (i = 0; trace_flags->opts[i].name; i++) {
-                opts = &trace_flags->opts[i];
+        for (i = 0; tracer_flags->opts[i].name; i++) {
+                opts = &tracer_flags->opts[i];
                 len = strlen(opts->name);
 
                 if (strncmp(cmp, opts->name, len) == 0) {
-                        ret = trace->set_flag(trace_flags->val,
+                        ret = trace->set_flag(tracer_flags->val,
                                 opts->bit, !neg);
                         break;
                 }
         }
         /* Not found */
-        if (!trace_flags->opts[i].name)
+        if (!tracer_flags->opts[i].name)
                 return -EINVAL;
 
         /* Refused to handle */
@@ -2325,9 +2393,9 @@ static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
                 return ret;
 
         if (neg)
-                trace_flags->val &= ~opts->bit;
+                tracer_flags->val &= ~opts->bit;
         else
-                trace_flags->val |= opts->bit;
+                tracer_flags->val |= opts->bit;
 
         return 0;
 }
@@ -2342,22 +2410,6 @@ static void set_tracer_flags(unsigned int mask, int enabled)
                 trace_flags |= mask;
         else
                 trace_flags &= ~mask;
-
-        if (mask == TRACE_ITER_GLOBAL_CLK) {
-                u64 (*func)(void);
-
-                if (enabled)
-                        func = trace_clock_global;
-                else
-                        func = trace_clock_local;
-
-                mutex_lock(&trace_types_lock);
-                ring_buffer_set_clock(global_trace.buffer, func);
-
-                if (max_tr.buffer)
-                        ring_buffer_set_clock(max_tr.buffer, func);
-                mutex_unlock(&trace_types_lock);
-        }
 }
 
 static ssize_t
@@ -2414,21 +2466,20 @@ static const struct file_operations tracing_iter_fops = {
 
 static const char readme_msg[] =
         "tracing mini-HOWTO:\n\n"
-        "# mkdir /debug\n"
-        "# mount -t debugfs nodev /debug\n\n"
-        "# cat /debug/tracing/available_tracers\n"
+        "# mount -t debugfs nodev /sys/kernel/debug\n\n"
+        "# cat /sys/kernel/debug/tracing/available_tracers\n"
         "wakeup preemptirqsoff preemptoff irqsoff function sched_switch nop\n\n"
-        "# cat /debug/tracing/current_tracer\n"
+        "# cat /sys/kernel/debug/tracing/current_tracer\n"
         "nop\n"
-        "# echo sched_switch > /debug/tracing/current_tracer\n"
-        "# cat /debug/tracing/current_tracer\n"
+        "# echo sched_switch > /sys/kernel/debug/tracing/current_tracer\n"
+        "# cat /sys/kernel/debug/tracing/current_tracer\n"
         "sched_switch\n"
-        "# cat /debug/tracing/trace_options\n"
+        "# cat /sys/kernel/debug/tracing/trace_options\n"
         "noprint-parent nosym-offset nosym-addr noverbose\n"
-        "# echo print-parent > /debug/tracing/trace_options\n"
-        "# echo 1 > /debug/tracing/tracing_enabled\n"
-        "# cat /debug/tracing/trace > /tmp/trace.txt\n"
-        "# echo 0 > /debug/tracing/tracing_enabled\n"
+        "# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
+        "# echo 1 > /sys/kernel/debug/tracing/tracing_enabled\n"
+        "# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
+        "# echo 0 > /sys/kernel/debug/tracing/tracing_enabled\n"
 ;
 
 static ssize_t
@@ -3096,7 +3147,8 @@ tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
                         break;
                 }
 
-                trace_consume(iter);
+                if (ret != TRACE_TYPE_NO_CONSUME)
+                        trace_consume(iter);
                 rem -= count;
                 if (!find_next_entry_inc(iter)) {
                         rem = 0;
@@ -3325,6 +3377,62 @@ tracing_mark_write(struct file *filp, const char __user *ubuf,
         return cnt;
 }
 
+static ssize_t tracing_clock_read(struct file *filp, char __user *ubuf,
+                                  size_t cnt, loff_t *ppos)
+{
+        char buf[64];
+        int bufiter = 0;
+        int i;
+
+        for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
+                bufiter += snprintf(buf + bufiter, sizeof(buf) - bufiter,
+                        "%s%s%s%s", i ? " " : "",
+                        i == trace_clock_id ? "[" : "", trace_clocks[i].name,
+                        i == trace_clock_id ? "]" : "");
+        bufiter += snprintf(buf + bufiter, sizeof(buf) - bufiter, "\n");
+
+        return simple_read_from_buffer(ubuf, cnt, ppos, buf, bufiter);
+}
+
+static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
+                                   size_t cnt, loff_t *fpos)
+{
+        char buf[64];
+        const char *clockstr;
+        int i;
+
+        if (cnt >= sizeof(buf))
+                return -EINVAL;
+
+        if (copy_from_user(&buf, ubuf, cnt))
+                return -EFAULT;
+
+        buf[cnt] = 0;
+
+        clockstr = strstrip(buf);
+
+        for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
+                if (strcmp(trace_clocks[i].name, clockstr) == 0)
+                        break;
+        }
+        if (i == ARRAY_SIZE(trace_clocks))
+                return -EINVAL;
+
+        trace_clock_id = i;
+
+        mutex_lock(&trace_types_lock);
+
+        ring_buffer_set_clock(global_trace.buffer, trace_clocks[i].func);
+        if (max_tr.buffer)
+                ring_buffer_set_clock(max_tr.buffer, trace_clocks[i].func);
+
+        mutex_unlock(&trace_types_lock);
+
+        *fpos += cnt;
+
+        return cnt;
+}
+
 static const struct file_operations tracing_max_lat_fops = {
         .open           = tracing_open_generic,
         .read           = tracing_max_lat_read,
@@ -3362,6 +3470,12 @@ static const struct file_operations tracing_mark_fops = {
         .write          = tracing_mark_write,
 };
 
+static const struct file_operations trace_clock_fops = {
+        .open           = tracing_open_generic,
+        .read           = tracing_clock_read,
+        .write          = tracing_clock_write,
+};
+
 struct ftrace_buffer_info {
         struct trace_array      *tr;
         void                    *spare;
@@ -3627,7 +3741,7 @@ tracing_stats_read(struct file *filp, char __user *ubuf,
         struct trace_seq *s;
         unsigned long cnt;
 
-        s = kmalloc(sizeof(*s), GFP_ATOMIC);
+        s = kmalloc(sizeof(*s), GFP_KERNEL);
         if (!s)
                 return ENOMEM;
 
@@ -3642,9 +3756,6 @@ tracing_stats_read(struct file *filp, char __user *ubuf,
         cnt = ring_buffer_commit_overrun_cpu(tr->buffer, cpu);
         trace_seq_printf(s, "commit overrun: %ld\n", cnt);
 
-        cnt = ring_buffer_nmi_dropped_cpu(tr->buffer, cpu);
-        trace_seq_printf(s, "nmi dropped: %ld\n", cnt);
-
         count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
 
         kfree(s);
@@ -3905,17 +4016,9 @@ trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
         if (ret < 0)
                 return ret;
 
-        switch (val) {
-        case 0:
-                trace_flags &= ~(1 << index);
-                break;
-        case 1:
-                trace_flags |= 1 << index;
-                break;
-
-        default:
+        if (val != 0 && val != 1)
                 return -EINVAL;
-        }
+        set_tracer_flags(1 << index, val);
 
         *ppos += cnt;
 
@@ -4083,11 +4186,13 @@ static __init int tracer_init_debugfs(void)
         trace_create_file("current_tracer", 0644, d_tracer,
                         &global_trace, &set_tracer_fops);
 
+#ifdef CONFIG_TRACER_MAX_TRACE
         trace_create_file("tracing_max_latency", 0644, d_tracer,
                         &tracing_max_latency, &tracing_max_lat_fops);
 
         trace_create_file("tracing_thresh", 0644, d_tracer,
                         &tracing_thresh, &tracing_max_lat_fops);
+#endif
 
         trace_create_file("README", 0444, d_tracer,
                         NULL, &tracing_readme_fops);
@@ -4104,6 +4209,9 @@ static __init int tracer_init_debugfs(void)
         trace_create_file("saved_cmdlines", 0444, d_tracer,
                         NULL, &tracing_saved_cmdlines_fops);
 
+        trace_create_file("trace_clock", 0644, d_tracer, NULL,
+                          &trace_clock_fops);
+
 #ifdef CONFIG_DYNAMIC_FTRACE
         trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
                         &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
@@ -4244,8 +4352,11 @@ static void __ftrace_dump(bool disable_tracing)
                 iter.pos = -1;
 
                 if (find_next_entry_inc(&iter) != NULL) {
-                        print_trace_line(&iter);
-                        trace_consume(&iter);
+                        int ret;
+
+                        ret = print_trace_line(&iter);
+                        if (ret != TRACE_TYPE_NO_CONSUME)
+                                trace_consume(&iter);
                 }
 
                 trace_printk_seq(&iter.seq);
@@ -4279,7 +4390,6 @@ void ftrace_dump(void)
 
 __init static int tracer_alloc_buffers(void)
 {
-        struct trace_array_cpu *data;
         int ring_buf_size;
         int i;
         int ret = -ENOMEM;
@@ -4329,7 +4439,7 @@ __init static int tracer_alloc_buffers(void)
 
         /* Allocate the first page for all buffers */
         for_each_tracing_cpu(i) {
-                data = global_trace.data[i] = &per_cpu(global_trace_cpu, i);
+                global_trace.data[i] = &per_cpu(global_trace_cpu, i);
                 max_tr.data[i] = &per_cpu(max_data, i);
         }
 
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index 6e735d4771f8..86bcff94791a 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -7,6 +7,7 @@
 #include <linux/clocksource.h>
 #include <linux/ring_buffer.h>
 #include <linux/mmiotrace.h>
+#include <linux/tracepoint.h>
 #include <linux/ftrace.h>
 #include <trace/boot.h>
 #include <linux/kmemtrace.h>
@@ -34,8 +35,6 @@ enum trace_type {
         TRACE_GRAPH_ENT,
         TRACE_USER_STACK,
         TRACE_HW_BRANCHES,
-        TRACE_SYSCALL_ENTER,
-        TRACE_SYSCALL_EXIT,
         TRACE_KMEM_ALLOC,
         TRACE_KMEM_FREE,
         TRACE_POWER,
@@ -44,157 +43,54 @@ enum trace_type {
         __TRACE_LAST_TYPE,
 };
 
-/*
- * Function trace entry - function address and parent function addres:
- */
-struct ftrace_entry {
-        struct trace_entry      ent;
-        unsigned long           ip;
-        unsigned long           parent_ip;
-};
-
-/* Function call entry */
-struct ftrace_graph_ent_entry {
-        struct trace_entry              ent;
-        struct ftrace_graph_ent         graph_ent;
+enum kmemtrace_type_id {
+        KMEMTRACE_TYPE_KMALLOC = 0,     /* kmalloc() or kfree(). */
+        KMEMTRACE_TYPE_CACHE,           /* kmem_cache_*(). */
+        KMEMTRACE_TYPE_PAGES,           /* __get_free_pages() and friends. */
 };
 
-/* Function return entry */
-struct ftrace_graph_ret_entry {
-        struct trace_entry              ent;
-        struct ftrace_graph_ret         ret;
-};
 extern struct tracer boot_tracer;
 
-/*
- * Context switch trace entry - which task (and prio) we switched from/to:
- */
-struct ctx_switch_entry {
-        struct trace_entry      ent;
-        unsigned int            prev_pid;
-        unsigned char           prev_prio;
-        unsigned char           prev_state;
-        unsigned int            next_pid;
-        unsigned char           next_prio;
-        unsigned char           next_state;
-        unsigned int            next_cpu;
-};
-
-/*
- * Special (free-form) trace entry:
- */
-struct special_entry {
-        struct trace_entry      ent;
-        unsigned long           arg1;
-        unsigned long           arg2;
-        unsigned long           arg3;
-};
-
-/*
- * Stack-trace entry:
- */
-
-#define FTRACE_STACK_ENTRIES    8
-
-struct stack_entry {
-        struct trace_entry      ent;
-        unsigned long           caller[FTRACE_STACK_ENTRIES];
-};
-
-struct userstack_entry {
-        struct trace_entry      ent;
-        unsigned long           caller[FTRACE_STACK_ENTRIES];
-};
+#undef __field
+#define __field(type, item)             type    item;
 
-/*
- * trace_printk entry:
- */
-struct bprint_entry {
-        struct trace_entry      ent;
-        unsigned long           ip;
-        const char              *fmt;
-        u32                     buf[];
-};
+#undef __field_struct
+#define __field_struct(type, item)      __field(type, item)
 
-struct print_entry {
-        struct trace_entry      ent;
-        unsigned long           ip;
-        char                    buf[];
-};
+#undef __field_desc
+#define __field_desc(type, container, item)
 
-#define TRACE_OLD_SIZE          88
+#undef __array
+#define __array(type, item, size)       type    item[size];
 
-struct trace_field_cont {
-        unsigned char           type;
-        /* Temporary till we get rid of this completely */
-        char                    buf[TRACE_OLD_SIZE - 1];
-};
+#undef __array_desc
+#define __array_desc(type, container, item, size)
 
-struct trace_mmiotrace_rw {
-        struct trace_entry      ent;
-        struct mmiotrace_rw     rw;
-};
+#undef __dynamic_array
+#define __dynamic_array(type, item)     type    item[];
 
-struct trace_mmiotrace_map {
-        struct trace_entry      ent;
-        struct mmiotrace_map    map;
-};
+#undef F_STRUCT
+#define F_STRUCT(args...)               args
 
-struct trace_boot_call {
-        struct trace_entry      ent;
-        struct boot_trace_call boot_call;
-};
-
-struct trace_boot_ret {
-        struct trace_entry      ent;
-        struct boot_trace_ret boot_ret;
-};
-
-#define TRACE_FUNC_SIZE 30
-#define TRACE_FILE_SIZE 20
-struct trace_branch {
-        struct trace_entry      ent;
-        unsigned                line;
-        char                    func[TRACE_FUNC_SIZE+1];
-        char                    file[TRACE_FILE_SIZE+1];
-        char                    correct;
-};
-
-struct hw_branch_entry {
-        struct trace_entry      ent;
-        u64                     from;
-        u64                     to;
-};
-
-struct trace_power {
-        struct trace_entry      ent;
-        struct power_trace      state_data;
-};
+#undef FTRACE_ENTRY
+#define FTRACE_ENTRY(name, struct_name, id, tstruct, print)     \
+        struct struct_name {                                    \
+                struct trace_entry      ent;                    \
+                tstruct                                         \
+        }
 
-enum kmemtrace_type_id {
-        KMEMTRACE_TYPE_KMALLOC = 0,     /* kmalloc() or kfree(). */
-        KMEMTRACE_TYPE_CACHE,           /* kmem_cache_*(). */
-        KMEMTRACE_TYPE_PAGES,           /* __get_free_pages() and friends. */
-};
+#undef TP_ARGS
+#define TP_ARGS(args...)        args
 
-struct kmemtrace_alloc_entry {
-        struct trace_entry      ent;
-        enum kmemtrace_type_id type_id;
-        unsigned long call_site;
-        const void *ptr;
-        size_t bytes_req;
-        size_t bytes_alloc;
-        gfp_t gfp_flags;
-        int node;
-};
+#undef FTRACE_ENTRY_DUP
+#define FTRACE_ENTRY_DUP(name, name_struct, id, tstruct, printk)
 
-struct kmemtrace_free_entry {
-        struct trace_entry      ent;
-        enum kmemtrace_type_id type_id;
-        unsigned long call_site;
-        const void *ptr;
-};
+#include "trace_entries.h"
 
+/*
+ * syscalls are special, and need special handling, this is why
+ * they are not included in trace_entries.h
+ */
 struct syscall_trace_enter {
         struct trace_entry      ent;
         int                     nr;
@@ -207,13 +103,12 @@ struct syscall_trace_exit {
         unsigned long           ret;
 };
 
-
 /*
  * trace_flag_type is an enumeration that holds different
  * states when a trace occurs. These are:
  *  IRQS_OFF            - interrupts were disabled
  *  IRQS_NOSUPPORT      - arch does not support irqs_disabled_flags
- *  NEED_RESCED         - reschedule is requested
+ *  NEED_RESCHED        - reschedule is requested
  *  HARDIRQ             - inside an interrupt handler
  *  SOFTIRQ             - inside a softirq handler
  */
@@ -236,9 +131,6 @@ struct trace_array_cpu {
         atomic_t                disabled;
         void                    *buffer_page;   /* ring buffer spare */
 
-        /* these fields get copied into max-trace: */
-        unsigned long           trace_idx;
-        unsigned long           overrun;
         unsigned long           saved_latency;
         unsigned long           critical_start;
         unsigned long           critical_end;
@@ -246,6 +138,7 @@ struct trace_array_cpu {
         unsigned long           nice;
         unsigned long           policy;
         unsigned long           rt_priority;
+        unsigned long           skipped_entries;
         cycle_t                 preempt_timestamp;
         pid_t                   pid;
         uid_t                   uid;
@@ -319,10 +212,6 @@ extern void __ftrace_bad_type(void);
                           TRACE_KMEM_ALLOC);    \
                 IF_ASSIGN(var, ent, struct kmemtrace_free_entry,        \
                           TRACE_KMEM_FREE);     \
-                IF_ASSIGN(var, ent, struct syscall_trace_enter,         \
-                          TRACE_SYSCALL_ENTER);                         \
-                IF_ASSIGN(var, ent, struct syscall_trace_exit,          \
-                          TRACE_SYSCALL_EXIT);                          \
                 __ftrace_bad_type();                                    \
         } while (0)
 
@@ -398,7 +287,6 @@ struct tracer {
         struct tracer           *next;
         int                     print_max;
         struct tracer_flags     *flags;
-        struct tracer_stat      *stats;
 };
 
 
@@ -423,12 +311,13 @@ void init_tracer_sysprof_debugfs(struct dentry *d_tracer);
 
 struct ring_buffer_event;
 
-struct ring_buffer_event *trace_buffer_lock_reserve(struct trace_array *tr,
-                                                    int type,
-                                                    unsigned long len,
-                                                    unsigned long flags,
-                                                    int pc);
-void trace_buffer_unlock_commit(struct trace_array *tr,
+struct ring_buffer_event *
+trace_buffer_lock_reserve(struct ring_buffer *buffer,
+                          int type,
+                          unsigned long len,
+                          unsigned long flags,
+                          int pc);
+void trace_buffer_unlock_commit(struct ring_buffer *buffer,
                                 struct ring_buffer_event *event,
                                 unsigned long flags, int pc);
 
@@ -438,10 +327,6 @@ struct trace_entry *tracing_get_trace_entry(struct trace_array *tr,
 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
                                           int *ent_cpu, u64 *ent_ts);
 
-void tracing_generic_entry_update(struct trace_entry *entry,
-                                  unsigned long flags,
-                                  int pc);
-
 void default_wait_pipe(struct trace_iterator *iter);
 void poll_wait_pipe(struct trace_iterator *iter);
 
@@ -471,6 +356,7 @@ void trace_function(struct trace_array *tr,
 
 void trace_graph_return(struct ftrace_graph_ret *trace);
 int trace_graph_entry(struct ftrace_graph_ent *trace);
+void set_graph_array(struct trace_array *tr);
 
 void tracing_start_cmdline_record(void);
 void tracing_stop_cmdline_record(void);
@@ -479,35 +365,46 @@ void tracing_stop_sched_switch_record(void);
 void tracing_start_sched_switch_record(void);
 int register_tracer(struct tracer *type);
 void unregister_tracer(struct tracer *type);
+int is_tracing_stopped(void);
 
 extern unsigned long nsecs_to_usecs(unsigned long nsecs);
 
+#ifdef CONFIG_TRACER_MAX_TRACE
 extern unsigned long tracing_max_latency;
 extern unsigned long tracing_thresh;
 
 void update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu);
 void update_max_tr_single(struct trace_array *tr,
                           struct task_struct *tsk, int cpu);
+#endif /* CONFIG_TRACER_MAX_TRACE */
 
-void __trace_stack(struct trace_array *tr,
-                   unsigned long flags,
-                   int skip, int pc);
+#ifdef CONFIG_STACKTRACE
+void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
+                        int skip, int pc);
 
-extern cycle_t ftrace_now(int cpu);
+void ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags,
+                            int pc);
 
-#ifdef CONFIG_CONTEXT_SWITCH_TRACER
-typedef void
-(*tracer_switch_func_t)(void *private,
-                        void *__rq,
-                        struct task_struct *prev,
-                        struct task_struct *next);
-
-struct tracer_switch_ops {
-        tracer_switch_func_t            func;
-        void                            *private;
-        struct tracer_switch_ops        *next;
-};
-#endif /* CONFIG_CONTEXT_SWITCH_TRACER */
+void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
+                   int pc);
+#else
+static inline void ftrace_trace_stack(struct trace_array *tr,
+                                      unsigned long flags, int skip, int pc)
+{
+}
+
+static inline void ftrace_trace_userstack(struct trace_array *tr,
+                                          unsigned long flags, int pc)
+{
+}
+
+static inline void __trace_stack(struct trace_array *tr, unsigned long flags,
+                                 int skip, int pc)
+{
+}
+#endif /* CONFIG_STACKTRACE */
+
+extern cycle_t ftrace_now(int cpu);
 
 extern void trace_find_cmdline(int pid, char comm[]);
 
@@ -517,6 +414,10 @@ extern unsigned long ftrace_update_tot_cnt;
 extern int DYN_FTRACE_TEST_NAME(void);
 #endif
 
+extern int ring_buffer_expanded;
+extern bool tracing_selftest_disabled;
+DECLARE_PER_CPU(local_t, ftrace_cpu_disabled);
+
 #ifdef CONFIG_FTRACE_STARTUP_TEST
 extern int trace_selftest_startup_function(struct tracer *trace,
                                            struct trace_array *tr);
@@ -548,9 +449,16 @@ extern int
 trace_vbprintk(unsigned long ip, const char *fmt, va_list args);
 extern int
 trace_vprintk(unsigned long ip, const char *fmt, va_list args);
+extern int
+trace_array_vprintk(struct trace_array *tr,
+                    unsigned long ip, const char *fmt, va_list args);
+int trace_array_printk(struct trace_array *tr,
+                       unsigned long ip, const char *fmt, ...);
 
 extern unsigned long trace_flags;
 
+extern int trace_clock_id;
+
 /* Standard output formatting function used for function return traces */
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 extern enum print_line_t print_graph_function(struct trace_iterator *iter);
@@ -597,6 +505,7 @@ print_graph_function(struct trace_iterator *iter)
 
 extern struct pid *ftrace_pid_trace;
 
+#ifdef CONFIG_FUNCTION_TRACER
 static inline int ftrace_trace_task(struct task_struct *task)
 {
         if (!ftrace_pid_trace)
@@ -604,6 +513,47 @@ static inline int ftrace_trace_task(struct task_struct *task)
 
         return test_tsk_trace_trace(task);
 }
+#else
+static inline int ftrace_trace_task(struct task_struct *task)
+{
+        return 1;
+}
+#endif
+
+/*
+ * struct trace_parser - servers for reading the user input separated by spaces
+ * @cont: set if the input is not complete - no final space char was found
+ * @buffer: holds the parsed user input
+ * @idx: user input lenght
+ * @size: buffer size
+ */
+struct trace_parser {
+        bool            cont;
+        char            *buffer;
+        unsigned        idx;
+        unsigned        size;
+};
+
+static inline bool trace_parser_loaded(struct trace_parser *parser)
+{
+        return (parser->idx != 0);
+}
+
+static inline bool trace_parser_cont(struct trace_parser *parser)
+{
+        return parser->cont;
+}
+
+static inline void trace_parser_clear(struct trace_parser *parser)
+{
+        parser->cont = false;
+        parser->idx = 0;
+}
+
+extern int trace_parser_get_init(struct trace_parser *parser, int size);
+extern void trace_parser_put(struct trace_parser *parser);
+extern int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
+        size_t cnt, loff_t *ppos);
 
 /*
  * trace_iterator_flags is an enumeration that defines bit
@@ -632,9 +582,8 @@ enum trace_iterator_flags {
         TRACE_ITER_PRINTK_MSGONLY       = 0x10000,
         TRACE_ITER_CONTEXT_INFO         = 0x20000, /* Print pid/cpu/time */
         TRACE_ITER_LATENCY_FMT          = 0x40000,
-        TRACE_ITER_GLOBAL_CLK           = 0x80000,
-        TRACE_ITER_SLEEP_TIME           = 0x100000,
-        TRACE_ITER_GRAPH_TIME           = 0x200000,
+        TRACE_ITER_SLEEP_TIME           = 0x80000,
+        TRACE_ITER_GRAPH_TIME           = 0x100000,
 };
 
 /*
@@ -731,6 +680,7 @@ struct ftrace_event_field {
         struct list_head        link;
         char                    *name;
         char                    *type;
+        int                     filter_type;
         int                     offset;
         int                     size;
         int                     is_signed;
@@ -740,13 +690,15 @@ struct event_filter {
         int                     n_preds;
         struct filter_pred      **preds;
         char                    *filter_string;
+        bool                    no_reset;
 };
 
 struct event_subsystem {
         struct list_head        list;
         const char              *name;
         struct dentry           *entry;
-        void                    *filter;
+        struct event_filter     *filter;
+        int                     nr_events;
 };
 
 struct filter_pred;
@@ -774,6 +726,7 @@ extern int apply_subsystem_event_filter(struct event_subsystem *system,
                                         char *filter_string);
 extern void print_subsystem_event_filter(struct event_subsystem *system,
                                          struct trace_seq *s);
+extern int filter_assign_type(const char *type);
 
 static inline int
 filter_check_discard(struct ftrace_event_call *call, void *rec,
@@ -788,58 +741,18 @@ filter_check_discard(struct ftrace_event_call *call, void *rec,
         return 0;
 }
 
-#define DEFINE_COMPARISON_PRED(type)                                    \
-static int filter_pred_##type(struct filter_pred *pred, void *event,    \
-                              int val1, int val2)                       \
-{                                                                       \
-        type *addr = (type *)(event + pred->offset);                    \
-        type val = (type)pred->val;                                     \
-        int match = 0;                                                  \
-                                                                        \
-        switch (pred->op) {                                             \
-        case OP_LT:                                                     \
-                match = (*addr < val);                                  \
-                break;                                                  \
-        case OP_LE:                                                     \
-                match = (*addr <= val);                                 \
-                break;                                                  \
-        case OP_GT:                                                     \
-                match = (*addr > val);                                  \
-                break;                                                  \
-        case OP_GE:                                                     \
-                match = (*addr >= val);                                 \
-                break;                                                  \
-        default:                                                        \
-                break;                                                  \
-        }                                                               \
-                                                                        \
-        return match;                                                   \
-}
-
-#define DEFINE_EQUALITY_PRED(size)                                      \
-static int filter_pred_##size(struct filter_pred *pred, void *event,    \
-                              int val1, int val2)                       \
-{                                                                       \
-        u##size *addr = (u##size *)(event + pred->offset);              \
-        u##size val = (u##size)pred->val;                               \
-        int match;                                                      \
-                                                                        \
-        match = (val == *addr) ^ pred->not;                             \
-                                                                        \
-        return match;                                                   \
-}
-
 extern struct mutex event_mutex;
 extern struct list_head ftrace_events;
 
 extern const char *__start___trace_bprintk_fmt[];
 extern const char *__stop___trace_bprintk_fmt[];
 
-#undef TRACE_EVENT_FORMAT
-#define TRACE_EVENT_FORMAT(call, proto, args, fmt, tstruct, tpfmt)      \
+#undef FTRACE_ENTRY
+#define FTRACE_ENTRY(call, struct_name, id, tstruct, print)             \
         extern struct ftrace_event_call event_##call;
-#undef TRACE_EVENT_FORMAT_NOFILTER
-#define TRACE_EVENT_FORMAT_NOFILTER(call, proto, args, fmt, tstruct, tpfmt)
-#include "trace_event_types.h"
+#undef FTRACE_ENTRY_DUP
+#define FTRACE_ENTRY_DUP(call, struct_name, id, tstruct, print)         \
+        FTRACE_ENTRY(call, struct_name, id, PARAMS(tstruct), PARAMS(print))
+#include "trace_entries.h"
 
 #endif /* _LINUX_KERNEL_TRACE_H */
diff --git a/kernel/trace/trace_boot.c b/kernel/trace/trace_boot.c
index a29ef23ffb47..c21d5f3956ad 100644
--- a/kernel/trace/trace_boot.c
+++ b/kernel/trace/trace_boot.c
@@ -41,14 +41,12 @@ void disable_boot_trace(void)
 
 static int boot_trace_init(struct trace_array *tr)
 {
-        int cpu;
         boot_trace = tr;
 
         if (!tr)
                 return 0;
 
-        for_each_cpu(cpu, cpu_possible_mask)
-                tracing_reset(tr, cpu);
+        tracing_reset_online_cpus(tr);
 
         tracing_sched_switch_assign_trace(tr);
         return 0;
@@ -131,7 +129,9 @@ struct tracer boot_tracer __read_mostly =
 
 void trace_boot_call(struct boot_trace_call *bt, initcall_t fn)
 {
+        struct ftrace_event_call *call = &event_boot_call;
         struct ring_buffer_event *event;
+        struct ring_buffer *buffer;
         struct trace_boot_call *entry;
         struct trace_array *tr = boot_trace;
 
@@ -144,20 +144,24 @@ void trace_boot_call(struct boot_trace_call *bt, initcall_t fn)
         sprint_symbol(bt->func, (unsigned long)fn);
         preempt_disable();
 
-        event = trace_buffer_lock_reserve(tr, TRACE_BOOT_CALL,
+        buffer = tr->buffer;
+        event = trace_buffer_lock_reserve(buffer, TRACE_BOOT_CALL,
                                           sizeof(*entry), 0, 0);
         if (!event)
                 goto out;
         entry   = ring_buffer_event_data(event);
         entry->boot_call = *bt;
-        trace_buffer_unlock_commit(tr, event, 0, 0);
+        if (!filter_check_discard(call, entry, buffer, event))
+                trace_buffer_unlock_commit(buffer, event, 0, 0);
  out:
         preempt_enable();
 }
 
 void trace_boot_ret(struct boot_trace_ret *bt, initcall_t fn)
 {
+        struct ftrace_event_call *call = &event_boot_ret;
         struct ring_buffer_event *event;
+        struct ring_buffer *buffer;
         struct trace_boot_ret *entry;
         struct trace_array *tr = boot_trace;
 
@@ -167,13 +171,15 @@ void trace_boot_ret(struct boot_trace_ret *bt, initcall_t fn)
         sprint_symbol(bt->func, (unsigned long)fn);
         preempt_disable();
 
-        event = trace_buffer_lock_reserve(tr, TRACE_BOOT_RET,
+        buffer = tr->buffer;
+        event = trace_buffer_lock_reserve(buffer, TRACE_BOOT_RET,
                                           sizeof(*entry), 0, 0);
         if (!event)
                 goto out;
         entry   = ring_buffer_event_data(event);
         entry->boot_ret = *bt;
-        trace_buffer_unlock_commit(tr, event, 0, 0);
+        if (!filter_check_discard(call, entry, buffer, event))
+                trace_buffer_unlock_commit(buffer, event, 0, 0);
  out:
         preempt_enable();
 }
diff --git a/kernel/trace/trace_clock.c b/kernel/trace/trace_clock.c
index b588fd81f7f9..20c5f92e28a8 100644
--- a/kernel/trace/trace_clock.c
+++ b/kernel/trace/trace_clock.c
@@ -66,10 +66,14 @@ u64 notrace trace_clock(void)
  * Used by plugins that need globally coherent timestamps.
  */
 
-static u64 prev_trace_clock_time;
-
-static raw_spinlock_t trace_clock_lock ____cacheline_aligned_in_smp =
-        (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
+/* keep prev_time and lock in the same cacheline. */
+static struct {
+        u64 prev_time;
+        raw_spinlock_t lock;
+} trace_clock_struct ____cacheline_aligned_in_smp =
+        {
+                .lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED,
+        };
 
 u64 notrace trace_clock_global(void)
 {
@@ -88,19 +92,19 @@ u64 notrace trace_clock_global(void)
         if (unlikely(in_nmi()))
                 goto out;
 
-        __raw_spin_lock(&trace_clock_lock);
+        __raw_spin_lock(&trace_clock_struct.lock);
 
         /*
          * TODO: if this happens often then maybe we should reset
-         * my_scd->clock to prev_trace_clock_time+1, to make sure
+         * my_scd->clock to prev_time+1, to make sure
          * we start ticking with the local clock from now on?
          */
-        if ((s64)(now - prev_trace_clock_time) < 0)
-                now = prev_trace_clock_time + 1;
+        if ((s64)(now - trace_clock_struct.prev_time) < 0)
+                now = trace_clock_struct.prev_time + 1;
 
-        prev_trace_clock_time = now;
+        trace_clock_struct.prev_time = now;
 
-        __raw_spin_unlock(&trace_clock_lock);
+        __raw_spin_unlock(&trace_clock_struct.lock);
 
  out:
         raw_local_irq_restore(flags);
diff --git a/kernel/trace/trace_entries.h b/kernel/trace/trace_entries.h
new file mode 100644
index 000000000000..a431748ddd6e
--- /dev/null
+++ b/kernel/trace/trace_entries.h
@@ -0,0 +1,383 @@
+/*
+ * This file defines the trace event structures that go into the ring
+ * buffer directly. They are created via macros so that changes for them
+ * appear in the format file. Using macros will automate this process.
+ *
+ * The macro used to create a ftrace data structure is:
+ *
+ * FTRACE_ENTRY( name, struct_name, id, structure, print )
+ *
+ * @name: the name used the event name, as well as the name of
+ *   the directory that holds the format file.
+ *
+ * @struct_name: the name of the structure that is created.
+ *
+ * @id: The event identifier that is used to detect what event
+ *    this is from the ring buffer.
+ *
+ * @structure: the structure layout
+ *
+ *  - __field(  type,   item    )
+ *        This is equivalent to declaring
+ *              type    item;
+ *        in the structure.
+ *  - __array(  type,   item,   size    )
+ *        This is equivalent to declaring
+ *              type    item[size];
+ *        in the structure.
+ *
+ *   * for structures within structures, the format of the internal
+ *      structure is layed out. This allows the internal structure
+ *      to be deciphered for the format file. Although these macros
+ *      may become out of sync with the internal structure, they
+ *      will create a compile error if it happens. Since the
+ *      internel structures are just tracing helpers, this is not
+ *      an issue.
+ *
+ *      When an internal structure is used, it should use:
+ *
+ *      __field_struct( type,   item    )
+ *
+ *      instead of __field. This will prevent it from being shown in
+ *      the output file. The fields in the structure should use.
+ *
+ *      __field_desc(   type,   container,      item            )
+ *      __array_desc(   type,   container,      item,   len     )
+ *
+ *      type, item and len are the same as __field and __array, but
+ *      container is added. This is the name of the item in
+ *      __field_struct that this is describing.
+ *
+ *
+ * @print: the print format shown to users in the format file.
+ */
+
+/*
+ * Function trace entry - function address and parent function addres:
+ */
+FTRACE_ENTRY(function, ftrace_entry,
+
+        TRACE_FN,
+
+        F_STRUCT(
+                __field(        unsigned long,  ip              )
+                __field(        unsigned long,  parent_ip       )
+        ),
+
+        F_printk(" %lx <-- %lx", __entry->ip, __entry->parent_ip)
+);
+
+/* Function call entry */
+FTRACE_ENTRY(funcgraph_entry, ftrace_graph_ent_entry,
+
+        TRACE_GRAPH_ENT,
+
+        F_STRUCT(
+                __field_struct( struct ftrace_graph_ent,        graph_ent       )
+                __field_desc(   unsigned long,  graph_ent,      func            )
+                __field_desc(   int,            graph_ent,      depth           )
+        ),
+
+        F_printk("--> %lx (%d)", __entry->func, __entry->depth)
+);
+
+/* Function return entry */
+FTRACE_ENTRY(funcgraph_exit, ftrace_graph_ret_entry,
+
+        TRACE_GRAPH_RET,
+
+        F_STRUCT(
+                __field_struct( struct ftrace_graph_ret,        ret     )
+                __field_desc(   unsigned long,  ret,            func    )
+                __field_desc(   unsigned long long, ret,        calltime)
+                __field_desc(   unsigned long long, ret,        rettime )
+                __field_desc(   unsigned long,  ret,            overrun )
+                __field_desc(   int,            ret,            depth   )
+        ),
+
+        F_printk("<-- %lx (%d) (start: %llx  end: %llx) over: %d",
+                 __entry->func, __entry->depth,
+                 __entry->calltime, __entry->rettime,
+                 __entry->depth)
+);
+
+/*
+ * Context switch trace entry - which task (and prio) we switched from/to:
+ *
+ * This is used for both wakeup and context switches. We only want
+ * to create one structure, but we need two outputs for it.
+ */
+#define FTRACE_CTX_FIELDS                                       \
+        __field(        unsigned int,   prev_pid        )       \
+        __field(        unsigned char,  prev_prio       )       \
+        __field(        unsigned char,  prev_state      )       \
+        __field(        unsigned int,   next_pid        )       \
+        __field(        unsigned char,  next_prio       )       \
+        __field(        unsigned char,  next_state      )       \
+        __field(        unsigned int,   next_cpu        )
+
+FTRACE_ENTRY(context_switch, ctx_switch_entry,
+
+        TRACE_CTX,
+
+        F_STRUCT(
+                FTRACE_CTX_FIELDS
+        ),
+
+        F_printk("%u:%u:%u  ==> %u:%u:%u [%03u]",
+                 __entry->prev_pid, __entry->prev_prio, __entry->prev_state,
+                 __entry->next_pid, __entry->next_prio, __entry->next_state,
+                 __entry->next_cpu
+                )
+);
+
+/*
+ * FTRACE_ENTRY_DUP only creates the format file, it will not
+ *  create another structure.
+ */
+FTRACE_ENTRY_DUP(wakeup, ctx_switch_entry,
+
+        TRACE_WAKE,
+
+        F_STRUCT(
+                FTRACE_CTX_FIELDS
+        ),
+
+        F_printk("%u:%u:%u  ==+ %u:%u:%u [%03u]",
+                 __entry->prev_pid, __entry->prev_prio, __entry->prev_state,
+                 __entry->next_pid, __entry->next_prio, __entry->next_state,
+                 __entry->next_cpu
+                )
+);
+
+/*
+ * Special (free-form) trace entry:
+ */
+FTRACE_ENTRY(special, special_entry,
+
+        TRACE_SPECIAL,
+
+        F_STRUCT(
+                __field(        unsigned long,  arg1    )
+                __field(        unsigned long,  arg2    )
+                __field(        unsigned long,  arg3    )
+        ),
+
+        F_printk("(%08lx) (%08lx) (%08lx)",
+                 __entry->arg1, __entry->arg2, __entry->arg3)
+);
+
+/*
+ * Stack-trace entry:
+ */
+
+#define FTRACE_STACK_ENTRIES    8
+
+FTRACE_ENTRY(kernel_stack, stack_entry,
+
+        TRACE_STACK,
+
+        F_STRUCT(
+                __array(        unsigned long,  caller, FTRACE_STACK_ENTRIES    )
+        ),
+
+        F_printk("\t=> (%08lx)\n\t=> (%08lx)\n\t=> (%08lx)\n\t=> (%08lx)\n"
+                 "\t=> (%08lx)\n\t=> (%08lx)\n\t=> (%08lx)\n\t=> (%08lx)\n",
+                 __entry->caller[0], __entry->caller[1], __entry->caller[2],
+                 __entry->caller[3], __entry->caller[4], __entry->caller[5],
+                 __entry->caller[6], __entry->caller[7])
+);
+
+FTRACE_ENTRY(user_stack, userstack_entry,
+
+        TRACE_USER_STACK,
+
+        F_STRUCT(
+                __field(        unsigned int,   tgid    )
+                __array(        unsigned long,  caller, FTRACE_STACK_ENTRIES    )
+        ),
+
+        F_printk("\t=> (%08lx)\n\t=> (%08lx)\n\t=> (%08lx)\n\t=> (%08lx)\n"
+                 "\t=> (%08lx)\n\t=> (%08lx)\n\t=> (%08lx)\n\t=> (%08lx)\n",
+                 __entry->caller[0], __entry->caller[1], __entry->caller[2],
+                 __entry->caller[3], __entry->caller[4], __entry->caller[5],
+                 __entry->caller[6], __entry->caller[7])
+);
+
+/*
+ * trace_printk entry:
+ */
+FTRACE_ENTRY(bprint, bprint_entry,
+
+        TRACE_BPRINT,
+
+        F_STRUCT(
+                __field(        unsigned long,  ip      )
+                __field(        const char *,   fmt     )
+                __dynamic_array(        u32,    buf     )
+        ),
+
+        F_printk("%08lx fmt:%p",
+                 __entry->ip, __entry->fmt)
+);
+
+FTRACE_ENTRY(print, print_entry,
+
+        TRACE_PRINT,
+
+        F_STRUCT(
+                __field(        unsigned long,  ip      )
+                __dynamic_array(        char,   buf     )
+        ),
+
+        F_printk("%08lx %s",
+                 __entry->ip, __entry->buf)
+);
+
+FTRACE_ENTRY(mmiotrace_rw, trace_mmiotrace_rw,
+
+        TRACE_MMIO_RW,
+
+        F_STRUCT(
+                __field_struct( struct mmiotrace_rw,    rw      )
+                __field_desc(   resource_size_t, rw,    phys    )
+                __field_desc(   unsigned long,  rw,     value   )
+                __field_desc(   unsigned long,  rw,     pc      )
+                __field_desc(   int,            rw,     map_id  )
+                __field_desc(   unsigned char,  rw,     opcode  )
+                __field_desc(   unsigned char,  rw,     width   )
+        ),
+
+        F_printk("%lx %lx %lx %d %x %x",
+                 (unsigned long)__entry->phys, __entry->value, __entry->pc,
+                 __entry->map_id, __entry->opcode, __entry->width)
+);
+
+FTRACE_ENTRY(mmiotrace_map, trace_mmiotrace_map,
+
+        TRACE_MMIO_MAP,
+
+        F_STRUCT(
+                __field_struct( struct mmiotrace_map,   map     )
+                __field_desc(   resource_size_t, map,   phys    )
+                __field_desc(   unsigned long,  map,    virt    )
+                __field_desc(   unsigned long,  map,    len     )
+                __field_desc(   int,            map,    map_id  )
+                __field_desc(   unsigned char,  map,    opcode  )
+        ),
+
+        F_printk("%lx %lx %lx %d %x",
+                 (unsigned long)__entry->phys, __entry->virt, __entry->len,
+                 __entry->map_id, __entry->opcode)
+);
+
+FTRACE_ENTRY(boot_call, trace_boot_call,
+
+        TRACE_BOOT_CALL,
+
+        F_STRUCT(
+                __field_struct( struct boot_trace_call, boot_call       )
+                __field_desc(   pid_t,  boot_call,      caller          )
+                __array_desc(   char,   boot_call,      func,   KSYM_SYMBOL_LEN)
+        ),
+
+        F_printk("%d  %s", __entry->caller, __entry->func)
+);
+
+FTRACE_ENTRY(boot_ret, trace_boot_ret,
+
+        TRACE_BOOT_RET,
+
+        F_STRUCT(
+                __field_struct( struct boot_trace_ret,  boot_ret        )
+                __array_desc(   char,   boot_ret,       func,   KSYM_SYMBOL_LEN)
+                __field_desc(   int,    boot_ret,       result          )
+                __field_desc(   unsigned long, boot_ret, duration       )
+        ),
+
+        F_printk("%s %d %lx",
+                 __entry->func, __entry->result, __entry->duration)
+);
+
+#define TRACE_FUNC_SIZE 30
+#define TRACE_FILE_SIZE 20
+
+FTRACE_ENTRY(branch, trace_branch,
+
+        TRACE_BRANCH,
+
+        F_STRUCT(
+                __field(        unsigned int,   line                            )
+                __array(        char,           func,   TRACE_FUNC_SIZE+1       )
+                __array(        char,           file,   TRACE_FILE_SIZE+1       )
+                __field(        char,           correct                         )
+        ),
+
+        F_printk("%u:%s:%s (%u)",
+                 __entry->line,
+                 __entry->func, __entry->file, __entry->correct)
+);
+
+FTRACE_ENTRY(hw_branch, hw_branch_entry,
+
+        TRACE_HW_BRANCHES,
+
+        F_STRUCT(
+                __field(        u64,    from    )
+                __field(        u64,    to      )
+        ),
+
+        F_printk("from: %llx to: %llx", __entry->from, __entry->to)
+);
+
+FTRACE_ENTRY(power, trace_power,
+
+        TRACE_POWER,
+
+        F_STRUCT(
+                __field_struct( struct power_trace,     state_data      )
+                __field_desc(   s64,    state_data,     stamp           )
+                __field_desc(   s64,    state_data,     end             )
+                __field_desc(   int,    state_data,     type            )
+                __field_desc(   int,    state_data,     state           )
+        ),
+
+        F_printk("%llx->%llx type:%u state:%u",
+                 __entry->stamp, __entry->end,
+                 __entry->type, __entry->state)
+);
+
+FTRACE_ENTRY(kmem_alloc, kmemtrace_alloc_entry,
+
+        TRACE_KMEM_ALLOC,
+
+        F_STRUCT(
+                __field(        enum kmemtrace_type_id, type_id         )
+                __field(        unsigned long,          call_site       )
+                __field(        const void *,           ptr             )
+                __field(        size_t,                 bytes_req       )
+                __field(        size_t,                 bytes_alloc     )
+                __field(        gfp_t,                  gfp_flags       )
+                __field(        int,                    node            )
+        ),
+
+        F_printk("type:%u call_site:%lx ptr:%p req:%zi alloc:%zi"
+                 " flags:%x node:%d",
+                 __entry->type_id, __entry->call_site, __entry->ptr,
+                 __entry->bytes_req, __entry->bytes_alloc,
+                 __entry->gfp_flags, __entry->node)
+);
+
+FTRACE_ENTRY(kmem_free, kmemtrace_free_entry,
+
+        TRACE_KMEM_FREE,
+
+        F_STRUCT(
+                __field(        enum kmemtrace_type_id, type_id         )
+                __field(        unsigned long,          call_site       )
+                __field(        const void *,           ptr             )
+        ),
+
+        F_printk("type:%u call_site:%lx ptr:%p",
+                 __entry->type_id, __entry->call_site, __entry->ptr)
+);
diff --git a/kernel/trace/trace_event_profile.c b/kernel/trace/trace_event_profile.c
index 5b5895afecfe..55a25c933d15 100644
--- a/kernel/trace/trace_event_profile.c
+++ b/kernel/trace/trace_event_profile.c
@@ -5,6 +5,7 @@
  *
  */
 
+#include <linux/module.h>
 #include "trace.h"
 
 int ftrace_profile_enable(int event_id)
@@ -14,7 +15,8 @@ int ftrace_profile_enable(int event_id)
 
         mutex_lock(&event_mutex);
         list_for_each_entry(event, &ftrace_events, list) {
-                if (event->id == event_id) {
+                if (event->id == event_id && event->profile_enable &&
+                    try_module_get(event->mod)) {
                         ret = event->profile_enable(event);
                         break;
                 }
@@ -32,6 +34,7 @@ void ftrace_profile_disable(int event_id)
         list_for_each_entry(event, &ftrace_events, list) {
                 if (event->id == event_id) {
                         event->profile_disable(event);
+                        module_put(event->mod);
                         break;
                 }
         }
diff --git a/kernel/trace/trace_event_types.h b/kernel/trace/trace_event_types.h
deleted file mode 100644
index 5e32e375134d..000000000000
--- a/kernel/trace/trace_event_types.h
+++ /dev/null
@@ -1,175 +0,0 @@
-#undef TRACE_SYSTEM
-#define TRACE_SYSTEM    ftrace
-
-/*
- * We cheat and use the proto type field as the ID
- * and args as the entry type (minus 'struct')
- */
-TRACE_EVENT_FORMAT(function, TRACE_FN, ftrace_entry, ignore,
-        TRACE_STRUCT(
-                TRACE_FIELD(unsigned long, ip, ip)
-                TRACE_FIELD(unsigned long, parent_ip, parent_ip)
-        ),
-        TP_RAW_FMT(" %lx <-- %lx")
-);
-
-TRACE_EVENT_FORMAT(funcgraph_entry, TRACE_GRAPH_ENT,
-                   ftrace_graph_ent_entry, ignore,
-        TRACE_STRUCT(
-                TRACE_FIELD(unsigned long, graph_ent.func, func)
-                TRACE_FIELD(int, graph_ent.depth, depth)
-        ),
-        TP_RAW_FMT("--> %lx (%d)")
-);
-
-TRACE_EVENT_FORMAT(funcgraph_exit, TRACE_GRAPH_RET,
-                   ftrace_graph_ret_entry, ignore,
-        TRACE_STRUCT(
-                TRACE_FIELD(unsigned long, ret.func, func)
-                TRACE_FIELD(int, ret.depth, depth)
-        ),
-        TP_RAW_FMT("<-- %lx (%d)")
-);
-
-TRACE_EVENT_FORMAT(wakeup, TRACE_WAKE, ctx_switch_entry, ignore,
-        TRACE_STRUCT(
-                TRACE_FIELD(unsigned int, prev_pid, prev_pid)
-                TRACE_FIELD(unsigned char, prev_prio, prev_prio)
-                TRACE_FIELD(unsigned char, prev_state, prev_state)
-                TRACE_FIELD(unsigned int, next_pid, next_pid)
-                TRACE_FIELD(unsigned char, next_prio, next_prio)
-                TRACE_FIELD(unsigned char, next_state, next_state)
-                TRACE_FIELD(unsigned int, next_cpu, next_cpu)
-        ),
-        TP_RAW_FMT("%u:%u:%u  ==+ %u:%u:%u [%03u]")
-);
-
-TRACE_EVENT_FORMAT(context_switch, TRACE_CTX, ctx_switch_entry, ignore,
-        TRACE_STRUCT(
-                TRACE_FIELD(unsigned int, prev_pid, prev_pid)
-                TRACE_FIELD(unsigned char, prev_prio, prev_prio)
-                TRACE_FIELD(unsigned char, prev_state, prev_state)
-                TRACE_FIELD(unsigned int, next_pid, next_pid)
-                TRACE_FIELD(unsigned char, next_prio, next_prio)
-                TRACE_FIELD(unsigned char, next_state, next_state)
-                TRACE_FIELD(unsigned int, next_cpu, next_cpu)
-        ),
-        TP_RAW_FMT("%u:%u:%u  ==+ %u:%u:%u [%03u]")
-);
-
-TRACE_EVENT_FORMAT_NOFILTER(special, TRACE_SPECIAL, special_entry, ignore,
-        TRACE_STRUCT(
-                TRACE_FIELD(unsigned long, arg1, arg1)
-                TRACE_FIELD(unsigned long, arg2, arg2)
-                TRACE_FIELD(unsigned long, arg3, arg3)
-        ),
-        TP_RAW_FMT("(%08lx) (%08lx) (%08lx)")
-);
-
-/*
- * Stack-trace entry:
- */
-
-/* #define FTRACE_STACK_ENTRIES   8 */
-
-TRACE_EVENT_FORMAT(kernel_stack, TRACE_STACK, stack_entry, ignore,
-        TRACE_STRUCT(
-                TRACE_FIELD(unsigned long, caller[0], stack0)
-                TRACE_FIELD(unsigned long, caller[1], stack1)
-                TRACE_FIELD(unsigned long, caller[2], stack2)
-                TRACE_FIELD(unsigned long, caller[3], stack3)
-                TRACE_FIELD(unsigned long, caller[4], stack4)
-                TRACE_FIELD(unsigned long, caller[5], stack5)
-                TRACE_FIELD(unsigned long, caller[6], stack6)
-                TRACE_FIELD(unsigned long, caller[7], stack7)
-        ),
-        TP_RAW_FMT("\t=> (%08lx)\n\t=> (%08lx)\n\t=> (%08lx)\n\t=> (%08lx)\n"
-                 "\t=> (%08lx)\n\t=> (%08lx)\n\t=> (%08lx)\n\t=> (%08lx)\n")
-);
-
-TRACE_EVENT_FORMAT(user_stack, TRACE_USER_STACK, userstack_entry, ignore,
-        TRACE_STRUCT(
-                TRACE_FIELD(unsigned long, caller[0], stack0)
-                TRACE_FIELD(unsigned long, caller[1], stack1)
-                TRACE_FIELD(unsigned long, caller[2], stack2)
-                TRACE_FIELD(unsigned long, caller[3], stack3)
-                TRACE_FIELD(unsigned long, caller[4], stack4)
-                TRACE_FIELD(unsigned long, caller[5], stack5)
-                TRACE_FIELD(unsigned long, caller[6], stack6)
-                TRACE_FIELD(unsigned long, caller[7], stack7)
-        ),
-        TP_RAW_FMT("\t=> (%08lx)\n\t=> (%08lx)\n\t=> (%08lx)\n\t=> (%08lx)\n"
-                 "\t=> (%08lx)\n\t=> (%08lx)\n\t=> (%08lx)\n\t=> (%08lx)\n")
-);
-
-TRACE_EVENT_FORMAT(bprint, TRACE_BPRINT, bprint_entry, ignore,
-        TRACE_STRUCT(
-                TRACE_FIELD(unsigned long, ip, ip)
-                TRACE_FIELD(char *, fmt, fmt)
-                TRACE_FIELD_ZERO_CHAR(buf)
-        ),
-        TP_RAW_FMT("%08lx (%d) fmt:%p %s")
-);
-
-TRACE_EVENT_FORMAT(print, TRACE_PRINT, print_entry, ignore,
-        TRACE_STRUCT(
-                TRACE_FIELD(unsigned long, ip, ip)
-                TRACE_FIELD_ZERO_CHAR(buf)
-        ),
-        TP_RAW_FMT("%08lx (%d) fmt:%p %s")
-);
-
-TRACE_EVENT_FORMAT(branch, TRACE_BRANCH, trace_branch, ignore,
-        TRACE_STRUCT(
-                TRACE_FIELD(unsigned int, line, line)
-                TRACE_FIELD_SPECIAL(char func[TRACE_FUNC_SIZE+1], func,
-                                    TRACE_FUNC_SIZE+1, func)
-                TRACE_FIELD_SPECIAL(char file[TRACE_FUNC_SIZE+1], file,
-                                    TRACE_FUNC_SIZE+1, file)
-                TRACE_FIELD(char, correct, correct)
-        ),
-        TP_RAW_FMT("%u:%s:%s (%u)")
-);
-
-TRACE_EVENT_FORMAT(hw_branch, TRACE_HW_BRANCHES, hw_branch_entry, ignore,
-        TRACE_STRUCT(
-                TRACE_FIELD(u64, from, from)
-                TRACE_FIELD(u64, to, to)
-        ),
-        TP_RAW_FMT("from: %llx to: %llx")
-);
-
-TRACE_EVENT_FORMAT(power, TRACE_POWER, trace_power, ignore,
-        TRACE_STRUCT(
-                TRACE_FIELD_SIGN(ktime_t, state_data.stamp, stamp, 1)
-                TRACE_FIELD_SIGN(ktime_t, state_data.end, end, 1)
-                TRACE_FIELD(int, state_data.type, type)
-                TRACE_FIELD(int, state_data.state, state)
-        ),
-        TP_RAW_FMT("%llx->%llx type:%u state:%u")
-);
-
-TRACE_EVENT_FORMAT(kmem_alloc, TRACE_KMEM_ALLOC, kmemtrace_alloc_entry, ignore,
-        TRACE_STRUCT(
-                TRACE_FIELD(enum kmemtrace_type_id, type_id, type_id)
-                TRACE_FIELD(unsigned long, call_site, call_site)
-                TRACE_FIELD(const void *, ptr, ptr)
-                TRACE_FIELD(size_t, bytes_req, bytes_req)
-                TRACE_FIELD(size_t, bytes_alloc, bytes_alloc)
-                TRACE_FIELD(gfp_t, gfp_flags, gfp_flags)
-                TRACE_FIELD(int, node, node)
-        ),
-        TP_RAW_FMT("type:%u call_site:%lx ptr:%p req:%lu alloc:%lu"
-                 " flags:%x node:%d")
-);
-
-TRACE_EVENT_FORMAT(kmem_free, TRACE_KMEM_FREE, kmemtrace_free_entry, ignore,
-        TRACE_STRUCT(
-                TRACE_FIELD(enum kmemtrace_type_id, type_id, type_id)
-                TRACE_FIELD(unsigned long, call_site, call_site)
-                TRACE_FIELD(const void *, ptr, ptr)
-        ),
-        TP_RAW_FMT("type:%u call_site:%lx ptr:%p")
-);
-
-#undef TRACE_SYSTEM
diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c
index aa08be69a1b6..56c260b83a9c 100644
--- a/kernel/trace/trace_events.c
+++ b/kernel/trace/trace_events.c
@@ -17,16 +17,20 @@
 #include <linux/ctype.h>
 #include <linux/delay.h>
 
+#include <asm/setup.h>
+
 #include "trace_output.h"
 
+#undef TRACE_SYSTEM
 #define TRACE_SYSTEM "TRACE_SYSTEM"
 
 DEFINE_MUTEX(event_mutex);
 
 LIST_HEAD(ftrace_events);
 
-int trace_define_field(struct ftrace_event_call *call, char *type,
-                       char *name, int offset, int size, int is_signed)
+int trace_define_field(struct ftrace_event_call *call, const char *type,
+                       const char *name, int offset, int size, int is_signed,
+                       int filter_type)
 {
         struct ftrace_event_field *field;
 
@@ -42,9 +46,15 @@ int trace_define_field(struct ftrace_event_call *call, char *type,
         if (!field->type)
                 goto err;
 
+        if (filter_type == FILTER_OTHER)
+                field->filter_type = filter_assign_type(type);
+        else
+                field->filter_type = filter_type;
+
         field->offset = offset;
         field->size = size;
         field->is_signed = is_signed;
+
         list_add(&field->link, &call->fields);
 
         return 0;
@@ -60,6 +70,29 @@ err:
 }
 EXPORT_SYMBOL_GPL(trace_define_field);
 
+#define __common_field(type, item)                                      \
+        ret = trace_define_field(call, #type, "common_" #item,          \
+                                 offsetof(typeof(ent), item),           \
+                                 sizeof(ent.item),                      \
+                                 is_signed_type(type), FILTER_OTHER);   \
+        if (ret)                                                        \
+                return ret;
+
+int trace_define_common_fields(struct ftrace_event_call *call)
+{
+        int ret;
+        struct trace_entry ent;
+
+        __common_field(unsigned short, type);
+        __common_field(unsigned char, flags);
+        __common_field(unsigned char, preempt_count);
+        __common_field(int, pid);
+        __common_field(int, lock_depth);
+
+        return ret;
+}
+EXPORT_SYMBOL_GPL(trace_define_common_fields);
+
 #ifdef CONFIG_MODULES
 
 static void trace_destroy_fields(struct ftrace_event_call *call)
@@ -84,14 +117,14 @@ static void ftrace_event_enable_disable(struct ftrace_event_call *call,
                 if (call->enabled) {
                         call->enabled = 0;
                         tracing_stop_cmdline_record();
-                        call->unregfunc();
+                        call->unregfunc(call->data);
                 }
                 break;
         case 1:
                 if (!call->enabled) {
                         call->enabled = 1;
                         tracing_start_cmdline_record();
-                        call->regfunc();
+                        call->regfunc(call->data);
                 }
                 break;
         }
@@ -198,11 +231,9 @@ static ssize_t
 ftrace_event_write(struct file *file, const char __user *ubuf,
                    size_t cnt, loff_t *ppos)
 {
+        struct trace_parser parser;
         size_t read = 0;
-        int i, set = 1;
         ssize_t ret;
-        char *buf;
-        char ch;
 
         if (!cnt || cnt < 0)
                 return 0;
@@ -211,60 +242,28 @@ ftrace_event_write(struct file *file, const char __user *ubuf,
         if (ret < 0)
                 return ret;
 
-        ret = get_user(ch, ubuf++);
-        if (ret)
-                return ret;
-        read++;
-        cnt--;
-
-        /* skip white space */
-        while (cnt && isspace(ch)) {
-                ret = get_user(ch, ubuf++);
-                if (ret)
-                        return ret;
-                read++;
-                cnt--;
-        }
-
-        /* Only white space found? */
-        if (isspace(ch)) {
-                file->f_pos += read;
-                ret = read;
-                return ret;
-        }
-
-        buf = kmalloc(EVENT_BUF_SIZE+1, GFP_KERNEL);
-        if (!buf)
+        if (trace_parser_get_init(&parser, EVENT_BUF_SIZE + 1))
                 return -ENOMEM;
 
-        if (cnt > EVENT_BUF_SIZE)
-                cnt = EVENT_BUF_SIZE;
+        read = trace_get_user(&parser, ubuf, cnt, ppos);
+
+        if (trace_parser_loaded((&parser))) {
+                int set = 1;
 
-        i = 0;
-        while (cnt && !isspace(ch)) {
-                if (!i && ch == '!')
+                if (*parser.buffer == '!')
                         set = 0;
-                else
-                        buf[i++] = ch;
 
-                ret = get_user(ch, ubuf++);
+                parser.buffer[parser.idx] = 0;
+
+                ret = ftrace_set_clr_event(parser.buffer + !set, set);
                 if (ret)
-                        goto out_free;
-                read++;
-                cnt--;
+                        goto out_put;
         }
-        buf[i] = 0;
-
-        file->f_pos += read;
-
-        ret = ftrace_set_clr_event(buf, set);
-        if (ret)
-                goto out_free;
 
         ret = read;
 
- out_free:
-        kfree(buf);
+ out_put:
+        trace_parser_put(&parser);
 
         return ret;
 }
@@ -300,10 +299,18 @@ t_next(struct seq_file *m, void *v, loff_t *pos)
 
 static void *t_start(struct seq_file *m, loff_t *pos)
 {
+        struct ftrace_event_call *call = NULL;
+        loff_t l;
+
         mutex_lock(&event_mutex);
-        if (*pos == 0)
-                m->private = ftrace_events.next;
-        return t_next(m, NULL, pos);
+
+        m->private = ftrace_events.next;
+        for (l = 0; l <= *pos; ) {
+                call = t_next(m, NULL, &l);
+                if (!call)
+                        break;
+        }
+        return call;
 }
 
 static void *
@@ -332,10 +339,18 @@ s_next(struct seq_file *m, void *v, loff_t *pos)
 
 static void *s_start(struct seq_file *m, loff_t *pos)
 {
+        struct ftrace_event_call *call = NULL;
+        loff_t l;
+
         mutex_lock(&event_mutex);
-        if (*pos == 0)
-                m->private = ftrace_events.next;
-        return s_next(m, NULL, pos);
+
+        m->private = ftrace_events.next;
+        for (l = 0; l <= *pos; ) {
+                call = s_next(m, NULL, &l);
+                if (!call)
+                        break;
+        }
+        return call;
 }
 
 static int t_show(struct seq_file *m, void *v)
@@ -360,7 +375,7 @@ ftrace_event_seq_open(struct inode *inode, struct file *file)
         const struct seq_operations *seq_ops;
 
         if ((file->f_mode & FMODE_WRITE) &&
-            !(file->f_flags & O_APPEND))
+            (file->f_flags & O_TRUNC))
                 ftrace_clear_events();
 
         seq_ops = inode->i_private;
@@ -530,7 +545,7 @@ static int trace_write_header(struct trace_seq *s)
                                 FIELD(unsigned char, flags),
                                 FIELD(unsigned char, preempt_count),
                                 FIELD(int, pid),
-                                FIELD(int, tgid));
+                                FIELD(int, lock_depth));
 }
 
 static ssize_t
@@ -558,7 +573,7 @@ event_format_read(struct file *filp, char __user *ubuf, size_t cnt,
         trace_seq_printf(s, "format:\n");
         trace_write_header(s);
 
-        r = call->show_format(s);
+        r = call->show_format(call, s);
         if (!r) {
                 /*
                  * ug!  The format output is bigger than a PAGE!!
@@ -833,8 +848,10 @@ event_subsystem_dir(const char *name, struct dentry *d_events)
 
         /* First see if we did not already create this dir */
         list_for_each_entry(system, &event_subsystems, list) {
-                if (strcmp(system->name, name) == 0)
+                if (strcmp(system->name, name) == 0) {
+                        system->nr_events++;
                         return system->entry;
+                }
         }
 
         /* need to create new entry */
@@ -853,6 +870,7 @@ event_subsystem_dir(const char *name, struct dentry *d_events)
                 return d_events;
         }
 
+        system->nr_events = 1;
         system->name = kstrdup(name, GFP_KERNEL);
         if (!system->name) {
                 debugfs_remove(system->entry);
@@ -904,15 +922,6 @@ event_create_dir(struct ftrace_event_call *call, struct dentry *d_events,
         if (strcmp(call->system, TRACE_SYSTEM) != 0)
                 d_events = event_subsystem_dir(call->system, d_events);
 
-        if (call->raw_init) {
-                ret = call->raw_init();
-                if (ret < 0) {
-                        pr_warning("Could not initialize trace point"
-                                   " events/%s\n", call->name);
-                        return ret;
-                }
-        }
-
         call->dir = debugfs_create_dir(call->name, d_events);
         if (!call->dir) {
                 pr_warning("Could not create debugfs "
@@ -924,12 +933,12 @@ event_create_dir(struct ftrace_event_call *call, struct dentry *d_events,
                 entry = trace_create_file("enable", 0644, call->dir, call,
                                           enable);
 
-        if (call->id)
+        if (call->id && call->profile_enable)
                 entry = trace_create_file("id", 0444, call->dir, call,
                                           id);
 
         if (call->define_fields) {
-                ret = call->define_fields();
+                ret = call->define_fields(call);
                 if (ret < 0) {
                         pr_warning("Could not initialize trace point"
                                    " events/%s\n", call->name);
@@ -971,6 +980,32 @@ struct ftrace_module_file_ops {
         struct file_operations          filter;
 };
 
+static void remove_subsystem_dir(const char *name)
+{
+        struct event_subsystem *system;
+
+        if (strcmp(name, TRACE_SYSTEM) == 0)
+                return;
+
+        list_for_each_entry(system, &event_subsystems, list) {
+                if (strcmp(system->name, name) == 0) {
+                        if (!--system->nr_events) {
+                                struct event_filter *filter = system->filter;
+
+                                debugfs_remove_recursive(system->entry);
+                                list_del(&system->list);
+                                if (filter) {
+                                        kfree(filter->filter_string);
+                                        kfree(filter);
+                                }
+                                kfree(system->name);
+                                kfree(system);
+                        }
+                        break;
+                }
+        }
+}
+
 static struct ftrace_module_file_ops *
 trace_create_file_ops(struct module *mod)
 {
@@ -1011,6 +1046,7 @@ static void trace_module_add_events(struct module *mod)
         struct ftrace_module_file_ops *file_ops = NULL;
         struct ftrace_event_call *call, *start, *end;
         struct dentry *d_events;
+        int ret;
 
         start = mod->trace_events;
         end = mod->trace_events + mod->num_trace_events;
@@ -1026,7 +1062,15 @@ static void trace_module_add_events(struct module *mod)
                 /* The linker may leave blanks */
                 if (!call->name)
                         continue;
-
+                if (call->raw_init) {
+                        ret = call->raw_init();
+                        if (ret < 0) {
+                                if (ret != -ENOSYS)
+                                        pr_warning("Could not initialize trace "
+                                        "point events/%s\n", call->name);
+                                continue;
+                        }
+                }
                 /*
                  * This module has events, create file ops for this module
                  * if not already done.
@@ -1061,6 +1105,7 @@ static void trace_module_remove_events(struct module *mod)
                         list_del(&call->list);
                         trace_destroy_fields(call);
                         destroy_preds(call);
+                        remove_subsystem_dir(call->system);
                 }
         }
 
@@ -1109,7 +1154,7 @@ static int trace_module_notify(struct notifier_block *self,
 }
 #endif /* CONFIG_MODULES */
 
-struct notifier_block trace_module_nb = {
+static struct notifier_block trace_module_nb = {
         .notifier_call = trace_module_notify,
         .priority = 0,
 };
@@ -1117,6 +1162,18 @@ struct notifier_block trace_module_nb = {
 extern struct ftrace_event_call __start_ftrace_events[];
 extern struct ftrace_event_call __stop_ftrace_events[];
 
+static char bootup_event_buf[COMMAND_LINE_SIZE] __initdata;
+
+static __init int setup_trace_event(char *str)
+{
+        strlcpy(bootup_event_buf, str, COMMAND_LINE_SIZE);
+        ring_buffer_expanded = 1;
+        tracing_selftest_disabled = 1;
+
+        return 1;
+}
+__setup("trace_event=", setup_trace_event);
+
 static __init int event_trace_init(void)
 {
         struct ftrace_event_call *call;
@@ -1124,6 +1181,8 @@ static __init int event_trace_init(void)
         struct dentry *entry;
         struct dentry *d_events;
         int ret;
+        char *buf = bootup_event_buf;
+        char *token;
 
         d_tracer = tracing_init_dentry();
         if (!d_tracer)
@@ -1163,12 +1222,34 @@ static __init int event_trace_init(void)
                 /* The linker may leave blanks */
                 if (!call->name)
                         continue;
+                if (call->raw_init) {
+                        ret = call->raw_init();
+                        if (ret < 0) {
+                                if (ret != -ENOSYS)
+                                        pr_warning("Could not initialize trace "
+                                        "point events/%s\n", call->name);
+                                continue;
+                        }
+                }
                 list_add(&call->list, &ftrace_events);
                 event_create_dir(call, d_events, &ftrace_event_id_fops,
                                  &ftrace_enable_fops, &ftrace_event_filter_fops,
                                  &ftrace_event_format_fops);
         }
 
+        while (true) {
+                token = strsep(&buf, ",");
+
+                if (!token)
+                        break;
+                if (!*token)
+                        continue;
+
+                ret = ftrace_set_clr_event(token, 1);
+                if (ret)
+                        pr_warning("Failed to enable trace event: %s\n", token);
+        }
+
         ret = register_module_notifier(&trace_module_nb);
         if (ret)
                 pr_warning("Failed to register trace events module notifier\n");
@@ -1245,6 +1326,18 @@ static __init void event_trace_self_tests(void)
                 if (!call->regfunc)
                         continue;
 
+/*
+ * Testing syscall events here is pretty useless, but
+ * we still do it if configured. But this is time consuming.
+ * What we really need is a user thread to perform the
+ * syscalls as we test.
+ */
+#ifndef CONFIG_EVENT_TRACE_TEST_SYSCALLS
+                if (call->system &&
+                    strcmp(call->system, "syscalls") == 0)
+                        continue;
+#endif
+
                 pr_info("Testing event %s: ", call->name);
 
                 /*
@@ -1318,12 +1411,13 @@ static __init void event_trace_self_tests(void)
 
 #ifdef CONFIG_FUNCTION_TRACER
 
-static DEFINE_PER_CPU(atomic_t, test_event_disable);
+static DEFINE_PER_CPU(atomic_t, ftrace_test_event_disable);
 
 static void
 function_test_events_call(unsigned long ip, unsigned long parent_ip)
 {
         struct ring_buffer_event *event;
+        struct ring_buffer *buffer;
         struct ftrace_entry *entry;
         unsigned long flags;
         long disabled;
@@ -1334,14 +1428,15 @@ function_test_events_call(unsigned long ip, unsigned long parent_ip)
         pc = preempt_count();
         resched = ftrace_preempt_disable();
         cpu = raw_smp_processor_id();
-        disabled = atomic_inc_return(&per_cpu(test_event_disable, cpu));
+        disabled = atomic_inc_return(&per_cpu(ftrace_test_event_disable, cpu));
 
         if (disabled != 1)
                 goto out;
 
         local_save_flags(flags);
 
-        event = trace_current_buffer_lock_reserve(TRACE_FN, sizeof(*entry),
+        event = trace_current_buffer_lock_reserve(&buffer,
+                                                  TRACE_FN, sizeof(*entry),
                                                   flags, pc);
         if (!event)
                 goto out;
@@ -1349,10 +1444,10 @@ function_test_events_call(unsigned long ip, unsigned long parent_ip)
         entry->ip                       = ip;
         entry->parent_ip                = parent_ip;
 
-        trace_nowake_buffer_unlock_commit(event, flags, pc);
+        trace_nowake_buffer_unlock_commit(buffer, event, flags, pc);
 
  out:
-        atomic_dec(&per_cpu(test_event_disable, cpu));
+        atomic_dec(&per_cpu(ftrace_test_event_disable, cpu));
         ftrace_preempt_enable(resched);
 }
 
@@ -1376,10 +1471,10 @@ static __init void event_trace_self_test_with_function(void)
 
 static __init int event_trace_self_tests_init(void)
 {
-
-        event_trace_self_tests();
-
-        event_trace_self_test_with_function();
+        if (!tracing_selftest_disabled) {
+                event_trace_self_tests();
+                event_trace_self_test_with_function();
+        }
 
         return 0;
 }
diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c
index db6e54bdb596..23245785927f 100644
--- a/kernel/trace/trace_events_filter.c
+++ b/kernel/trace/trace_events_filter.c
@@ -27,8 +27,6 @@
 #include "trace.h"
 #include "trace_output.h"
 
-static DEFINE_MUTEX(filter_mutex);
-
 enum filter_op_ids
 {
         OP_OR,
@@ -123,6 +121,47 @@ struct filter_parse_state {
         } operand;
 };
 
+#define DEFINE_COMPARISON_PRED(type)                                    \
+static int filter_pred_##type(struct filter_pred *pred, void *event,    \
+                              int val1, int val2)                       \
+{                                                                       \
+        type *addr = (type *)(event + pred->offset);                    \
+        type val = (type)pred->val;                                     \
+        int match = 0;                                                  \
+                                                                        \
+        switch (pred->op) {                                             \
+        case OP_LT:                                                     \
+                match = (*addr < val);                                  \
+                break;                                                  \
+        case OP_LE:                                                     \
+                match = (*addr <= val);                                 \
+                break;                                                  \
+        case OP_GT:                                                     \
+                match = (*addr > val);                                  \
+                break;                                                  \
+        case OP_GE:                                                     \
+                match = (*addr >= val);                                 \
+                break;                                                  \
+        default:                                                        \
+                break;                                                  \
+        }                                                               \
+                                                                        \
+        return match;                                                   \
+}
+
+#define DEFINE_EQUALITY_PRED(size)                                      \
+static int filter_pred_##size(struct filter_pred *pred, void *event,    \
+                              int val1, int val2)                       \
+{                                                                       \
+        u##size *addr = (u##size *)(event + pred->offset);              \
+        u##size val = (u##size)pred->val;                               \
+        int match;                                                      \
+                                                                        \
+        match = (val == *addr) ^ pred->not;                             \
+                                                                        \
+        return match;                                                   \
+}
+
 DEFINE_COMPARISON_PRED(s64);
 DEFINE_COMPARISON_PRED(u64);
 DEFINE_COMPARISON_PRED(s32);
@@ -165,6 +204,20 @@ static int filter_pred_string(struct filter_pred *pred, void *event,
         return match;
 }
 
+/* Filter predicate for char * pointers */
+static int filter_pred_pchar(struct filter_pred *pred, void *event,
+                             int val1, int val2)
+{
+        char **addr = (char **)(event + pred->offset);
+        int cmp, match;
+
+        cmp = strncmp(*addr, pred->str_val, pred->str_len);
+
+        match = (!cmp) ^ pred->not;
+
+        return match;
+}
+
 /*
  * Filter predicate for dynamic sized arrays of characters.
  * These are implemented through a list of strings at the end
@@ -178,11 +231,13 @@ static int filter_pred_string(struct filter_pred *pred, void *event,
 static int filter_pred_strloc(struct filter_pred *pred, void *event,
                               int val1, int val2)
 {
-        int str_loc = *(int *)(event + pred->offset);
+        u32 str_item = *(u32 *)(event + pred->offset);
+        int str_loc = str_item & 0xffff;
+        int str_len = str_item >> 16;
         char *addr = (char *)(event + str_loc);
         int cmp, match;
 
-        cmp = strncmp(addr, pred->str_val, pred->str_len);
+        cmp = strncmp(addr, pred->str_val, str_len);
 
         match = (!cmp) ^ pred->not;
 
@@ -294,12 +349,12 @@ void print_event_filter(struct ftrace_event_call *call, struct trace_seq *s)
 {
         struct event_filter *filter = call->filter;
 
-        mutex_lock(&filter_mutex);
-        if (filter->filter_string)
+        mutex_lock(&event_mutex);
+        if (filter && filter->filter_string)
                 trace_seq_printf(s, "%s\n", filter->filter_string);
         else
                 trace_seq_printf(s, "none\n");
-        mutex_unlock(&filter_mutex);
+        mutex_unlock(&event_mutex);
 }
 
 void print_subsystem_event_filter(struct event_subsystem *system,
@@ -307,12 +362,12 @@ void print_subsystem_event_filter(struct event_subsystem *system,
 {
         struct event_filter *filter = system->filter;
 
-        mutex_lock(&filter_mutex);
-        if (filter->filter_string)
+        mutex_lock(&event_mutex);
+        if (filter && filter->filter_string)
                 trace_seq_printf(s, "%s\n", filter->filter_string);
         else
                 trace_seq_printf(s, "none\n");
-        mutex_unlock(&filter_mutex);
+        mutex_unlock(&event_mutex);
 }
 
 static struct ftrace_event_field *
@@ -376,26 +431,32 @@ void destroy_preds(struct ftrace_event_call *call)
         struct event_filter *filter = call->filter;
         int i;
 
+        if (!filter)
+                return;
+
         for (i = 0; i < MAX_FILTER_PRED; i++) {
                 if (filter->preds[i])
                         filter_free_pred(filter->preds[i]);
         }
         kfree(filter->preds);
+        kfree(filter->filter_string);
         kfree(filter);
         call->filter = NULL;
 }
 
-int init_preds(struct ftrace_event_call *call)
+static int init_preds(struct ftrace_event_call *call)
 {
         struct event_filter *filter;
         struct filter_pred *pred;
         int i;
 
+        if (call->filter)
+                return 0;
+
         filter = call->filter = kzalloc(sizeof(*filter), GFP_KERNEL);
         if (!call->filter)
                 return -ENOMEM;
 
-        call->filter_active = 0;
         filter->n_preds = 0;
 
         filter->preds = kzalloc(MAX_FILTER_PRED * sizeof(pred), GFP_KERNEL);
@@ -417,33 +478,56 @@ oom:
 
         return -ENOMEM;
 }
-EXPORT_SYMBOL_GPL(init_preds);
 
-static void filter_free_subsystem_preds(struct event_subsystem *system)
+static int init_subsystem_preds(struct event_subsystem *system)
 {
-        struct event_filter *filter = system->filter;
         struct ftrace_event_call *call;
-        int i;
+        int err;
 
-        if (filter->n_preds) {
-                for (i = 0; i < filter->n_preds; i++)
-                        filter_free_pred(filter->preds[i]);
-                kfree(filter->preds);
-                filter->preds = NULL;
-                filter->n_preds = 0;
+        list_for_each_entry(call, &ftrace_events, list) {
+                if (!call->define_fields)
+                        continue;
+
+                if (strcmp(call->system, system->name) != 0)
+                        continue;
+
+                err = init_preds(call);
+                if (err)
+                        return err;
         }
 
-        mutex_lock(&event_mutex);
+        return 0;
+}
+
+enum {
+        FILTER_DISABLE_ALL,
+        FILTER_INIT_NO_RESET,
+        FILTER_SKIP_NO_RESET,
+};
+
+static void filter_free_subsystem_preds(struct event_subsystem *system,
+                                        int flag)
+{
+        struct ftrace_event_call *call;
+
         list_for_each_entry(call, &ftrace_events, list) {
                 if (!call->define_fields)
                         continue;
 
-                if (!strcmp(call->system, system->name)) {
-                        filter_disable_preds(call);
-                        remove_filter_string(call->filter);
+                if (strcmp(call->system, system->name) != 0)
+                        continue;
+
+                if (flag == FILTER_INIT_NO_RESET) {
+                        call->filter->no_reset = false;
+                        continue;
                 }
+
+                if (flag == FILTER_SKIP_NO_RESET && call->filter->no_reset)
+                        continue;
+
+                filter_disable_preds(call);
+                remove_filter_string(call->filter);
         }
-        mutex_unlock(&event_mutex);
 }
 
 static int filter_add_pred_fn(struct filter_parse_state *ps,
@@ -471,12 +555,7 @@ static int filter_add_pred_fn(struct filter_parse_state *ps,
         return 0;
 }
 
-enum {
-        FILTER_STATIC_STRING = 1,
-        FILTER_DYN_STRING
-};
-
-static int is_string_field(const char *type)
+int filter_assign_type(const char *type)
 {
         if (strstr(type, "__data_loc") && strstr(type, "char"))
                 return FILTER_DYN_STRING;
@@ -484,12 +563,19 @@ static int is_string_field(const char *type)
         if (strchr(type, '[') && strstr(type, "char"))
                 return FILTER_STATIC_STRING;
 
-        return 0;
+        return FILTER_OTHER;
+}
+
+static bool is_string_field(struct ftrace_event_field *field)
+{
+        return field->filter_type == FILTER_DYN_STRING ||
+               field->filter_type == FILTER_STATIC_STRING ||
+               field->filter_type == FILTER_PTR_STRING;
 }
 
 static int is_legal_op(struct ftrace_event_field *field, int op)
 {
-        if (is_string_field(field->type) && (op != OP_EQ && op != OP_NE))
+        if (is_string_field(field) && (op != OP_EQ && op != OP_NE))
                 return 0;
 
         return 1;
@@ -540,21 +626,24 @@ static filter_pred_fn_t select_comparison_fn(int op, int field_size,
 
 static int filter_add_pred(struct filter_parse_state *ps,
                            struct ftrace_event_call *call,
-                           struct filter_pred *pred)
+                           struct filter_pred *pred,
+                           bool dry_run)
 {
         struct ftrace_event_field *field;
         filter_pred_fn_t fn;
         unsigned long long val;
-        int string_type;
+        int ret;
 
         pred->fn = filter_pred_none;
 
         if (pred->op == OP_AND) {
                 pred->pop_n = 2;
-                return filter_add_pred_fn(ps, call, pred, filter_pred_and);
+                fn = filter_pred_and;
+                goto add_pred_fn;
         } else if (pred->op == OP_OR) {
                 pred->pop_n = 2;
-                return filter_add_pred_fn(ps, call, pred, filter_pred_or);
+                fn = filter_pred_or;
+                goto add_pred_fn;
         }
 
         field = find_event_field(call, pred->field_name);
@@ -570,62 +659,55 @@ static int filter_add_pred(struct filter_parse_state *ps,
                 return -EINVAL;
         }
 
-        string_type = is_string_field(field->type);
-        if (string_type) {
-                if (string_type == FILTER_STATIC_STRING)
+        if (is_string_field(field)) {
+                pred->str_len = field->size;
+
+                if (field->filter_type == FILTER_STATIC_STRING)
                         fn = filter_pred_string;
-                else
+                else if (field->filter_type == FILTER_DYN_STRING)
                         fn = filter_pred_strloc;
-                pred->str_len = field->size;
-                if (pred->op == OP_NE)
-                        pred->not = 1;
-                return filter_add_pred_fn(ps, call, pred, fn);
+                else {
+                        fn = filter_pred_pchar;
+                        pred->str_len = strlen(pred->str_val);
+                }
         } else {
-                if (strict_strtoull(pred->str_val, 0, &val)) {
+                if (field->is_signed)
+                        ret = strict_strtoll(pred->str_val, 0, &val);
+                else
+                        ret = strict_strtoull(pred->str_val, 0, &val);
+                if (ret) {
                         parse_error(ps, FILT_ERR_ILLEGAL_INTVAL, 0);
                         return -EINVAL;
                 }
                 pred->val = val;
-        }
 
-        fn = select_comparison_fn(pred->op, field->size, field->is_signed);
-        if (!fn) {
-                parse_error(ps, FILT_ERR_INVALID_OP, 0);
-                return -EINVAL;
+                fn = select_comparison_fn(pred->op, field->size,
+                                          field->is_signed);
+                if (!fn) {
+                        parse_error(ps, FILT_ERR_INVALID_OP, 0);
+                        return -EINVAL;
+                }
         }
 
         if (pred->op == OP_NE)
                 pred->not = 1;
 
-        return filter_add_pred_fn(ps, call, pred, fn);
+add_pred_fn:
+        if (!dry_run)
+                return filter_add_pred_fn(ps, call, pred, fn);
+        return 0;
 }
 
 static int filter_add_subsystem_pred(struct filter_parse_state *ps,
                                      struct event_subsystem *system,
                                      struct filter_pred *pred,
-                                     char *filter_string)
+                                     char *filter_string,
+                                     bool dry_run)
 {
-        struct event_filter *filter = system->filter;
         struct ftrace_event_call *call;
         int err = 0;
+        bool fail = true;
 
-        if (!filter->preds) {
-                filter->preds = kzalloc(MAX_FILTER_PRED * sizeof(pred),
-                                        GFP_KERNEL);
-
-                if (!filter->preds)
-                        return -ENOMEM;
-        }
-
-        if (filter->n_preds == MAX_FILTER_PRED) {
-                parse_error(ps, FILT_ERR_TOO_MANY_PREDS, 0);
-                return -ENOSPC;
-        }
-
-        filter->preds[filter->n_preds] = pred;
-        filter->n_preds++;
-
-        mutex_lock(&event_mutex);
         list_for_each_entry(call, &ftrace_events, list) {
 
                 if (!call->define_fields)
@@ -634,18 +716,24 @@ static int filter_add_subsystem_pred(struct filter_parse_state *ps,
                 if (strcmp(call->system, system->name))
                         continue;
 
-                err = filter_add_pred(ps, call, pred);
-                if (err) {
-                        mutex_unlock(&event_mutex);
-                        filter_free_subsystem_preds(system);
-                        parse_error(ps, FILT_ERR_BAD_SUBSYS_FILTER, 0);
-                        goto out;
-                }
-                replace_filter_string(call->filter, filter_string);
+                if (call->filter->no_reset)
+                        continue;
+
+                err = filter_add_pred(ps, call, pred, dry_run);
+                if (err)
+                        call->filter->no_reset = true;
+                else
+                        fail = false;
+
+                if (!dry_run)
+                        replace_filter_string(call->filter, filter_string);
         }
-        mutex_unlock(&event_mutex);
-out:
-        return err;
+
+        if (fail) {
+                parse_error(ps, FILT_ERR_BAD_SUBSYS_FILTER, 0);
+                return err;
+        }
+        return 0;
 }
 
 static void parse_init(struct filter_parse_state *ps,
@@ -1004,12 +1092,14 @@ static int check_preds(struct filter_parse_state *ps)
 static int replace_preds(struct event_subsystem *system,
                          struct ftrace_event_call *call,
                          struct filter_parse_state *ps,
-                         char *filter_string)
+                         char *filter_string,
+                         bool dry_run)
 {
         char *operand1 = NULL, *operand2 = NULL;
         struct filter_pred *pred;
         struct postfix_elt *elt;
         int err;
+        int n_preds = 0;
 
         err = check_preds(ps);
         if (err)
@@ -1028,19 +1118,14 @@ static int replace_preds(struct event_subsystem *system,
                         continue;
                 }
 
+                if (n_preds++ == MAX_FILTER_PRED) {
+                        parse_error(ps, FILT_ERR_TOO_MANY_PREDS, 0);
+                        return -ENOSPC;
+                }
+
                 if (elt->op == OP_AND || elt->op == OP_OR) {
                         pred = create_logical_pred(elt->op);
-                        if (call) {
-                                err = filter_add_pred(ps, call, pred);
-                                filter_free_pred(pred);
-                        } else
-                                err = filter_add_subsystem_pred(ps, system,
-                                                        pred, filter_string);
-                        if (err)
-                                return err;
-
-                        operand1 = operand2 = NULL;
-                        continue;
+                        goto add_pred;
                 }
 
                 if (!operand1 || !operand2) {
@@ -1049,12 +1134,15 @@ static int replace_preds(struct event_subsystem *system,
                 }
 
                 pred = create_pred(elt->op, operand1, operand2);
-                if (call) {
-                        err = filter_add_pred(ps, call, pred);
-                        filter_free_pred(pred);
-                } else
+add_pred:
+                if (!pred)
+                        return -ENOMEM;
+                if (call)
+                        err = filter_add_pred(ps, call, pred, false);
+                else
                         err = filter_add_subsystem_pred(ps, system, pred,
-                                                        filter_string);
+                                                filter_string, dry_run);
+                filter_free_pred(pred);
                 if (err)
                         return err;
 
@@ -1070,12 +1158,16 @@ int apply_event_filter(struct ftrace_event_call *call, char *filter_string)
 
         struct filter_parse_state *ps;
 
-        mutex_lock(&filter_mutex);
+        mutex_lock(&event_mutex);
+
+        err = init_preds(call);
+        if (err)
+                goto out_unlock;
 
         if (!strcmp(strstrip(filter_string), "0")) {
                 filter_disable_preds(call);
                 remove_filter_string(call->filter);
-                mutex_unlock(&filter_mutex);
+                mutex_unlock(&event_mutex);
                 return 0;
         }
 
@@ -1094,7 +1186,7 @@ int apply_event_filter(struct ftrace_event_call *call, char *filter_string)
                 goto out;
         }
 
-        err = replace_preds(NULL, call, ps, filter_string);
+        err = replace_preds(NULL, call, ps, filter_string, false);
         if (err)
                 append_filter_err(ps, call->filter);
 
@@ -1103,7 +1195,7 @@ out:
         postfix_clear(ps);
         kfree(ps);
 out_unlock:
-        mutex_unlock(&filter_mutex);
+        mutex_unlock(&event_mutex);
 
         return err;
 }
@@ -1115,12 +1207,16 @@ int apply_subsystem_event_filter(struct event_subsystem *system,
 
         struct filter_parse_state *ps;
 
-        mutex_lock(&filter_mutex);
+        mutex_lock(&event_mutex);
+
+        err = init_subsystem_preds(system);
+        if (err)
+                goto out_unlock;
 
         if (!strcmp(strstrip(filter_string), "0")) {
-                filter_free_subsystem_preds(system);
+                filter_free_subsystem_preds(system, FILTER_DISABLE_ALL);
                 remove_filter_string(system->filter);
-                mutex_unlock(&filter_mutex);
+                mutex_unlock(&event_mutex);
                 return 0;
         }
 
@@ -1129,7 +1225,6 @@ int apply_subsystem_event_filter(struct event_subsystem *system,
         if (!ps)
                 goto out_unlock;
 
-        filter_free_subsystem_preds(system);
         replace_filter_string(system->filter, filter_string);
 
         parse_init(ps, filter_ops, filter_string);
@@ -1139,16 +1234,30 @@ int apply_subsystem_event_filter(struct event_subsystem *system,
                 goto out;
         }
 
-        err = replace_preds(system, NULL, ps, filter_string);
-        if (err)
+        filter_free_subsystem_preds(system, FILTER_INIT_NO_RESET);
+
+        /* try to see the filter can be applied to which events */
+        err = replace_preds(system, NULL, ps, filter_string, true);
+        if (err) {
                 append_filter_err(ps, system->filter);
+                goto out;
+        }
+
+        filter_free_subsystem_preds(system, FILTER_SKIP_NO_RESET);
+
+        /* really apply the filter to the events */
+        err = replace_preds(system, NULL, ps, filter_string, false);
+        if (err) {
+                append_filter_err(ps, system->filter);
+                filter_free_subsystem_preds(system, 2);
+        }
 
 out:
         filter_opstack_clear(ps);
         postfix_clear(ps);
         kfree(ps);
 out_unlock:
-        mutex_unlock(&filter_mutex);
+        mutex_unlock(&event_mutex);
 
         return err;
 }
diff --git a/kernel/trace/trace_export.c b/kernel/trace/trace_export.c
index d06cf898dc86..9753fcc61bc5 100644
--- a/kernel/trace/trace_export.c
+++ b/kernel/trace/trace_export.c
@@ -15,116 +15,209 @@
 
 #include "trace_output.h"
 
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM    ftrace
 
-#undef TRACE_STRUCT
-#define TRACE_STRUCT(args...) args
+/* not needed for this file */
+#undef __field_struct
+#define __field_struct(type, item)
 
-extern void __bad_type_size(void);
+#undef __field
+#define __field(type, item)                             type item;
 
-#undef TRACE_FIELD
-#define TRACE_FIELD(type, item, assign)                                 \
-        if (sizeof(type) != sizeof(field.item))                         \
-                __bad_type_size();                                      \
+#undef __field_desc
+#define __field_desc(type, container, item)             type item;
+
+#undef __array
+#define __array(type, item, size)                       type item[size];
+
+#undef __array_desc
+#define __array_desc(type, container, item, size)       type item[size];
+
+#undef __dynamic_array
+#define __dynamic_array(type, item)                     type item[];
+
+#undef F_STRUCT
+#define F_STRUCT(args...)                               args
+
+#undef F_printk
+#define F_printk(fmt, args...) fmt, args
+
+#undef FTRACE_ENTRY
+#define FTRACE_ENTRY(name, struct_name, id, tstruct, print)     \
+struct ____ftrace_##name {                                      \
+        tstruct                                                 \
+};                                                              \
+static void __used ____ftrace_check_##name(void)                \
+{                                                               \
+        struct ____ftrace_##name *__entry = NULL;               \
+                                                                \
+        /* force cmpile-time check on F_printk() */             \
+        printk(print);                                          \
+}
+
+#undef FTRACE_ENTRY_DUP
+#define FTRACE_ENTRY_DUP(name, struct_name, id, tstruct, print) \
+        FTRACE_ENTRY(name, struct_name, id, PARAMS(tstruct), PARAMS(print))
+
+#include "trace_entries.h"
+
+
+#undef __field
+#define __field(type, item)                                             \
         ret = trace_seq_printf(s, "\tfield:" #type " " #item ";\t"      \
-                               "offset:%u;\tsize:%u;\n",                \
-                               (unsigned int)offsetof(typeof(field), item), \
-                               (unsigned int)sizeof(field.item));       \
+                               "offset:%zu;\tsize:%zu;\n",              \
+                               offsetof(typeof(field), item),           \
+                               sizeof(field.item));                     \
         if (!ret)                                                       \
                 return 0;
 
+#undef __field_desc
+#define __field_desc(type, container, item)                             \
+        ret = trace_seq_printf(s, "\tfield:" #type " " #item ";\t"      \
+                               "offset:%zu;\tsize:%zu;\n",              \
+                               offsetof(typeof(field), container.item), \
+                               sizeof(field.container.item));           \
+        if (!ret)                                                       \
+                return 0;
+
+#undef __array
+#define __array(type, item, len)                                        \
+        ret = trace_seq_printf(s, "\tfield:" #type " " #item "[" #len "];\t" \
+                               "offset:%zu;\tsize:%zu;\n",              \
+                               offsetof(typeof(field), item),   \
+                               sizeof(field.item));             \
+        if (!ret)                                                       \
+                return 0;
 
-#undef TRACE_FIELD_SPECIAL
-#define TRACE_FIELD_SPECIAL(type_item, item, len, cmd)                  \
-        ret = trace_seq_printf(s, "\tfield special:" #type_item ";\t"   \
-                               "offset:%u;\tsize:%u;\n",                \
-                               (unsigned int)offsetof(typeof(field), item), \
-                               (unsigned int)sizeof(field.item));       \
+#undef __array_desc
+#define __array_desc(type, container, item, len)                        \
+        ret = trace_seq_printf(s, "\tfield:" #type " " #item "[" #len "];\t" \
+                               "offset:%zu;\tsize:%zu;\n",              \
+                               offsetof(typeof(field), container.item), \
+                               sizeof(field.container.item));           \
         if (!ret)                                                       \
                 return 0;
 
-#undef TRACE_FIELD_ZERO_CHAR
-#define TRACE_FIELD_ZERO_CHAR(item)                                     \
-        ret = trace_seq_printf(s, "\tfield:char " #item ";\t"           \
-                               "offset:%u;\tsize:0;\n",                 \
-                               (unsigned int)offsetof(typeof(field), item)); \
+#undef __dynamic_array
+#define __dynamic_array(type, item)                                     \
+        ret = trace_seq_printf(s, "\tfield:" #type " " #item ";\t"      \
+                               "offset:%zu;\tsize:0;\n",                \
+                               offsetof(typeof(field), item));          \
         if (!ret)                                                       \
                 return 0;
 
-#undef TRACE_FIELD_SIGN
-#define TRACE_FIELD_SIGN(type, item, assign, is_signed) \
-        TRACE_FIELD(type, item, assign)
+#undef F_printk
+#define F_printk(fmt, args...) "%s, %s\n", #fmt, __stringify(args)
 
-#undef TP_RAW_FMT
-#define TP_RAW_FMT(args...) args
+#undef __entry
+#define __entry REC
 
-#undef TRACE_EVENT_FORMAT
-#define TRACE_EVENT_FORMAT(call, proto, args, fmt, tstruct, tpfmt)      \
+#undef FTRACE_ENTRY
+#define FTRACE_ENTRY(name, struct_name, id, tstruct, print)             \
 static int                                                              \
-ftrace_format_##call(struct trace_seq *s)                               \
+ftrace_format_##name(struct ftrace_event_call *unused,                  \
+                     struct trace_seq *s)                               \
 {                                                                       \
-        struct args field;                                              \
-        int ret;                                                        \
+        struct struct_name field __attribute__((unused));               \
+        int ret = 0;                                                    \
                                                                         \
         tstruct;                                                        \
                                                                         \
-        trace_seq_printf(s, "\nprint fmt: \"%s\"\n", tpfmt);            \
+        trace_seq_printf(s, "\nprint fmt: " print);                     \
                                                                         \
         return ret;                                                     \
 }
 
-#undef TRACE_EVENT_FORMAT_NOFILTER
-#define TRACE_EVENT_FORMAT_NOFILTER(call, proto, args, fmt, tstruct,    \
-                                    tpfmt)                              \
-static int                                                              \
-ftrace_format_##call(struct trace_seq *s)                               \
+#include "trace_entries.h"
+
+
+#undef __field
+#define __field(type, item)                                             \
+        ret = trace_define_field(event_call, #type, #item,              \
+                                 offsetof(typeof(field), item),         \
+                                 sizeof(field.item),                    \
+                                 is_signed_type(type), FILTER_OTHER);   \
+        if (ret)                                                        \
+                return ret;
+
+#undef __field_desc
+#define __field_desc(type, container, item)     \
+        ret = trace_define_field(event_call, #type, #item,              \
+                                 offsetof(typeof(field),                \
+                                          container.item),              \
+                                 sizeof(field.container.item),          \
+                                 is_signed_type(type), FILTER_OTHER);   \
+        if (ret)                                                        \
+                return ret;
+
+#undef __array
+#define __array(type, item, len)                                        \
+        BUILD_BUG_ON(len > MAX_FILTER_STR_VAL);                         \
+        ret = trace_define_field(event_call, #type "[" #len "]", #item, \
+                                 offsetof(typeof(field), item),         \
+                                 sizeof(field.item), 0, FILTER_OTHER);  \
+        if (ret)                                                        \
+                return ret;
+
+#undef __array_desc
+#define __array_desc(type, container, item, len)                        \
+        BUILD_BUG_ON(len > MAX_FILTER_STR_VAL);                         \
+        ret = trace_define_field(event_call, #type "[" #len "]", #item, \
+                                 offsetof(typeof(field),                \
+                                          container.item),              \
+                                 sizeof(field.container.item), 0,       \
+                                 FILTER_OTHER);                         \
+        if (ret)                                                        \
+                return ret;
+
+#undef __dynamic_array
+#define __dynamic_array(type, item)
+
+#undef FTRACE_ENTRY
+#define FTRACE_ENTRY(name, struct_name, id, tstruct, print)             \
+int                                                                     \
+ftrace_define_fields_##name(struct ftrace_event_call *event_call)       \
 {                                                                       \
-        struct args field;                                              \
+        struct struct_name field;                                       \
         int ret;                                                        \
                                                                         \
-        tstruct;                                                        \
+        ret = trace_define_common_fields(event_call);                   \
+        if (ret)                                                        \
+                return ret;                                             \
                                                                         \
-        trace_seq_printf(s, "\nprint fmt: \"%s\"\n", tpfmt);            \
+        tstruct;                                                        \
                                                                         \
         return ret;                                                     \
 }
 
-#include "trace_event_types.h"
-
-#undef TRACE_ZERO_CHAR
-#define TRACE_ZERO_CHAR(arg)
+#include "trace_entries.h"
 
-#undef TRACE_FIELD
-#define TRACE_FIELD(type, item, assign)\
-        entry->item = assign;
 
-#undef TRACE_FIELD
-#define TRACE_FIELD(type, item, assign)\
-        entry->item = assign;
+#undef __field
+#define __field(type, item)
 
-#undef TRACE_FIELD_SIGN
-#define TRACE_FIELD_SIGN(type, item, assign, is_signed) \
-        TRACE_FIELD(type, item, assign)
+#undef __field_desc
+#define __field_desc(type, container, item)
 
-#undef TP_CMD
-#define TP_CMD(cmd...)  cmd
+#undef __array
+#define __array(type, item, len)
 
-#undef TRACE_ENTRY
-#define TRACE_ENTRY     entry
+#undef __array_desc
+#define __array_desc(type, container, item, len)
 
-#undef TRACE_FIELD_SPECIAL
-#define TRACE_FIELD_SPECIAL(type_item, item, len, cmd)  \
-        cmd;
+#undef __dynamic_array
+#define __dynamic_array(type, item)
 
-#undef TRACE_EVENT_FORMAT
-#define TRACE_EVENT_FORMAT(call, proto, args, fmt, tstruct, tpfmt)      \
-int ftrace_define_fields_##call(void);                                  \
+#undef FTRACE_ENTRY
+#define FTRACE_ENTRY(call, struct_name, type, tstruct, print)           \
 static int ftrace_raw_init_event_##call(void);                          \
                                                                         \
 struct ftrace_event_call __used                                         \
 __attribute__((__aligned__(4)))                                         \
 __attribute__((section("_ftrace_events"))) event_##call = {             \
         .name                   = #call,                                \
-        .id                     = proto,                                \
+        .id                     = type,                                 \
         .system                 = __stringify(TRACE_SYSTEM),            \
         .raw_init               = ftrace_raw_init_event_##call,         \
         .show_format            = ftrace_format_##call,                 \
@@ -133,74 +226,7 @@ __attribute__((section("_ftrace_events"))) event_##call = {		\
 static int ftrace_raw_init_event_##call(void)                           \
 {                                                                       \
         INIT_LIST_HEAD(&event_##call.fields);                           \
-        init_preds(&event_##call);                                      \
         return 0;                                                       \
 }                                                                       \
 
-#undef TRACE_EVENT_FORMAT_NOFILTER
-#define TRACE_EVENT_FORMAT_NOFILTER(call, proto, args, fmt, tstruct,    \
-                                    tpfmt)                              \
-                                                                        \
-struct ftrace_event_call __used                                         \
-__attribute__((__aligned__(4)))                                         \
-__attribute__((section("_ftrace_events"))) event_##call = {             \
-        .name                   = #call,                                \
-        .id                     = proto,                                \
-        .system                 = __stringify(TRACE_SYSTEM),            \
-        .show_format            = ftrace_format_##call,                 \
-};
-
-#include "trace_event_types.h"
-
-#undef TRACE_FIELD
-#define TRACE_FIELD(type, item, assign)                                 \
-        ret = trace_define_field(event_call, #type, #item,              \
-                                 offsetof(typeof(field), item),         \
-                                 sizeof(field.item), is_signed_type(type));     \
-        if (ret)                                                        \
-                return ret;
-
-#undef TRACE_FIELD_SPECIAL
-#define TRACE_FIELD_SPECIAL(type, item, len, cmd)                       \
-        ret = trace_define_field(event_call, #type "[" #len "]", #item, \
-                                 offsetof(typeof(field), item),         \
-                                 sizeof(field.item), 0);                \
-        if (ret)                                                        \
-                return ret;
-
-#undef TRACE_FIELD_SIGN
-#define TRACE_FIELD_SIGN(type, item, assign, is_signed)                 \
-        ret = trace_define_field(event_call, #type, #item,              \
-                                 offsetof(typeof(field), item),         \
-                                 sizeof(field.item), is_signed);        \
-        if (ret)                                                        \
-                return ret;
-
-#undef TRACE_FIELD_ZERO_CHAR
-#define TRACE_FIELD_ZERO_CHAR(item)
-
-#undef TRACE_EVENT_FORMAT
-#define TRACE_EVENT_FORMAT(call, proto, args, fmt, tstruct, tpfmt)      \
-int                                                                     \
-ftrace_define_fields_##call(void)                                       \
-{                                                                       \
-        struct ftrace_event_call *event_call = &event_##call;           \
-        struct args field;                                              \
-        int ret;                                                        \
-                                                                        \
-        __common_field(unsigned char, type, 0);                         \
-        __common_field(unsigned char, flags, 0);                        \
-        __common_field(unsigned char, preempt_count, 0);                \
-        __common_field(int, pid, 1);                                    \
-        __common_field(int, tgid, 1);                                   \
-                                                                        \
-        tstruct;                                                        \
-                                                                        \
-        return ret;                                                     \
-}
-
-#undef TRACE_EVENT_FORMAT_NOFILTER
-#define TRACE_EVENT_FORMAT_NOFILTER(call, proto, args, fmt, tstruct,    \
-                                    tpfmt)
-
-#include "trace_event_types.h"
+#include "trace_entries.h"
diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c
index c9a0b7df44ff..b3f3776b0cd6 100644
--- a/kernel/trace/trace_functions.c
+++ b/kernel/trace/trace_functions.c
@@ -193,9 +193,11 @@ static void tracing_start_function_trace(void)
 static void tracing_stop_function_trace(void)
 {
         ftrace_function_enabled = 0;
-        /* OK if they are not registered */
-        unregister_ftrace_function(&trace_stack_ops);
-        unregister_ftrace_function(&trace_ops);
+
+        if (func_flags.val & TRACE_FUNC_OPT_STACK)
+                unregister_ftrace_function(&trace_stack_ops);
+        else
+                unregister_ftrace_function(&trace_ops);
 }
 
 static int func_set_flag(u32 old_flags, u32 bit, int set)
@@ -286,11 +288,9 @@ static int
 ftrace_trace_onoff_print(struct seq_file *m, unsigned long ip,
                          struct ftrace_probe_ops *ops, void *data)
 {
-        char str[KSYM_SYMBOL_LEN];
         long count = (long)data;
 
-        kallsyms_lookup(ip, NULL, NULL, NULL, str);
-        seq_printf(m, "%s:", str);
+        seq_printf(m, "%ps:", (void *)ip);
 
         if (ops == &traceon_probe_ops)
                 seq_printf(m, "traceon");
@@ -300,8 +300,7 @@ ftrace_trace_onoff_print(struct seq_file *m, unsigned long ip,
         if (count == -1)
                 seq_printf(m, ":unlimited\n");
         else
-                seq_printf(m, ":count=%ld", count);
-        seq_putc(m, '\n');
+                seq_printf(m, ":count=%ld\n", count);
 
         return 0;
 }
@@ -362,7 +361,7 @@ ftrace_trace_onoff_callback(char *glob, char *cmd, char *param, int enable)
  out_reg:
         ret = register_ftrace_function_probe(glob, ops, count);
 
-        return ret;
+        return ret < 0 ? ret : 0;
 }
 
 static struct ftrace_func_command ftrace_traceon_cmd = {
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c
index 8b592418d8b2..45e6c01b2e4d 100644
--- a/kernel/trace/trace_functions_graph.c
+++ b/kernel/trace/trace_functions_graph.c
@@ -52,12 +52,13 @@ static struct tracer_flags tracer_flags = {
         .opts = trace_opts
 };
 
-/* pid on the last trace processed */
+static struct trace_array *graph_array;
 
 
 /* Add a function return address to the trace stack on thread info.*/
 int
-ftrace_push_return_trace(unsigned long ret, unsigned long func, int *depth)
+ftrace_push_return_trace(unsigned long ret, unsigned long func, int *depth,
+                         unsigned long frame_pointer)
 {
         unsigned long long calltime;
         int index;
@@ -85,6 +86,7 @@ ftrace_push_return_trace(unsigned long ret, unsigned long func, int *depth)
         current->ret_stack[index].func = func;
         current->ret_stack[index].calltime = calltime;
         current->ret_stack[index].subtime = 0;
+        current->ret_stack[index].fp = frame_pointer;
         *depth = index;
 
         return 0;
@@ -92,7 +94,8 @@ ftrace_push_return_trace(unsigned long ret, unsigned long func, int *depth)
 
 /* Retrieve a function return address to the trace stack on thread info.*/
 static void
-ftrace_pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret)
+ftrace_pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret,
+                        unsigned long frame_pointer)
 {
         int index;
 
@@ -106,6 +109,31 @@ ftrace_pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret)
                 return;
         }
 
+#ifdef CONFIG_HAVE_FUNCTION_GRAPH_FP_TEST
+        /*
+         * The arch may choose to record the frame pointer used
+         * and check it here to make sure that it is what we expect it
+         * to be. If gcc does not set the place holder of the return
+         * address in the frame pointer, and does a copy instead, then
+         * the function graph trace will fail. This test detects this
+         * case.
+         *
+         * Currently, x86_32 with optimize for size (-Os) makes the latest
+         * gcc do the above.
+         */
+        if (unlikely(current->ret_stack[index].fp != frame_pointer)) {
+                ftrace_graph_stop();
+                WARN(1, "Bad frame pointer: expected %lx, received %lx\n"
+                     "  from func %ps return to %lx\n",
+                     current->ret_stack[index].fp,
+                     frame_pointer,
+                     (void *)current->ret_stack[index].func,
+                     current->ret_stack[index].ret);
+                *ret = (unsigned long)panic;
+                return;
+        }
+#endif
+
         *ret = current->ret_stack[index].ret;
         trace->func = current->ret_stack[index].func;
         trace->calltime = current->ret_stack[index].calltime;
@@ -117,12 +145,12 @@ ftrace_pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret)
  * Send the trace to the ring-buffer.
  * @return the original return address.
  */
-unsigned long ftrace_return_to_handler(void)
+unsigned long ftrace_return_to_handler(unsigned long frame_pointer)
 {
         struct ftrace_graph_ret trace;
         unsigned long ret;
 
-        ftrace_pop_return_trace(&trace, &ret);
+        ftrace_pop_return_trace(&trace, &ret, frame_pointer);
         trace.rettime = trace_clock_local();
         ftrace_graph_return(&trace);
         barrier();
@@ -138,10 +166,123 @@ unsigned long ftrace_return_to_handler(void)
         return ret;
 }
 
+static int __trace_graph_entry(struct trace_array *tr,
+                                struct ftrace_graph_ent *trace,
+                                unsigned long flags,
+                                int pc)
+{
+        struct ftrace_event_call *call = &event_funcgraph_entry;
+        struct ring_buffer_event *event;
+        struct ring_buffer *buffer = tr->buffer;
+        struct ftrace_graph_ent_entry *entry;
+
+        if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
+                return 0;
+
+        event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_ENT,
+                                          sizeof(*entry), flags, pc);
+        if (!event)
+                return 0;
+        entry   = ring_buffer_event_data(event);
+        entry->graph_ent                        = *trace;
+        if (!filter_current_check_discard(buffer, call, entry, event))
+                ring_buffer_unlock_commit(buffer, event);
+
+        return 1;
+}
+
+int trace_graph_entry(struct ftrace_graph_ent *trace)
+{
+        struct trace_array *tr = graph_array;
+        struct trace_array_cpu *data;
+        unsigned long flags;
+        long disabled;
+        int ret;
+        int cpu;
+        int pc;
+
+        if (unlikely(!tr))
+                return 0;
+
+        if (!ftrace_trace_task(current))
+                return 0;
+
+        if (!ftrace_graph_addr(trace->func))
+                return 0;
+
+        local_irq_save(flags);
+        cpu = raw_smp_processor_id();
+        data = tr->data[cpu];
+        disabled = atomic_inc_return(&data->disabled);
+        if (likely(disabled == 1)) {
+                pc = preempt_count();
+                ret = __trace_graph_entry(tr, trace, flags, pc);
+        } else {
+                ret = 0;
+        }
+        /* Only do the atomic if it is not already set */
+        if (!test_tsk_trace_graph(current))
+                set_tsk_trace_graph(current);
+
+        atomic_dec(&data->disabled);
+        local_irq_restore(flags);
+
+        return ret;
+}
+
+static void __trace_graph_return(struct trace_array *tr,
+                                struct ftrace_graph_ret *trace,
+                                unsigned long flags,
+                                int pc)
+{
+        struct ftrace_event_call *call = &event_funcgraph_exit;
+        struct ring_buffer_event *event;
+        struct ring_buffer *buffer = tr->buffer;
+        struct ftrace_graph_ret_entry *entry;
+
+        if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
+                return;
+
+        event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_RET,
+                                          sizeof(*entry), flags, pc);
+        if (!event)
+                return;
+        entry   = ring_buffer_event_data(event);
+        entry->ret                              = *trace;
+        if (!filter_current_check_discard(buffer, call, entry, event))
+                ring_buffer_unlock_commit(buffer, event);
+}
+
+void trace_graph_return(struct ftrace_graph_ret *trace)
+{
+        struct trace_array *tr = graph_array;
+        struct trace_array_cpu *data;
+        unsigned long flags;
+        long disabled;
+        int cpu;
+        int pc;
+
+        local_irq_save(flags);
+        cpu = raw_smp_processor_id();
+        data = tr->data[cpu];
+        disabled = atomic_inc_return(&data->disabled);
+        if (likely(disabled == 1)) {
+                pc = preempt_count();
+                __trace_graph_return(tr, trace, flags, pc);
+        }
+        if (!trace->depth)
+                clear_tsk_trace_graph(current);
+        atomic_dec(&data->disabled);
+        local_irq_restore(flags);
+}
+
 static int graph_trace_init(struct trace_array *tr)
 {
-        int ret = register_ftrace_graph(&trace_graph_return,
-                                        &trace_graph_entry);
+        int ret;
+
+        graph_array = tr;
+        ret = register_ftrace_graph(&trace_graph_return,
+                                    &trace_graph_entry);
         if (ret)
                 return ret;
         tracing_start_cmdline_record();
@@ -149,49 +290,30 @@ static int graph_trace_init(struct trace_array *tr)
         return 0;
 }
 
+void set_graph_array(struct trace_array *tr)
+{
+        graph_array = tr;
+}
+
 static void graph_trace_reset(struct trace_array *tr)
 {
         tracing_stop_cmdline_record();
         unregister_ftrace_graph();
 }
 
-static inline int log10_cpu(int nb)
-{
-        if (nb / 100)
-                return 3;
-        if (nb / 10)
-                return 2;
-        return 1;
-}
+static int max_bytes_for_cpu;
 
 static enum print_line_t
 print_graph_cpu(struct trace_seq *s, int cpu)
 {
-        int i;
         int ret;
-        int log10_this = log10_cpu(cpu);
-        int log10_all = log10_cpu(cpumask_weight(cpu_online_mask));
-
 
         /*
          * Start with a space character - to make it stand out
          * to the right a bit when trace output is pasted into
          * email:
          */
-        ret = trace_seq_printf(s, " ");
-
-        /*
-         * Tricky - we space the CPU field according to the max
-         * number of online CPUs. On a 2-cpu system it would take
-         * a maximum of 1 digit - on a 128 cpu system it would
-         * take up to 3 digits:
-         */
-        for (i = 0; i < log10_all - log10_this; i++) {
-                ret = trace_seq_printf(s, " ");
-                if (!ret)
-                        return TRACE_TYPE_PARTIAL_LINE;
-        }
-        ret = trace_seq_printf(s, "%d) ", cpu);
+        ret = trace_seq_printf(s, " %*d) ", max_bytes_for_cpu, cpu);
         if (!ret)
                 return TRACE_TYPE_PARTIAL_LINE;
 
@@ -242,6 +364,15 @@ print_graph_proc(struct trace_seq *s, pid_t pid)
 }
 
 
+static enum print_line_t
+print_graph_lat_fmt(struct trace_seq *s, struct trace_entry *entry)
+{
+        if (!trace_seq_putc(s, ' '))
+                return 0;
+
+        return trace_print_lat_fmt(s, entry);
+}
+
 /* If the pid changed since the last trace, output this event */
 static enum print_line_t
 verif_pid(struct trace_seq *s, pid_t pid, int cpu, struct fgraph_data *data)
@@ -399,6 +530,7 @@ print_graph_irq(struct trace_iterator *iter, unsigned long addr,
                 if (ret == TRACE_TYPE_PARTIAL_LINE)
                         return TRACE_TYPE_PARTIAL_LINE;
         }
+
         /* Proc */
         if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC) {
                 ret = print_graph_proc(s, pid);
@@ -537,11 +669,7 @@ print_graph_entry_leaf(struct trace_iterator *iter,
                         return TRACE_TYPE_PARTIAL_LINE;
         }
 
-        ret = seq_print_ip_sym(s, call->func, 0);
-        if (!ret)
-                return TRACE_TYPE_PARTIAL_LINE;
-
-        ret = trace_seq_printf(s, "();\n");
+        ret = trace_seq_printf(s, "%ps();\n", (void *)call->func);
         if (!ret)
                 return TRACE_TYPE_PARTIAL_LINE;
 
@@ -584,11 +712,7 @@ print_graph_entry_nested(struct trace_iterator *iter,
                         return TRACE_TYPE_PARTIAL_LINE;
         }
 
-        ret = seq_print_ip_sym(s, call->func, 0);
-        if (!ret)
-                return TRACE_TYPE_PARTIAL_LINE;
-
-        ret = trace_seq_printf(s, "() {\n");
+        ret = trace_seq_printf(s, "%ps() {\n", (void *)call->func);
         if (!ret)
                 return TRACE_TYPE_PARTIAL_LINE;
 
@@ -644,6 +768,13 @@ print_graph_prologue(struct trace_iterator *iter, struct trace_seq *s,
                         return TRACE_TYPE_PARTIAL_LINE;
         }
 
+        /* Latency format */
+        if (trace_flags & TRACE_ITER_LATENCY_FMT) {
+                ret = print_graph_lat_fmt(s, ent);
+                if (ret == TRACE_TYPE_PARTIAL_LINE)
+                        return TRACE_TYPE_PARTIAL_LINE;
+        }
+
         return 0;
 }
 
@@ -815,9 +946,16 @@ print_graph_function(struct trace_iterator *iter)
 
         switch (entry->type) {
         case TRACE_GRAPH_ENT: {
-                struct ftrace_graph_ent_entry *field;
+                /*
+                 * print_graph_entry() may consume the current event,
+                 * thus @field may become invalid, so we need to save it.
+                 * sizeof(struct ftrace_graph_ent_entry) is very small,
+                 * it can be safely saved at the stack.
+                 */
+                struct ftrace_graph_ent_entry *field, saved;
                 trace_assign_type(field, entry);
-                return print_graph_entry(field, s, iter);
+                saved = *field;
+                return print_graph_entry(&saved, s, iter);
         }
         case TRACE_GRAPH_RET: {
                 struct ftrace_graph_ret_entry *field;
@@ -831,28 +969,59 @@ print_graph_function(struct trace_iterator *iter)
         return TRACE_TYPE_HANDLED;
 }
 
+static void print_lat_header(struct seq_file *s)
+{
+        static const char spaces[] = "                " /* 16 spaces */
+                "    "                                  /* 4 spaces */
+                "                 ";                    /* 17 spaces */
+        int size = 0;
+
+        if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME)
+                size += 16;
+        if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU)
+                size += 4;
+        if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC)
+                size += 17;
+
+        seq_printf(s, "#%.*s  _-----=> irqs-off        \n", size, spaces);
+        seq_printf(s, "#%.*s / _----=> need-resched    \n", size, spaces);
+        seq_printf(s, "#%.*s| / _---=> hardirq/softirq \n", size, spaces);
+        seq_printf(s, "#%.*s|| / _--=> preempt-depth   \n", size, spaces);
+        seq_printf(s, "#%.*s||| / _-=> lock-depth      \n", size, spaces);
+        seq_printf(s, "#%.*s|||| /                     \n", size, spaces);
+}
+
 static void print_graph_headers(struct seq_file *s)
 {
+        int lat = trace_flags & TRACE_ITER_LATENCY_FMT;
+
+        if (lat)
+                print_lat_header(s);
+
         /* 1st line */
-        seq_printf(s, "# ");
+        seq_printf(s, "#");
         if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME)
                 seq_printf(s, "     TIME       ");
         if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU)
-                seq_printf(s, "CPU");
+                seq_printf(s, " CPU");
         if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC)
-                seq_printf(s, "  TASK/PID      ");
+                seq_printf(s, "  TASK/PID       ");
+        if (lat)
+                seq_printf(s, "|||||");
         if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION)
                 seq_printf(s, "  DURATION   ");
         seq_printf(s, "               FUNCTION CALLS\n");
 
         /* 2nd line */
-        seq_printf(s, "# ");
+        seq_printf(s, "#");
         if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME)
                 seq_printf(s, "      |         ");
         if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU)
-                seq_printf(s, "|  ");
+                seq_printf(s, " |  ");
         if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC)
-                seq_printf(s, "  |    |        ");
+                seq_printf(s, "   |    |        ");
+        if (lat)
+                seq_printf(s, "|||||");
         if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION)
                 seq_printf(s, "   |   |      ");
         seq_printf(s, "               |   |   |   |\n");
@@ -899,6 +1068,8 @@ static struct tracer graph_trace __read_mostly = {
 
 static __init int init_graph_trace(void)
 {
+        max_bytes_for_cpu = snprintf(NULL, 0, "%d", nr_cpu_ids - 1);
+
         return register_tracer(&graph_trace);
 }
 
diff --git a/kernel/trace/trace_irqsoff.c b/kernel/trace/trace_irqsoff.c
index b923d13e2fad..3aa7eaa2114c 100644
--- a/kernel/trace/trace_irqsoff.c
+++ b/kernel/trace/trace_irqsoff.c
@@ -129,15 +129,10 @@ check_critical_timing(struct trace_array *tr,
                       unsigned long parent_ip,
                       int cpu)
 {
-        unsigned long latency, t0, t1;
         cycle_t T0, T1, delta;
         unsigned long flags;
         int pc;
 
-        /*
-         * usecs conversion is slow so we try to delay the conversion
-         * as long as possible:
-         */
         T0 = data->preempt_timestamp;
         T1 = ftrace_now(cpu);
         delta = T1-T0;
@@ -157,18 +152,15 @@ check_critical_timing(struct trace_array *tr,
 
         trace_function(tr, CALLER_ADDR0, parent_ip, flags, pc);
 
-        latency = nsecs_to_usecs(delta);
-
         if (data->critical_sequence != max_sequence)
                 goto out_unlock;
 
-        tracing_max_latency = delta;
-        t0 = nsecs_to_usecs(T0);
-        t1 = nsecs_to_usecs(T1);
-
         data->critical_end = parent_ip;
 
-        update_max_tr_single(tr, current, cpu);
+        if (likely(!is_tracing_stopped())) {
+                tracing_max_latency = delta;
+                update_max_tr_single(tr, current, cpu);
+        }
 
         max_sequence++;
 
@@ -178,7 +170,6 @@ out_unlock:
 out:
         data->critical_sequence = max_sequence;
         data->preempt_timestamp = ftrace_now(cpu);
-        tracing_reset(tr, cpu);
         trace_function(tr, CALLER_ADDR0, parent_ip, flags, pc);
 }
 
@@ -208,7 +199,6 @@ start_critical_timing(unsigned long ip, unsigned long parent_ip)
         data->critical_sequence = max_sequence;
         data->preempt_timestamp = ftrace_now(cpu);
         data->critical_start = parent_ip ? : ip;
-        tracing_reset(tr, cpu);
 
         local_save_flags(flags);
 
@@ -379,6 +369,7 @@ static void __irqsoff_tracer_init(struct trace_array *tr)
         irqsoff_trace = tr;
         /* make sure that the tracer is visible */
         smp_wmb();
+        tracing_reset_online_cpus(tr);
         start_irqsoff_tracer(tr);
 }
 
diff --git a/kernel/trace/trace_mmiotrace.c b/kernel/trace/trace_mmiotrace.c
index d53b45ed0806..0acd834659ed 100644
--- a/kernel/trace/trace_mmiotrace.c
+++ b/kernel/trace/trace_mmiotrace.c
@@ -307,11 +307,13 @@ static void __trace_mmiotrace_rw(struct trace_array *tr,
                                 struct trace_array_cpu *data,
                                 struct mmiotrace_rw *rw)
 {
+        struct ftrace_event_call *call = &event_mmiotrace_rw;
+        struct ring_buffer *buffer = tr->buffer;
         struct ring_buffer_event *event;
         struct trace_mmiotrace_rw *entry;
         int pc = preempt_count();
 
-        event = trace_buffer_lock_reserve(tr, TRACE_MMIO_RW,
+        event = trace_buffer_lock_reserve(buffer, TRACE_MMIO_RW,
                                           sizeof(*entry), 0, pc);
         if (!event) {
                 atomic_inc(&dropped_count);
@@ -319,7 +321,9 @@ static void __trace_mmiotrace_rw(struct trace_array *tr,
         }
         entry   = ring_buffer_event_data(event);
         entry->rw                       = *rw;
-        trace_buffer_unlock_commit(tr, event, 0, pc);
+
+        if (!filter_check_discard(call, entry, buffer, event))
+                trace_buffer_unlock_commit(buffer, event, 0, pc);
 }
 
 void mmio_trace_rw(struct mmiotrace_rw *rw)
@@ -333,11 +337,13 @@ static void __trace_mmiotrace_map(struct trace_array *tr,
                                 struct trace_array_cpu *data,
                                 struct mmiotrace_map *map)
 {
+        struct ftrace_event_call *call = &event_mmiotrace_map;
+        struct ring_buffer *buffer = tr->buffer;
         struct ring_buffer_event *event;
         struct trace_mmiotrace_map *entry;
         int pc = preempt_count();
 
-        event = trace_buffer_lock_reserve(tr, TRACE_MMIO_MAP,
+        event = trace_buffer_lock_reserve(buffer, TRACE_MMIO_MAP,
                                           sizeof(*entry), 0, pc);
         if (!event) {
                 atomic_inc(&dropped_count);
@@ -345,7 +351,9 @@ static void __trace_mmiotrace_map(struct trace_array *tr,
         }
         entry   = ring_buffer_event_data(event);
         entry->map                      = *map;
-        trace_buffer_unlock_commit(tr, event, 0, pc);
+
+        if (!filter_check_discard(call, entry, buffer, event))
+                trace_buffer_unlock_commit(buffer, event, 0, pc);
 }
 
 void mmio_trace_mapping(struct mmiotrace_map *map)
diff --git a/kernel/trace/trace_output.c b/kernel/trace/trace_output.c
index 7938f3ae93e3..f572f44c6e1e 100644
--- a/kernel/trace/trace_output.c
+++ b/kernel/trace/trace_output.c
@@ -27,8 +27,7 @@ void trace_print_seq(struct seq_file *m, struct trace_seq *s)
 {
         int len = s->len >= PAGE_SIZE ? PAGE_SIZE - 1 : s->len;
 
-        s->buffer[len] = 0;
-        seq_puts(m, s->buffer);
+        seq_write(m, s->buffer, len);
 
         trace_seq_init(s);
 }
@@ -408,7 +407,7 @@ seq_print_userip_objs(const struct userstack_entry *entry, struct trace_seq *s,
                  * since individual threads might have already quit!
                  */
                 rcu_read_lock();
-                task = find_task_by_vpid(entry->ent.tgid);
+                task = find_task_by_vpid(entry->tgid);
                 if (task)
                         mm = get_task_mm(task);
                 rcu_read_unlock();
@@ -461,18 +460,23 @@ seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags)
         return ret;
 }
 
-static int
-lat_print_generic(struct trace_seq *s, struct trace_entry *entry, int cpu)
+/**
+ * trace_print_lat_fmt - print the irq, preempt and lockdep fields
+ * @s: trace seq struct to write to
+ * @entry: The trace entry field from the ring buffer
+ *
+ * Prints the generic fields of irqs off, in hard or softirq, preempt
+ * count and lock depth.
+ */
+int trace_print_lat_fmt(struct trace_seq *s, struct trace_entry *entry)
 {
         int hardirq, softirq;
-        char comm[TASK_COMM_LEN];
+        int ret;
 
-        trace_find_cmdline(entry->pid, comm);
         hardirq = entry->flags & TRACE_FLAG_HARDIRQ;
         softirq = entry->flags & TRACE_FLAG_SOFTIRQ;
 
-        if (!trace_seq_printf(s, "%8.8s-%-5d %3d%c%c%c",
-                              comm, entry->pid, cpu,
+        if (!trace_seq_printf(s, "%c%c%c",
                               (entry->flags & TRACE_FLAG_IRQS_OFF) ? 'd' :
                                 (entry->flags & TRACE_FLAG_IRQS_NOSUPPORT) ?
                                   'X' : '.',
@@ -482,9 +486,30 @@ lat_print_generic(struct trace_seq *s, struct trace_entry *entry, int cpu)
                                 hardirq ? 'h' : softirq ? 's' : '.'))
                 return 0;
 
+        if (entry->lock_depth < 0)
+                ret = trace_seq_putc(s, '.');
+        else
+                ret = trace_seq_printf(s, "%d", entry->lock_depth);
+        if (!ret)
+                return 0;
+
         if (entry->preempt_count)
                 return trace_seq_printf(s, "%x", entry->preempt_count);
-        return trace_seq_puts(s, ".");
+        return trace_seq_putc(s, '.');
+}
+
+static int
+lat_print_generic(struct trace_seq *s, struct trace_entry *entry, int cpu)
+{
+        char comm[TASK_COMM_LEN];
+
+        trace_find_cmdline(entry->pid, comm);
+
+        if (!trace_seq_printf(s, "%8.8s-%-5d %3d",
+                              comm, entry->pid, cpu))
+                return 0;
+
+        return trace_print_lat_fmt(s, entry);
 }
 
 static unsigned long preempt_mark_thresh = 100;
diff --git a/kernel/trace/trace_output.h b/kernel/trace/trace_output.h
index d38bec4a9c30..9d91c72ba38b 100644
--- a/kernel/trace/trace_output.h
+++ b/kernel/trace/trace_output.h
@@ -26,6 +26,8 @@ extern struct trace_event *ftrace_find_event(int type);
 
 extern enum print_line_t trace_nop_print(struct trace_iterator *iter,
                                          int flags);
+extern int
+trace_print_lat_fmt(struct trace_seq *s, struct trace_entry *entry);
 
 /* used by module unregistering */
 extern int __unregister_ftrace_event(struct trace_event *event);
diff --git a/kernel/trace/trace_power.c b/kernel/trace/trace_power.c
index 8a30d9874cd4..fe1a00f1445a 100644
--- a/kernel/trace/trace_power.c
+++ b/kernel/trace/trace_power.c
@@ -38,6 +38,7 @@ static void probe_power_end(struct power_trace *it)
 {
         struct ftrace_event_call *call = &event_power;
         struct ring_buffer_event *event;
+        struct ring_buffer *buffer;
         struct trace_power *entry;
         struct trace_array_cpu *data;
         struct trace_array *tr = power_trace;
@@ -45,18 +46,20 @@ static void probe_power_end(struct power_trace *it)
         if (!trace_power_enabled)
                 return;
 
+        buffer = tr->buffer;
+
         preempt_disable();
         it->end = ktime_get();
         data = tr->data[smp_processor_id()];
 
-        event = trace_buffer_lock_reserve(tr, TRACE_POWER,
+        event = trace_buffer_lock_reserve(buffer, TRACE_POWER,
                                           sizeof(*entry), 0, 0);
         if (!event)
                 goto out;
         entry   = ring_buffer_event_data(event);
         entry->state_data = *it;
-        if (!filter_check_discard(call, entry, tr->buffer, event))
-                trace_buffer_unlock_commit(tr, event, 0, 0);
+        if (!filter_check_discard(call, entry, buffer, event))
+                trace_buffer_unlock_commit(buffer, event, 0, 0);
  out:
         preempt_enable();
 }
@@ -66,6 +69,7 @@ static void probe_power_mark(struct power_trace *it, unsigned int type,
 {
         struct ftrace_event_call *call = &event_power;
         struct ring_buffer_event *event;
+        struct ring_buffer *buffer;
         struct trace_power *entry;
         struct trace_array_cpu *data;
         struct trace_array *tr = power_trace;
@@ -73,6 +77,8 @@ static void probe_power_mark(struct power_trace *it, unsigned int type,
         if (!trace_power_enabled)
                 return;
 
+        buffer = tr->buffer;
+
         memset(it, 0, sizeof(struct power_trace));
         it->state = level;
         it->type = type;
@@ -81,14 +87,14 @@ static void probe_power_mark(struct power_trace *it, unsigned int type,
         it->end = it->stamp;
         data = tr->data[smp_processor_id()];
 
-        event = trace_buffer_lock_reserve(tr, TRACE_POWER,
+        event = trace_buffer_lock_reserve(buffer, TRACE_POWER,
                                           sizeof(*entry), 0, 0);
         if (!event)
                 goto out;
         entry   = ring_buffer_event_data(event);
         entry->state_data = *it;
-        if (!filter_check_discard(call, entry, tr->buffer, event))
-                trace_buffer_unlock_commit(tr, event, 0, 0);
+        if (!filter_check_discard(call, entry, buffer, event))
+                trace_buffer_unlock_commit(buffer, event, 0, 0);
  out:
         preempt_enable();
 }
@@ -144,14 +150,12 @@ static void power_trace_reset(struct trace_array *tr)
 
 static int power_trace_init(struct trace_array *tr)
 {
-        int cpu;
         power_trace = tr;
 
         trace_power_enabled = 1;
         tracing_power_register();
 
-        for_each_cpu(cpu, cpu_possible_mask)
-                tracing_reset(tr, cpu);
+        tracing_reset_online_cpus(tr);
         return 0;
 }
 
diff --git a/kernel/trace/trace_printk.c b/kernel/trace/trace_printk.c
index 9bece9687b62..687699d365ae 100644
--- a/kernel/trace/trace_printk.c
+++ b/kernel/trace/trace_printk.c
@@ -155,25 +155,19 @@ int __ftrace_vprintk(unsigned long ip, const char *fmt, va_list ap)
 EXPORT_SYMBOL_GPL(__ftrace_vprintk);
 
 static void *
-t_next(struct seq_file *m, void *v, loff_t *pos)
+t_start(struct seq_file *m, loff_t *pos)
 {
-        const char **fmt = m->private;
-        const char **next = fmt;
-
-        (*pos)++;
+        const char **fmt = __start___trace_bprintk_fmt + *pos;
 
         if ((unsigned long)fmt >= (unsigned long)__stop___trace_bprintk_fmt)
                 return NULL;
-
-        next = fmt;
-        m->private = ++next;
-
         return fmt;
 }
 
-static void *t_start(struct seq_file *m, loff_t *pos)
+static void *t_next(struct seq_file *m, void * v, loff_t *pos)
 {
-        return t_next(m, NULL, pos);
+        (*pos)++;
+        return t_start(m, pos);
 }
 
 static int t_show(struct seq_file *m, void *v)
@@ -182,7 +176,7 @@ static int t_show(struct seq_file *m, void *v)
         const char *str = *fmt;
         int i;
 
-        seq_printf(m, "0x%lx : \"", (unsigned long)fmt);
+        seq_printf(m, "0x%lx : \"", *(unsigned long *)fmt);
 
         /*
          * Tabs and new lines need to be converted.
@@ -224,15 +218,7 @@ static const struct seq_operations show_format_seq_ops = {
 static int
 ftrace_formats_open(struct inode *inode, struct file *file)
 {
-        int ret;
-
-        ret = seq_open(file, &show_format_seq_ops);
-        if (!ret) {
-                struct seq_file *m = file->private_data;
-
-                m->private = __start___trace_bprintk_fmt;
-        }
-        return ret;
+        return seq_open(file, &show_format_seq_ops);
 }
 
 static const struct file_operations ftrace_formats_fops = {
diff --git a/kernel/trace/trace_sched_switch.c b/kernel/trace/trace_sched_switch.c
index a98106dd979c..5fca0f51fde4 100644
--- a/kernel/trace/trace_sched_switch.c
+++ b/kernel/trace/trace_sched_switch.c
@@ -20,6 +20,35 @@ static int			sched_ref;
 static DEFINE_MUTEX(sched_register_mutex);
 static int                      sched_stopped;
 
+
+void
+tracing_sched_switch_trace(struct trace_array *tr,
+                           struct task_struct *prev,
+                           struct task_struct *next,
+                           unsigned long flags, int pc)
+{
+        struct ftrace_event_call *call = &event_context_switch;
+        struct ring_buffer *buffer = tr->buffer;
+        struct ring_buffer_event *event;
+        struct ctx_switch_entry *entry;
+
+        event = trace_buffer_lock_reserve(buffer, TRACE_CTX,
+                                          sizeof(*entry), flags, pc);
+        if (!event)
+                return;
+        entry   = ring_buffer_event_data(event);
+        entry->prev_pid                 = prev->pid;
+        entry->prev_prio                = prev->prio;
+        entry->prev_state               = prev->state;
+        entry->next_pid                 = next->pid;
+        entry->next_prio                = next->prio;
+        entry->next_state               = next->state;
+        entry->next_cpu = task_cpu(next);
+
+        if (!filter_check_discard(call, entry, buffer, event))
+                trace_buffer_unlock_commit(buffer, event, flags, pc);
+}
+
 static void
 probe_sched_switch(struct rq *__rq, struct task_struct *prev,
                         struct task_struct *next)
@@ -49,6 +78,36 @@ probe_sched_switch(struct rq *__rq, struct task_struct *prev,
         local_irq_restore(flags);
 }
 
+void
+tracing_sched_wakeup_trace(struct trace_array *tr,
+                           struct task_struct *wakee,
+                           struct task_struct *curr,
+                           unsigned long flags, int pc)
+{
+        struct ftrace_event_call *call = &event_wakeup;
+        struct ring_buffer_event *event;
+        struct ctx_switch_entry *entry;
+        struct ring_buffer *buffer = tr->buffer;
+
+        event = trace_buffer_lock_reserve(buffer, TRACE_WAKE,
+                                          sizeof(*entry), flags, pc);
+        if (!event)
+                return;
+        entry   = ring_buffer_event_data(event);
+        entry->prev_pid                 = curr->pid;
+        entry->prev_prio                = curr->prio;
+        entry->prev_state               = curr->state;
+        entry->next_pid                 = wakee->pid;
+        entry->next_prio                = wakee->prio;
+        entry->next_state               = wakee->state;
+        entry->next_cpu                 = task_cpu(wakee);
+
+        if (!filter_check_discard(call, entry, buffer, event))
+                ring_buffer_unlock_commit(buffer, event);
+        ftrace_trace_stack(tr->buffer, flags, 6, pc);
+        ftrace_trace_userstack(tr->buffer, flags, pc);
+}
+
 static void
 probe_sched_wakeup(struct rq *__rq, struct task_struct *wakee, int success)
 {
diff --git a/kernel/trace/trace_sched_wakeup.c b/kernel/trace/trace_sched_wakeup.c
index eacb27225173..26185d727676 100644
--- a/kernel/trace/trace_sched_wakeup.c
+++ b/kernel/trace/trace_sched_wakeup.c
@@ -24,6 +24,7 @@ static int __read_mostly	tracer_enabled;
 
 static struct task_struct       *wakeup_task;
 static int                      wakeup_cpu;
+static int                      wakeup_current_cpu;
 static unsigned                 wakeup_prio = -1;
 static int                      wakeup_rt;
 
@@ -56,33 +57,23 @@ wakeup_tracer_call(unsigned long ip, unsigned long parent_ip)
         resched = ftrace_preempt_disable();
 
         cpu = raw_smp_processor_id();
+        if (cpu != wakeup_current_cpu)
+                goto out_enable;
+
         data = tr->data[cpu];
         disabled = atomic_inc_return(&data->disabled);
         if (unlikely(disabled != 1))
                 goto out;
 
         local_irq_save(flags);
-        __raw_spin_lock(&wakeup_lock);
-
-        if (unlikely(!wakeup_task))
-                goto unlock;
-
-        /*
-         * The task can't disappear because it needs to
-         * wake up first, and we have the wakeup_lock.
-         */
-        if (task_cpu(wakeup_task) != cpu)
-                goto unlock;
 
         trace_function(tr, ip, parent_ip, flags, pc);
 
- unlock:
-        __raw_spin_unlock(&wakeup_lock);
         local_irq_restore(flags);
 
  out:
         atomic_dec(&data->disabled);
-
+ out_enable:
         ftrace_preempt_enable(resched);
 }
 
@@ -107,11 +98,18 @@ static int report_latency(cycle_t delta)
         return 1;
 }
 
+static void probe_wakeup_migrate_task(struct task_struct *task, int cpu)
+{
+        if (task != wakeup_task)
+                return;
+
+        wakeup_current_cpu = cpu;
+}
+
 static void notrace
 probe_wakeup_sched_switch(struct rq *rq, struct task_struct *prev,
         struct task_struct *next)
 {
-        unsigned long latency = 0, t0 = 0, t1 = 0;
         struct trace_array_cpu *data;
         cycle_t T0, T1, delta;
         unsigned long flags;
@@ -157,10 +155,6 @@ probe_wakeup_sched_switch(struct rq *rq, struct task_struct *prev,
         trace_function(wakeup_trace, CALLER_ADDR0, CALLER_ADDR1, flags, pc);
         tracing_sched_switch_trace(wakeup_trace, prev, next, flags, pc);
 
-        /*
-         * usecs conversion is slow so we try to delay the conversion
-         * as long as possible:
-         */
         T0 = data->preempt_timestamp;
         T1 = ftrace_now(cpu);
         delta = T1-T0;
@@ -168,13 +162,10 @@ probe_wakeup_sched_switch(struct rq *rq, struct task_struct *prev,
         if (!report_latency(delta))
                 goto out_unlock;
 
-        latency = nsecs_to_usecs(delta);
-
-        tracing_max_latency = delta;
-        t0 = nsecs_to_usecs(T0);
-        t1 = nsecs_to_usecs(T1);
-
-        update_max_tr(wakeup_trace, wakeup_task, wakeup_cpu);
+        if (likely(!is_tracing_stopped())) {
+                tracing_max_latency = delta;
+                update_max_tr(wakeup_trace, wakeup_task, wakeup_cpu);
+        }
 
 out_unlock:
         __wakeup_reset(wakeup_trace);
@@ -186,11 +177,6 @@ out:
 
 static void __wakeup_reset(struct trace_array *tr)
 {
-        int cpu;
-
-        for_each_possible_cpu(cpu)
-                tracing_reset(tr, cpu);
-
         wakeup_cpu = -1;
         wakeup_prio = -1;
 
@@ -204,6 +190,8 @@ static void wakeup_reset(struct trace_array *tr)
 {
         unsigned long flags;
 
+        tracing_reset_online_cpus(tr);
+
         local_irq_save(flags);
         __raw_spin_lock(&wakeup_lock);
         __wakeup_reset(tr);
@@ -247,6 +235,7 @@ probe_wakeup(struct rq *rq, struct task_struct *p, int success)
         __wakeup_reset(wakeup_trace);
 
         wakeup_cpu = task_cpu(p);
+        wakeup_current_cpu = wakeup_cpu;
         wakeup_prio = p->prio;
 
         wakeup_task = p;
@@ -296,6 +285,13 @@ static void start_wakeup_tracer(struct trace_array *tr)
                 goto fail_deprobe_wake_new;
         }
 
+        ret = register_trace_sched_migrate_task(probe_wakeup_migrate_task);
+        if (ret) {
+                pr_info("wakeup trace: Couldn't activate tracepoint"
+                        " probe to kernel_sched_migrate_task\n");
+                return;
+        }
+
         wakeup_reset(tr);
 
         /*
@@ -328,6 +324,7 @@ static void stop_wakeup_tracer(struct trace_array *tr)
         unregister_trace_sched_switch(probe_wakeup_sched_switch);
         unregister_trace_sched_wakeup_new(probe_wakeup);
         unregister_trace_sched_wakeup(probe_wakeup);
+        unregister_trace_sched_migrate_task(probe_wakeup_migrate_task);
 }
 
 static int __wakeup_tracer_init(struct trace_array *tr)
diff --git a/kernel/trace/trace_selftest.c b/kernel/trace/trace_selftest.c
index 00dd6485bdd7..d2cdbabb4ead 100644
--- a/kernel/trace/trace_selftest.c
+++ b/kernel/trace/trace_selftest.c
@@ -288,6 +288,7 @@ trace_selftest_startup_function_graph(struct tracer *trace,
          * to detect and recover from possible hangs
          */
         tracing_reset_online_cpus(tr);
+        set_graph_array(tr);
         ret = register_ftrace_graph(&trace_graph_return,
                                     &trace_graph_entry_watchdog);
         if (ret) {
diff --git a/kernel/trace/trace_stack.c b/kernel/trace/trace_stack.c
index 2d7aebd71dbd..0f6facb050a1 100644
--- a/kernel/trace/trace_stack.c
+++ b/kernel/trace/trace_stack.c
@@ -186,43 +186,33 @@ static const struct file_operations stack_max_size_fops = {
 };
 
 static void *
-t_next(struct seq_file *m, void *v, loff_t *pos)
+__next(struct seq_file *m, loff_t *pos)
 {
-        long i;
-
-        (*pos)++;
-
-        if (v == SEQ_START_TOKEN)
-                i = 0;
-        else {
-                i = *(long *)v;
-                i++;
-        }
+        long n = *pos - 1;
 
-        if (i >= max_stack_trace.nr_entries ||
-            stack_dump_trace[i] == ULONG_MAX)
+        if (n >= max_stack_trace.nr_entries || stack_dump_trace[n] == ULONG_MAX)
                 return NULL;
 
-        m->private = (void *)i;
-
+        m->private = (void *)n;
         return &m->private;
 }
 
-static void *t_start(struct seq_file *m, loff_t *pos)
+static void *
+t_next(struct seq_file *m, void *v, loff_t *pos)
 {
-        void *t = SEQ_START_TOKEN;
-        loff_t l = 0;
+        (*pos)++;
+        return __next(m, pos);
+}
 
+static void *t_start(struct seq_file *m, loff_t *pos)
+{
         local_irq_disable();
         __raw_spin_lock(&max_stack_lock);
 
         if (*pos == 0)
                 return SEQ_START_TOKEN;
 
-        for (; t && l < *pos; t = t_next(m, t, &l))
-                ;
-
-        return t;
+        return __next(m, pos);
 }
 
 static void t_stop(struct seq_file *m, void *p)
@@ -234,15 +224,8 @@ static void t_stop(struct seq_file *m, void *p)
 static int trace_lookup_stack(struct seq_file *m, long i)
 {
         unsigned long addr = stack_dump_trace[i];
-#ifdef CONFIG_KALLSYMS
-        char str[KSYM_SYMBOL_LEN];
 
-        sprint_symbol(str, addr);
-
-        return seq_printf(m, "%s\n", str);
-#else
-        return seq_printf(m, "%p\n", (void*)addr);
-#endif
+        return seq_printf(m, "%pF\n", (void *)addr);
 }
 
 static void print_disabled(struct seq_file *m)
@@ -301,17 +284,14 @@ static const struct seq_operations stack_trace_seq_ops = {
 
 static int stack_trace_open(struct inode *inode, struct file *file)
 {
-        int ret;
-
-        ret = seq_open(file, &stack_trace_seq_ops);
-
-        return ret;
+        return seq_open(file, &stack_trace_seq_ops);
 }
 
 static const struct file_operations stack_trace_fops = {
         .open           = stack_trace_open,
         .read           = seq_read,
         .llseek         = seq_lseek,
+        .release        = seq_release,
 };
 
 int
@@ -326,10 +306,10 @@ stack_trace_sysctl(struct ctl_table *table, int write,
         ret = proc_dointvec(table, write, file, buffer, lenp, ppos);
 
         if (ret || !write ||
-            (last_stack_tracer_enabled == stack_tracer_enabled))
+            (last_stack_tracer_enabled == !!stack_tracer_enabled))
                 goto out;
 
-        last_stack_tracer_enabled = stack_tracer_enabled;
+        last_stack_tracer_enabled = !!stack_tracer_enabled;
 
         if (stack_tracer_enabled)
                 register_ftrace_function(&trace_ops);
diff --git a/kernel/trace/trace_stat.c b/kernel/trace/trace_stat.c
index c00643733f4c..a4bb239eb987 100644
--- a/kernel/trace/trace_stat.c
+++ b/kernel/trace/trace_stat.c
@@ -49,7 +49,8 @@ static struct dentry		*stat_dir;
  * but it will at least advance closer to the next one
  * to be released.
  */
-static struct rb_node *release_next(struct rb_node *node)
+static struct rb_node *release_next(struct tracer_stat *ts,
+                                    struct rb_node *node)
 {
         struct stat_node *snode;
         struct rb_node *parent = rb_parent(node);
@@ -67,26 +68,35 @@ static struct rb_node *release_next(struct rb_node *node)
                         parent->rb_right = NULL;
 
                 snode = container_of(node, struct stat_node, node);
+                if (ts->stat_release)
+                        ts->stat_release(snode->stat);
                 kfree(snode);
 
                 return parent;
         }
 }
 
-static void reset_stat_session(struct stat_session *session)
+static void __reset_stat_session(struct stat_session *session)
 {
         struct rb_node *node = session->stat_root.rb_node;
 
         while (node)
-                node = release_next(node);
+                node = release_next(session->ts, node);
 
         session->stat_root = RB_ROOT;
 }
 
+static void reset_stat_session(struct stat_session *session)
+{
+        mutex_lock(&session->stat_mutex);
+        __reset_stat_session(session);
+        mutex_unlock(&session->stat_mutex);
+}
+
 static void destroy_session(struct stat_session *session)
 {
         debugfs_remove(session->file);
-        reset_stat_session(session);
+        __reset_stat_session(session);
         mutex_destroy(&session->stat_mutex);
         kfree(session);
 }
@@ -150,7 +160,7 @@ static int stat_seq_init(struct stat_session *session)
         int i;
 
         mutex_lock(&session->stat_mutex);
-        reset_stat_session(session);
+        __reset_stat_session(session);
 
         if (!ts->stat_cmp)
                 ts->stat_cmp = dummy_cmp;
@@ -183,7 +193,7 @@ exit:
         return ret;
 
 exit_free_rbtree:
-        reset_stat_session(session);
+        __reset_stat_session(session);
         mutex_unlock(&session->stat_mutex);
         return ret;
 }
@@ -193,23 +203,23 @@ static void *stat_seq_start(struct seq_file *s, loff_t *pos)
 {
         struct stat_session *session = s->private;
         struct rb_node *node;
+        int n = *pos;
         int i;
 
         /* Prevent from tracer switch or rbtree modification */
         mutex_lock(&session->stat_mutex);
 
         /* If we are in the beginning of the file, print the headers */
-        if (!*pos && session->ts->stat_headers) {
-                (*pos)++;
-                return SEQ_START_TOKEN;
+        if (session->ts->stat_headers) {
+                if (n == 0)
+                        return SEQ_START_TOKEN;
+                n--;
         }
 
         node = rb_first(&session->stat_root);
-        for (i = 0; node && i < *pos; i++)
+        for (i = 0; node && i < n; i++)
                 node = rb_next(node);
 
-        (*pos)++;
-
         return node;
 }
 
@@ -254,16 +264,21 @@ static const struct seq_operations trace_stat_seq_ops = {
 static int tracing_stat_open(struct inode *inode, struct file *file)
 {
         int ret;
-
+        struct seq_file *m;
         struct stat_session *session = inode->i_private;
 
+        ret = stat_seq_init(session);
+        if (ret)
+                return ret;
+
         ret = seq_open(file, &trace_stat_seq_ops);
-        if (!ret) {
-                struct seq_file *m = file->private_data;
-                m->private = session;
-                ret = stat_seq_init(session);
+        if (ret) {
+                reset_stat_session(session);
+                return ret;
         }
 
+        m = file->private_data;
+        m->private = session;
         return ret;
 }
 
@@ -274,11 +289,9 @@ static int tracing_stat_release(struct inode *i, struct file *f)
 {
         struct stat_session *session = i->i_private;
 
-        mutex_lock(&session->stat_mutex);
         reset_stat_session(session);
-        mutex_unlock(&session->stat_mutex);
 
-        return 0;
+        return seq_release(i, f);
 }
 
 static const struct file_operations tracing_stat_fops = {
diff --git a/kernel/trace/trace_stat.h b/kernel/trace/trace_stat.h
index f3546a2cd826..8f03914b9a6a 100644
--- a/kernel/trace/trace_stat.h
+++ b/kernel/trace/trace_stat.h
@@ -18,6 +18,8 @@ struct tracer_stat {
         int                     (*stat_cmp)(void *p1, void *p2);
         /* Print a stat entry */
         int                     (*stat_show)(struct seq_file *s, void *p);
+        /* Release an entry */
+        void                    (*stat_release)(void *stat);
         /* Print the headers of your stat entries */
         int                     (*stat_headers)(struct seq_file *s);
 };
diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c
index 5e579645ac86..8712ce3c6a0e 100644
--- a/kernel/trace/trace_syscalls.c
+++ b/kernel/trace/trace_syscalls.c
@@ -1,30 +1,18 @@
 #include <trace/syscall.h>
+#include <trace/events/syscalls.h>
 #include <linux/kernel.h>
+#include <linux/ftrace.h>
+#include <linux/perf_counter.h>
 #include <asm/syscall.h>
 
 #include "trace_output.h"
 #include "trace.h"
 
-/* Keep a counter of the syscall tracing users */
-static int refcount;
-
-/* Prevent from races on thread flags toggling */
 static DEFINE_MUTEX(syscall_trace_lock);
-
-/* Option to display the parameters types */
-enum {
-        TRACE_SYSCALLS_OPT_TYPES = 0x1,
-};
-
-static struct tracer_opt syscalls_opts[] = {
-        { TRACER_OPT(syscall_arg_type, TRACE_SYSCALLS_OPT_TYPES) },
-        { }
-};
-
-static struct tracer_flags syscalls_flags = {
-        .val = 0, /* By default: no parameters types */
-        .opts = syscalls_opts
-};
+static int sys_refcount_enter;
+static int sys_refcount_exit;
+static DECLARE_BITMAP(enabled_enter_syscalls, NR_syscalls);
+static DECLARE_BITMAP(enabled_exit_syscalls, NR_syscalls);
 
 enum print_line_t
 print_syscall_enter(struct trace_iterator *iter, int flags)
@@ -35,35 +23,46 @@ print_syscall_enter(struct trace_iterator *iter, int flags)
         struct syscall_metadata *entry;
         int i, ret, syscall;
 
-        trace_assign_type(trace, ent);
-
+        trace = (typeof(trace))ent;
         syscall = trace->nr;
-
         entry = syscall_nr_to_meta(syscall);
+
         if (!entry)
                 goto end;
 
+        if (entry->enter_id != ent->type) {
+                WARN_ON_ONCE(1);
+                goto end;
+        }
+
         ret = trace_seq_printf(s, "%s(", entry->name);
         if (!ret)
                 return TRACE_TYPE_PARTIAL_LINE;
 
         for (i = 0; i < entry->nb_args; i++) {
                 /* parameter types */
-                if (syscalls_flags.val & TRACE_SYSCALLS_OPT_TYPES) {
+                if (trace_flags & TRACE_ITER_VERBOSE) {
                         ret = trace_seq_printf(s, "%s ", entry->types[i]);
                         if (!ret)
                                 return TRACE_TYPE_PARTIAL_LINE;
                 }
                 /* parameter values */
-                ret = trace_seq_printf(s, "%s: %lx%s ", entry->args[i],
+                ret = trace_seq_printf(s, "%s: %lx%s", entry->args[i],
                                        trace->args[i],
-                                       i == entry->nb_args - 1 ? ")" : ",");
+                                       i == entry->nb_args - 1 ? "" : ", ");
                 if (!ret)
                         return TRACE_TYPE_PARTIAL_LINE;
         }
 
+        ret = trace_seq_putc(s, ')');
+        if (!ret)
+                return TRACE_TYPE_PARTIAL_LINE;
+
 end:
-        trace_seq_printf(s, "\n");
+        ret =  trace_seq_putc(s, '\n');
+        if (!ret)
+                return TRACE_TYPE_PARTIAL_LINE;
+
         return TRACE_TYPE_HANDLED;
 }
 
@@ -77,16 +76,20 @@ print_syscall_exit(struct trace_iterator *iter, int flags)
         struct syscall_metadata *entry;
         int ret;
 
-        trace_assign_type(trace, ent);
-
+        trace = (typeof(trace))ent;
         syscall = trace->nr;
-
         entry = syscall_nr_to_meta(syscall);
+
         if (!entry) {
                 trace_seq_printf(s, "\n");
                 return TRACE_TYPE_HANDLED;
         }
 
+        if (entry->exit_id != ent->type) {
+                WARN_ON_ONCE(1);
+                return TRACE_TYPE_UNHANDLED;
+        }
+
         ret = trace_seq_printf(s, "%s -> 0x%lx\n", entry->name,
                                 trace->ret);
         if (!ret)
@@ -95,62 +98,140 @@ print_syscall_exit(struct trace_iterator *iter, int flags)
         return TRACE_TYPE_HANDLED;
 }
 
-void start_ftrace_syscalls(void)
+extern char *__bad_type_size(void);
+
+#define SYSCALL_FIELD(type, name)                                       \
+        sizeof(type) != sizeof(trace.name) ?                            \
+                __bad_type_size() :                                     \
+                #type, #name, offsetof(typeof(trace), name), sizeof(trace.name)
+
+int syscall_enter_format(struct ftrace_event_call *call, struct trace_seq *s)
 {
-        unsigned long flags;
-        struct task_struct *g, *t;
+        int i;
+        int nr;
+        int ret;
+        struct syscall_metadata *entry;
+        struct syscall_trace_enter trace;
+        int offset = offsetof(struct syscall_trace_enter, args);
 
-        mutex_lock(&syscall_trace_lock);
+        nr = syscall_name_to_nr(call->data);
+        entry = syscall_nr_to_meta(nr);
 
-        /* Don't enable the flag on the tasks twice */
-        if (++refcount != 1)
-                goto unlock;
+        if (!entry)
+                return 0;
 
-        arch_init_ftrace_syscalls();
-        read_lock_irqsave(&tasklist_lock, flags);
+        ret = trace_seq_printf(s, "\tfield:%s %s;\toffset:%zu;\tsize:%zu;\n",
+                               SYSCALL_FIELD(int, nr));
+        if (!ret)
+                return 0;
 
-        do_each_thread(g, t) {
-                set_tsk_thread_flag(t, TIF_SYSCALL_FTRACE);
-        } while_each_thread(g, t);
+        for (i = 0; i < entry->nb_args; i++) {
+                ret = trace_seq_printf(s, "\tfield:%s %s;", entry->types[i],
+                                        entry->args[i]);
+                if (!ret)
+                        return 0;
+                ret = trace_seq_printf(s, "\toffset:%d;\tsize:%zu;\n", offset,
+                                       sizeof(unsigned long));
+                if (!ret)
+                        return 0;
+                offset += sizeof(unsigned long);
+        }
 
-        read_unlock_irqrestore(&tasklist_lock, flags);
+        trace_seq_puts(s, "\nprint fmt: \"");
+        for (i = 0; i < entry->nb_args; i++) {
+                ret = trace_seq_printf(s, "%s: 0x%%0%zulx%s", entry->args[i],
+                                        sizeof(unsigned long),
+                                        i == entry->nb_args - 1 ? "" : ", ");
+                if (!ret)
+                        return 0;
+        }
+        trace_seq_putc(s, '"');
 
-unlock:
-        mutex_unlock(&syscall_trace_lock);
+        for (i = 0; i < entry->nb_args; i++) {
+                ret = trace_seq_printf(s, ", ((unsigned long)(REC->%s))",
+                                       entry->args[i]);
+                if (!ret)
+                        return 0;
+        }
+
+        return trace_seq_putc(s, '\n');
 }
 
-void stop_ftrace_syscalls(void)
+int syscall_exit_format(struct ftrace_event_call *call, struct trace_seq *s)
 {
-        unsigned long flags;
-        struct task_struct *g, *t;
+        int ret;
+        struct syscall_trace_exit trace;
 
-        mutex_lock(&syscall_trace_lock);
+        ret = trace_seq_printf(s,
+                               "\tfield:%s %s;\toffset:%zu;\tsize:%zu;\n"
+                               "\tfield:%s %s;\toffset:%zu;\tsize:%zu;\n",
+                               SYSCALL_FIELD(int, nr),
+                               SYSCALL_FIELD(unsigned long, ret));
+        if (!ret)
+                return 0;
 
-        /* There are perhaps still some users */
-        if (--refcount)
-                goto unlock;
+        return trace_seq_printf(s, "\nprint fmt: \"0x%%lx\", REC->ret\n");
+}
 
-        read_lock_irqsave(&tasklist_lock, flags);
+int syscall_enter_define_fields(struct ftrace_event_call *call)
+{
+        struct syscall_trace_enter trace;
+        struct syscall_metadata *meta;
+        int ret;
+        int nr;
+        int i;
+        int offset = offsetof(typeof(trace), args);
+
+        nr = syscall_name_to_nr(call->data);
+        meta = syscall_nr_to_meta(nr);
+
+        if (!meta)
+                return 0;
+
+        ret = trace_define_common_fields(call);
+        if (ret)
+                return ret;
+
+        for (i = 0; i < meta->nb_args; i++) {
+                ret = trace_define_field(call, meta->types[i],
+                                         meta->args[i], offset,
+                                         sizeof(unsigned long), 0,
+                                         FILTER_OTHER);
+                offset += sizeof(unsigned long);
+        }
 
-        do_each_thread(g, t) {
-                clear_tsk_thread_flag(t, TIF_SYSCALL_FTRACE);
-        } while_each_thread(g, t);
+        return ret;
+}
 
-        read_unlock_irqrestore(&tasklist_lock, flags);
+int syscall_exit_define_fields(struct ftrace_event_call *call)
+{
+        struct syscall_trace_exit trace;
+        int ret;
 
-unlock:
-        mutex_unlock(&syscall_trace_lock);
+        ret = trace_define_common_fields(call);
+        if (ret)
+                return ret;
+
+        ret = trace_define_field(call, SYSCALL_FIELD(unsigned long, ret), 0,
+                                 FILTER_OTHER);
+
+        return ret;
 }
 
-void ftrace_syscall_enter(struct pt_regs *regs)
+void ftrace_syscall_enter(struct pt_regs *regs, long id)
 {
         struct syscall_trace_enter *entry;
         struct syscall_metadata *sys_data;
         struct ring_buffer_event *event;
+        struct ring_buffer *buffer;
         int size;
         int syscall_nr;
 
         syscall_nr = syscall_get_nr(current, regs);
+        if (syscall_nr < 0)
+                return;
+        if (!test_bit(syscall_nr, enabled_enter_syscalls))
+                return;
 
         sys_data = syscall_nr_to_meta(syscall_nr);
         if (!sys_data)
@@ -158,8 +239,8 @@ void ftrace_syscall_enter(struct pt_regs *regs)
 
         size = sizeof(*entry) + sizeof(unsigned long) * sys_data->nb_args;
 
-        event = trace_current_buffer_lock_reserve(TRACE_SYSCALL_ENTER, size,
-                                                        0, 0);
+        event = trace_current_buffer_lock_reserve(&buffer, sys_data->enter_id,
+                                                  size, 0, 0);
         if (!event)
                 return;
 
@@ -167,24 +248,30 @@ void ftrace_syscall_enter(struct pt_regs *regs)
         entry->nr = syscall_nr;
         syscall_get_arguments(current, regs, 0, sys_data->nb_args, entry->args);
 
-        trace_current_buffer_unlock_commit(event, 0, 0);
-        trace_wake_up();
+        if (!filter_current_check_discard(buffer, sys_data->enter_event,
+                                          entry, event))
+                trace_current_buffer_unlock_commit(buffer, event, 0, 0);
 }
 
-void ftrace_syscall_exit(struct pt_regs *regs)
+void ftrace_syscall_exit(struct pt_regs *regs, long ret)
 {
         struct syscall_trace_exit *entry;
         struct syscall_metadata *sys_data;
         struct ring_buffer_event *event;
+        struct ring_buffer *buffer;
         int syscall_nr;
 
         syscall_nr = syscall_get_nr(current, regs);
+        if (syscall_nr < 0)
+                return;
+        if (!test_bit(syscall_nr, enabled_exit_syscalls))
+                return;
 
         sys_data = syscall_nr_to_meta(syscall_nr);
         if (!sys_data)
                 return;
 
-        event = trace_current_buffer_lock_reserve(TRACE_SYSCALL_EXIT,
+        event = trace_current_buffer_lock_reserve(&buffer, sys_data->exit_id,
                                 sizeof(*entry), 0, 0);
         if (!event)
                 return;
@@ -193,58 +280,244 @@ void ftrace_syscall_exit(struct pt_regs *regs)
         entry->nr = syscall_nr;
         entry->ret = syscall_get_return_value(current, regs);
 
-        trace_current_buffer_unlock_commit(event, 0, 0);
-        trace_wake_up();
+        if (!filter_current_check_discard(buffer, sys_data->exit_event,
+                                          entry, event))
+                trace_current_buffer_unlock_commit(buffer, event, 0, 0);
 }
 
-static int init_syscall_tracer(struct trace_array *tr)
+int reg_event_syscall_enter(void *ptr)
 {
-        start_ftrace_syscalls();
+        int ret = 0;
+        int num;
+        char *name;
+
+        name = (char *)ptr;
+        num = syscall_name_to_nr(name);
+        if (num < 0 || num >= NR_syscalls)
+                return -ENOSYS;
+        mutex_lock(&syscall_trace_lock);
+        if (!sys_refcount_enter)
+                ret = register_trace_sys_enter(ftrace_syscall_enter);
+        if (ret) {
+                pr_info("event trace: Could not activate"
+                                "syscall entry trace point");
+        } else {
+                set_bit(num, enabled_enter_syscalls);
+                sys_refcount_enter++;
+        }
+        mutex_unlock(&syscall_trace_lock);
+        return ret;
+}
+
+void unreg_event_syscall_enter(void *ptr)
+{
+        int num;
+        char *name;
 
-        return 0;
+        name = (char *)ptr;
+        num = syscall_name_to_nr(name);
+        if (num < 0 || num >= NR_syscalls)
+                return;
+        mutex_lock(&syscall_trace_lock);
+        sys_refcount_enter--;
+        clear_bit(num, enabled_enter_syscalls);
+        if (!sys_refcount_enter)
+                unregister_trace_sys_enter(ftrace_syscall_enter);
+        mutex_unlock(&syscall_trace_lock);
 }
 
-static void reset_syscall_tracer(struct trace_array *tr)
+int reg_event_syscall_exit(void *ptr)
 {
-        stop_ftrace_syscalls();
-        tracing_reset_online_cpus(tr);
+        int ret = 0;
+        int num;
+        char *name;
+
+        name = (char *)ptr;
+        num = syscall_name_to_nr(name);
+        if (num < 0 || num >= NR_syscalls)
+                return -ENOSYS;
+        mutex_lock(&syscall_trace_lock);
+        if (!sys_refcount_exit)
+                ret = register_trace_sys_exit(ftrace_syscall_exit);
+        if (ret) {
+                pr_info("event trace: Could not activate"
+                                "syscall exit trace point");
+        } else {
+                set_bit(num, enabled_exit_syscalls);
+                sys_refcount_exit++;
+        }
+        mutex_unlock(&syscall_trace_lock);
+        return ret;
 }
 
-static struct trace_event syscall_enter_event = {
-        .type           = TRACE_SYSCALL_ENTER,
-        .trace          = print_syscall_enter,
-};
+void unreg_event_syscall_exit(void *ptr)
+{
+        int num;
+        char *name;
+
+        name = (char *)ptr;
+        num = syscall_name_to_nr(name);
+        if (num < 0 || num >= NR_syscalls)
+                return;
+        mutex_lock(&syscall_trace_lock);
+        sys_refcount_exit--;
+        clear_bit(num, enabled_exit_syscalls);
+        if (!sys_refcount_exit)
+                unregister_trace_sys_exit(ftrace_syscall_exit);
+        mutex_unlock(&syscall_trace_lock);
+}
 
-static struct trace_event syscall_exit_event = {
-        .type           = TRACE_SYSCALL_EXIT,
-        .trace          = print_syscall_exit,
+struct trace_event event_syscall_enter = {
+        .trace                  = print_syscall_enter,
 };
 
-static struct tracer syscall_tracer __read_mostly = {
-        .name           = "syscall",
-        .init           = init_syscall_tracer,
-        .reset          = reset_syscall_tracer,
-        .flags          = &syscalls_flags,
+struct trace_event event_syscall_exit = {
+        .trace                  = print_syscall_exit,
 };
 
-__init int register_ftrace_syscalls(void)
+#ifdef CONFIG_EVENT_PROFILE
+
+static DECLARE_BITMAP(enabled_prof_enter_syscalls, NR_syscalls);
+static DECLARE_BITMAP(enabled_prof_exit_syscalls, NR_syscalls);
+static int sys_prof_refcount_enter;
+static int sys_prof_refcount_exit;
+
+static void prof_syscall_enter(struct pt_regs *regs, long id)
 {
-        int ret;
+        struct syscall_trace_enter *rec;
+        struct syscall_metadata *sys_data;
+        int syscall_nr;
+        int size;
 
-        ret = register_ftrace_event(&syscall_enter_event);
-        if (!ret) {
-                printk(KERN_WARNING "event %d failed to register\n",
-                       syscall_enter_event.type);
-                WARN_ON_ONCE(1);
+        syscall_nr = syscall_get_nr(current, regs);
+        if (!test_bit(syscall_nr, enabled_prof_enter_syscalls))
+                return;
+
+        sys_data = syscall_nr_to_meta(syscall_nr);
+        if (!sys_data)
+                return;
+
+        /* get the size after alignment with the u32 buffer size field */
+        size = sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
+        size = ALIGN(size + sizeof(u32), sizeof(u64));
+        size -= sizeof(u32);
+
+        do {
+                char raw_data[size];
+
+                /* zero the dead bytes from align to not leak stack to user */
+                *(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL;
+
+                rec = (struct syscall_trace_enter *) raw_data;
+                tracing_generic_entry_update(&rec->ent, 0, 0);
+                rec->ent.type = sys_data->enter_id;
+                rec->nr = syscall_nr;
+                syscall_get_arguments(current, regs, 0, sys_data->nb_args,
+                                       (unsigned long *)&rec->args);
+                perf_tpcounter_event(sys_data->enter_id, 0, 1, rec, size);
+        } while(0);
+}
+
+int reg_prof_syscall_enter(char *name)
+{
+        int ret = 0;
+        int num;
+
+        num = syscall_name_to_nr(name);
+        if (num < 0 || num >= NR_syscalls)
+                return -ENOSYS;
+
+        mutex_lock(&syscall_trace_lock);
+        if (!sys_prof_refcount_enter)
+                ret = register_trace_sys_enter(prof_syscall_enter);
+        if (ret) {
+                pr_info("event trace: Could not activate"
+                                "syscall entry trace point");
+        } else {
+                set_bit(num, enabled_prof_enter_syscalls);
+                sys_prof_refcount_enter++;
         }
+        mutex_unlock(&syscall_trace_lock);
+        return ret;
+}
 
-        ret = register_ftrace_event(&syscall_exit_event);
-        if (!ret) {
-                printk(KERN_WARNING "event %d failed to register\n",
-                       syscall_exit_event.type);
-                WARN_ON_ONCE(1);
+void unreg_prof_syscall_enter(char *name)
+{
+        int num;
+
+        num = syscall_name_to_nr(name);
+        if (num < 0 || num >= NR_syscalls)
+                return;
+
+        mutex_lock(&syscall_trace_lock);
+        sys_prof_refcount_enter--;
+        clear_bit(num, enabled_prof_enter_syscalls);
+        if (!sys_prof_refcount_enter)
+                unregister_trace_sys_enter(prof_syscall_enter);
+        mutex_unlock(&syscall_trace_lock);
+}
+
+static void prof_syscall_exit(struct pt_regs *regs, long ret)
+{
+        struct syscall_metadata *sys_data;
+        struct syscall_trace_exit rec;
+        int syscall_nr;
+
+        syscall_nr = syscall_get_nr(current, regs);
+        if (!test_bit(syscall_nr, enabled_prof_exit_syscalls))
+                return;
+
+        sys_data = syscall_nr_to_meta(syscall_nr);
+        if (!sys_data)
+                return;
+
+        tracing_generic_entry_update(&rec.ent, 0, 0);
+        rec.ent.type = sys_data->exit_id;
+        rec.nr = syscall_nr;
+        rec.ret = syscall_get_return_value(current, regs);
+
+        perf_tpcounter_event(sys_data->exit_id, 0, 1, &rec, sizeof(rec));
+}
+
+int reg_prof_syscall_exit(char *name)
+{
+        int ret = 0;
+        int num;
+
+        num = syscall_name_to_nr(name);
+        if (num < 0 || num >= NR_syscalls)
+                return -ENOSYS;
+
+        mutex_lock(&syscall_trace_lock);
+        if (!sys_prof_refcount_exit)
+                ret = register_trace_sys_exit(prof_syscall_exit);
+        if (ret) {
+                pr_info("event trace: Could not activate"
+                                "syscall entry trace point");
+        } else {
+                set_bit(num, enabled_prof_exit_syscalls);
+                sys_prof_refcount_exit++;
         }
+        mutex_unlock(&syscall_trace_lock);
+        return ret;
+}
 
-        return register_tracer(&syscall_tracer);
+void unreg_prof_syscall_exit(char *name)
+{
+        int num;
+
+        num = syscall_name_to_nr(name);
+        if (num < 0 || num >= NR_syscalls)
+                return;
+
+        mutex_lock(&syscall_trace_lock);
+        sys_prof_refcount_exit--;
+        clear_bit(num, enabled_prof_exit_syscalls);
+        if (!sys_prof_refcount_exit)
+                unregister_trace_sys_exit(prof_syscall_exit);
+        mutex_unlock(&syscall_trace_lock);
 }
-device_initcall(register_ftrace_syscalls);
+
+#endif
+
+
diff --git a/kernel/trace/trace_workqueue.c b/kernel/trace/trace_workqueue.c
index 97fcea4acce1..40cafb07dffd 100644
--- a/kernel/trace/trace_workqueue.c
+++ b/kernel/trace/trace_workqueue.c
@@ -9,6 +9,7 @@
 #include <trace/events/workqueue.h>
 #include <linux/list.h>
 #include <linux/percpu.h>
+#include <linux/kref.h>
 #include "trace_stat.h"
 #include "trace.h"
 
@@ -16,6 +17,7 @@
 /* A cpu workqueue thread */
 struct cpu_workqueue_stats {
         struct list_head            list;
+        struct kref                 kref;
         int                         cpu;
         pid_t                       pid;
 /* Can be inserted from interrupt or user context, need to be atomic */
@@ -39,6 +41,11 @@ struct workqueue_global_stats {
 static DEFINE_PER_CPU(struct workqueue_global_stats, all_workqueue_stat);
 #define workqueue_cpu_stat(cpu) (&per_cpu(all_workqueue_stat, cpu))
 
+static void cpu_workqueue_stat_free(struct kref *kref)
+{
+        kfree(container_of(kref, struct cpu_workqueue_stats, kref));
+}
+
 /* Insertion of a work */
 static void
 probe_workqueue_insertion(struct task_struct *wq_thread,
@@ -96,8 +103,8 @@ static void probe_workqueue_creation(struct task_struct *wq_thread, int cpu)
                 return;
         }
         INIT_LIST_HEAD(&cws->list);
+        kref_init(&cws->kref);
         cws->cpu = cpu;
-
         cws->pid = wq_thread->pid;
 
         spin_lock_irqsave(&workqueue_cpu_stat(cpu)->lock, flags);
@@ -118,7 +125,7 @@ static void probe_workqueue_destruction(struct task_struct *wq_thread)
                                                         list) {
                 if (node->pid == wq_thread->pid) {
                         list_del(&node->list);
-                        kfree(node);
+                        kref_put(&node->kref, cpu_workqueue_stat_free);
                         goto found;
                 }
         }
@@ -137,9 +144,11 @@ static struct cpu_workqueue_stats *workqueue_stat_start_cpu(int cpu)
 
         spin_lock_irqsave(&workqueue_cpu_stat(cpu)->lock, flags);
 
-        if (!list_empty(&workqueue_cpu_stat(cpu)->list))
+        if (!list_empty(&workqueue_cpu_stat(cpu)->list)) {
                 ret = list_entry(workqueue_cpu_stat(cpu)->list.next,
                                  struct cpu_workqueue_stats, list);
+                kref_get(&ret->kref);
+        }
 
         spin_unlock_irqrestore(&workqueue_cpu_stat(cpu)->lock, flags);
 
@@ -162,9 +171,9 @@ static void *workqueue_stat_start(struct tracer_stat *trace)
 static void *workqueue_stat_next(void *prev, int idx)
 {
         struct cpu_workqueue_stats *prev_cws = prev;
+        struct cpu_workqueue_stats *ret;
         int cpu = prev_cws->cpu;
         unsigned long flags;
-        void *ret = NULL;
 
         spin_lock_irqsave(&workqueue_cpu_stat(cpu)->lock, flags);
         if (list_is_last(&prev_cws->list, &workqueue_cpu_stat(cpu)->list)) {
@@ -175,11 +184,14 @@ static void *workqueue_stat_next(void *prev, int idx)
                                 return NULL;
                 } while (!(ret = workqueue_stat_start_cpu(cpu)));
                 return ret;
+        } else {
+                ret = list_entry(prev_cws->list.next,
+                                 struct cpu_workqueue_stats, list);
+                kref_get(&ret->kref);
         }
         spin_unlock_irqrestore(&workqueue_cpu_stat(cpu)->lock, flags);
 
-        return list_entry(prev_cws->list.next, struct cpu_workqueue_stats,
-                          list);
+        return ret;
 }
 
 static int workqueue_stat_show(struct seq_file *s, void *p)
@@ -203,6 +215,13 @@ static int workqueue_stat_show(struct seq_file *s, void *p)
         return 0;
 }
 
+static void workqueue_stat_release(void *stat)
+{
+        struct cpu_workqueue_stats *node = stat;
+
+        kref_put(&node->kref, cpu_workqueue_stat_free);
+}
+
 static int workqueue_stat_headers(struct seq_file *s)
 {
         seq_printf(s, "# CPU  INSERTED  EXECUTED   NAME\n");
@@ -215,6 +234,7 @@ struct tracer_stat workqueue_stats __read_mostly = {
         .stat_start = workqueue_stat_start,
         .stat_next = workqueue_stat_next,
         .stat_show = workqueue_stat_show,
+        .stat_release = workqueue_stat_release,
         .stat_headers = workqueue_stat_headers
 };
 
diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c
index 1ef5d3a601c7..9489a0a9b1be 100644
--- a/kernel/tracepoint.c
+++ b/kernel/tracepoint.c
@@ -24,6 +24,7 @@
 #include <linux/tracepoint.h>
 #include <linux/err.h>
 #include <linux/slab.h>
+#include <linux/sched.h>
 
 extern struct tracepoint __start___tracepoints[];
 extern struct tracepoint __stop___tracepoints[];
@@ -242,6 +243,11 @@ static void set_tracepoint(struct tracepoint_entry **entry,
 {
         WARN_ON(strcmp((*entry)->name, elem->name) != 0);
 
+        if (elem->regfunc && !elem->state && active)
+                elem->regfunc();
+        else if (elem->unregfunc && elem->state && !active)
+                elem->unregfunc();
+
         /*
          * rcu_assign_pointer has a smp_wmb() which makes sure that the new
          * probe callbacks array is consistent before setting a pointer to it.
@@ -261,6 +267,9 @@ static void set_tracepoint(struct tracepoint_entry **entry,
  */
 static void disable_tracepoint(struct tracepoint *elem)
 {
+        if (elem->unregfunc && elem->state)
+                elem->unregfunc();
+
         elem->state = 0;
         rcu_assign_pointer(elem->funcs, NULL);
 }
@@ -554,9 +563,6 @@ int tracepoint_module_notify(struct notifier_block *self,
 
         switch (val) {
         case MODULE_STATE_COMING:
-                tracepoint_update_probe_range(mod->tracepoints,
-                        mod->tracepoints + mod->num_tracepoints);
-                break;
         case MODULE_STATE_GOING:
                 tracepoint_update_probe_range(mod->tracepoints,
                         mod->tracepoints + mod->num_tracepoints);
@@ -577,3 +583,41 @@ static int init_tracepoints(void)
 __initcall(init_tracepoints);
 
 #endif /* CONFIG_MODULES */
+
+#ifdef CONFIG_HAVE_SYSCALL_TRACEPOINTS
+
+/* NB: reg/unreg are called while guarded with the tracepoints_mutex */
+static int sys_tracepoint_refcount;
+
+void syscall_regfunc(void)
+{
+        unsigned long flags;
+        struct task_struct *g, *t;
+
+        if (!sys_tracepoint_refcount) {
+                read_lock_irqsave(&tasklist_lock, flags);
+                do_each_thread(g, t) {
+                        /* Skip kernel threads. */
+                        if (t->mm)
+                                set_tsk_thread_flag(t, TIF_SYSCALL_TRACEPOINT);
+                } while_each_thread(g, t);
+                read_unlock_irqrestore(&tasklist_lock, flags);
+        }
+        sys_tracepoint_refcount++;
+}
+
+void syscall_unregfunc(void)
+{
+        unsigned long flags;
+        struct task_struct *g, *t;
+
+        sys_tracepoint_refcount--;
+        if (!sys_tracepoint_refcount) {
+                read_lock_irqsave(&tasklist_lock, flags);
+                do_each_thread(g, t) {
+                        clear_tsk_thread_flag(t, TIF_SYSCALL_TRACEPOINT);
+                } while_each_thread(g, t);
+                read_unlock_irqrestore(&tasklist_lock, flags);
+        }
+}
+#endif
diff --git a/kernel/user.c b/kernel/user.c
index 850e0ba41c1e..2c000e7132ac 100644
--- a/kernel/user.c
+++ b/kernel/user.c
@@ -75,21 +75,6 @@ static void uid_hash_remove(struct user_struct *up)
         put_user_ns(up->user_ns);
 }
 
-static struct user_struct *uid_hash_find(uid_t uid, struct hlist_head *hashent)
-{
-        struct user_struct *user;
-        struct hlist_node *h;
-
-        hlist_for_each_entry(user, h, hashent, uidhash_node) {
-                if (user->uid == uid) {
-                        atomic_inc(&user->__count);
-                        return user;
-                }
-        }
-
-        return NULL;
-}
-
 #ifdef CONFIG_USER_SCHED
 
 static void sched_destroy_user(struct user_struct *up)
@@ -119,6 +104,23 @@ static int sched_create_user(struct user_struct *up) { return 0; }
 
 #if defined(CONFIG_USER_SCHED) && defined(CONFIG_SYSFS)
 
+static struct user_struct *uid_hash_find(uid_t uid, struct hlist_head *hashent)
+{
+        struct user_struct *user;
+        struct hlist_node *h;
+
+        hlist_for_each_entry(user, h, hashent, uidhash_node) {
+                if (user->uid == uid) {
+                        /* possibly resurrect an "almost deleted" object */
+                        if (atomic_inc_return(&user->__count) == 1)
+                                cancel_delayed_work(&user->work);
+                        return user;
+                }
+        }
+
+        return NULL;
+}
+
 static struct kset *uids_kset; /* represents the /sys/kernel/uids/ directory */
 static DEFINE_MUTEX(uids_mutex);
 
@@ -283,12 +285,12 @@ int __init uids_sysfs_init(void)
         return uids_user_create(&root_user);
 }
 
-/* work function to remove sysfs directory for a user and free up
+/* delayed work function to remove sysfs directory for a user and free up
  * corresponding structures.
  */
 static void cleanup_user_struct(struct work_struct *w)
 {
-        struct user_struct *up = container_of(w, struct user_struct, work);
+        struct user_struct *up = container_of(w, struct user_struct, work.work);
         unsigned long flags;
         int remove_user = 0;
 
@@ -297,15 +299,12 @@ static void cleanup_user_struct(struct work_struct *w)
          */
         uids_mutex_lock();
 
-        local_irq_save(flags);
-
-        if (atomic_dec_and_lock(&up->__count, &uidhash_lock)) {
+        spin_lock_irqsave(&uidhash_lock, flags);
+        if (atomic_read(&up->__count) == 0) {
                 uid_hash_remove(up);
                 remove_user = 1;
-                spin_unlock_irqrestore(&uidhash_lock, flags);
-        } else {
-                local_irq_restore(flags);
         }
+        spin_unlock_irqrestore(&uidhash_lock, flags);
 
         if (!remove_user)
                 goto done;
@@ -331,16 +330,28 @@ done:
  */
 static void free_user(struct user_struct *up, unsigned long flags)
 {
-        /* restore back the count */
-        atomic_inc(&up->__count);
         spin_unlock_irqrestore(&uidhash_lock, flags);
-
-        INIT_WORK(&up->work, cleanup_user_struct);
-        schedule_work(&up->work);
+        INIT_DELAYED_WORK(&up->work, cleanup_user_struct);
+        schedule_delayed_work(&up->work, msecs_to_jiffies(1000));
 }
 
 #else   /* CONFIG_USER_SCHED && CONFIG_SYSFS */
 
+static struct user_struct *uid_hash_find(uid_t uid, struct hlist_head *hashent)
+{
+        struct user_struct *user;
+        struct hlist_node *h;
+
+        hlist_for_each_entry(user, h, hashent, uidhash_node) {
+                if (user->uid == uid) {
+                        atomic_inc(&user->__count);
+                        return user;
+                }
+        }
+
+        return NULL;
+}
+
 int uids_sysfs_init(void) { return 0; }
 static inline int uids_user_create(struct user_struct *up) { return 0; }
 static inline void uids_mutex_lock(void) { }
diff --git a/kernel/utsname.c b/kernel/utsname.c
index 815237a55af8..8a82b4b8ea52 100644
--- a/kernel/utsname.c
+++ b/kernel/utsname.c
@@ -15,6 +15,16 @@
 #include <linux/err.h>
 #include <linux/slab.h>
 
+static struct uts_namespace *create_uts_ns(void)
+{
+        struct uts_namespace *uts_ns;
+
+        uts_ns = kmalloc(sizeof(struct uts_namespace), GFP_KERNEL);
+        if (uts_ns)
+                kref_init(&uts_ns->kref);
+        return uts_ns;
+}
+
 /*
  * Clone a new ns copying an original utsname, setting refcount to 1
  * @old_ns: namespace to clone
@@ -24,14 +34,13 @@ static struct uts_namespace *clone_uts_ns(struct uts_namespace *old_ns)
 {
         struct uts_namespace *ns;
 
-        ns = kmalloc(sizeof(struct uts_namespace), GFP_KERNEL);
+        ns = create_uts_ns();
         if (!ns)
                 return ERR_PTR(-ENOMEM);
 
         down_read(&uts_sem);
         memcpy(&ns->name, &old_ns->name, sizeof(ns->name));
         up_read(&uts_sem);
-        kref_init(&ns->kref);
         return ns;
 }
 
diff --git a/kernel/wait.c b/kernel/wait.c
index ea7c3b4275cf..c4bd3d825f35 100644
--- a/kernel/wait.c
+++ b/kernel/wait.c
@@ -10,13 +10,14 @@
 #include <linux/wait.h>
 #include <linux/hash.h>
 
-void init_waitqueue_head(wait_queue_head_t *q)
+void __init_waitqueue_head(wait_queue_head_t *q, struct lock_class_key *key)
 {
         spin_lock_init(&q->lock);
+        lockdep_set_class(&q->lock, key);
         INIT_LIST_HEAD(&q->task_list);
 }
 
-EXPORT_SYMBOL(init_waitqueue_head);
+EXPORT_SYMBOL(__init_waitqueue_head);
 
 void add_wait_queue(wait_queue_head_t *q, wait_queue_t *wait)
 {
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 0668795d8818..addfe2df93b1 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -317,8 +317,6 @@ static int worker_thread(void *__cwq)
         if (cwq->wq->freezeable)
                 set_freezable();
 
-        set_user_nice(current, -5);
-
         for (;;) {
                 prepare_to_wait(&cwq->more_work, &wait, TASK_INTERRUPTIBLE);
                 if (!freezing(current) &&
@@ -600,7 +598,12 @@ static struct workqueue_struct *keventd_wq __read_mostly;
  * schedule_work - put work task in global workqueue
  * @work: job to be done
  *
- * This puts a job in the kernel-global workqueue.
+ * Returns zero if @work was already on the kernel-global workqueue and
+ * non-zero otherwise.
+ *
+ * This puts a job in the kernel-global workqueue if it was not already
+ * queued and leaves it in the same position on the kernel-global
+ * workqueue otherwise.
  */
 int schedule_work(struct work_struct *work)
 {