Merge branch 'linus' into tracing/core

Merge reason: Pick up kernel/softirq.c update for dependent fix.

Signed-off-by: Ingo Molnar <mingo@elte.hu>

55 files changed, 4003 insertions, 4375 deletions

diff --git a/kernel/Makefile b/kernel/Makefile
index 2093a691f1c2..3d9c7e27e3f9 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -80,11 +80,9 @@ obj-$(CONFIG_DETECT_HUNG_TASK) += hung_task.o
 obj-$(CONFIG_GENERIC_HARDIRQS) += irq/
 obj-$(CONFIG_SECCOMP) += seccomp.o
 obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o
-obj-$(CONFIG_CLASSIC_RCU) += rcuclassic.o
 obj-$(CONFIG_TREE_RCU) += rcutree.o
-obj-$(CONFIG_PREEMPT_RCU) += rcupreempt.o
+obj-$(CONFIG_TREE_PREEMPT_RCU) += rcutree.o
 obj-$(CONFIG_TREE_RCU_TRACE) += rcutree_trace.o
-obj-$(CONFIG_PREEMPT_RCU_TRACE) += rcupreempt_trace.o
 obj-$(CONFIG_RELAY) += relay.o
 obj-$(CONFIG_SYSCTL) += utsname_sysctl.o
 obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o
@@ -92,7 +90,6 @@ obj-$(CONFIG_TASKSTATS) += taskstats.o tsacct.o
 obj-$(CONFIG_MARKERS) += marker.o
 obj-$(CONFIG_TRACEPOINTS) += tracepoint.o
 obj-$(CONFIG_LATENCYTOP) += latencytop.o
-obj-$(CONFIG_HAVE_GENERIC_DMA_COHERENT) += dma-coherent.o
 obj-$(CONFIG_FUNCTION_TRACER) += trace/
 obj-$(CONFIG_TRACING) += trace/
 obj-$(CONFIG_X86_DS) += trace/
@@ -119,7 +116,7 @@ $(obj)/config_data.gz: .config FORCE
         $(call if_changed,gzip)
 
 quiet_cmd_ikconfiggz = IKCFG   $@
-      cmd_ikconfiggz = (echo "static const char kernel_config_data[] = MAGIC_START"; cat $< | scripts/bin2c; echo "MAGIC_END;") > $@
+      cmd_ikconfiggz = (echo "static const char kernel_config_data[] __used = MAGIC_START"; cat $< | scripts/bin2c; echo "MAGIC_END;") > $@
 targets += config_data.h
 $(obj)/config_data.h: $(obj)/config_data.gz FORCE
         $(call if_changed,ikconfiggz)
diff --git a/kernel/acct.c b/kernel/acct.c
index 9f3391090b3e..9a4715a2f6bf 100644
--- a/kernel/acct.c
+++ b/kernel/acct.c
@@ -491,13 +491,17 @@ static void do_acct_process(struct bsd_acct_struct *acct,
         u64 run_time;
         struct timespec uptime;
         struct tty_struct *tty;
+        const struct cred *orig_cred;
+
+        /* Perform file operations on behalf of whoever enabled accounting */
+        orig_cred = override_creds(file->f_cred);
 
         /*
          * First check to see if there is enough free_space to continue
          * the process accounting system.
          */
         if (!check_free_space(acct, file))
-                return;
+                goto out;
 
         /*
          * Fill the accounting struct with the needed info as recorded
@@ -578,6 +582,8 @@ static void do_acct_process(struct bsd_acct_struct *acct,
                                sizeof(acct_t), &file->f_pos);
         current->signal->rlim[RLIMIT_FSIZE].rlim_cur = flim;
         set_fs(fs);
+out:
+        revert_creds(orig_cred);
 }
 
 /**
diff --git a/kernel/cgroup.c b/kernel/cgroup.c
index b6eadfe30e7b..c7ece8f027f2 100644
--- a/kernel/cgroup.c
+++ b/kernel/cgroup.c
@@ -600,6 +600,7 @@ static struct inode_operations cgroup_dir_inode_operations;
 static struct file_operations proc_cgroupstats_operations;
 
 static struct backing_dev_info cgroup_backing_dev_info = {
+        .name           = "cgroup",
         .capabilities   = BDI_CAP_NO_ACCT_AND_WRITEBACK,
 };
 
diff --git a/kernel/cpu.c b/kernel/cpu.c
index 8ce10043e4ac..6ba0f1ecb212 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -401,6 +401,7 @@ int disable_nonboot_cpus(void)
                         break;
                 }
         }
+
         if (!error) {
                 BUG_ON(num_online_cpus() > 1);
                 /* Make sure the CPUs won't be enabled by someone else */
@@ -413,6 +414,14 @@ int disable_nonboot_cpus(void)
         return error;
 }
 
+void __weak arch_enable_nonboot_cpus_begin(void)
+{
+}
+
+void __weak arch_enable_nonboot_cpus_end(void)
+{
+}
+
 void __ref enable_nonboot_cpus(void)
 {
         int cpu, error;
@@ -424,6 +433,9 @@ void __ref enable_nonboot_cpus(void)
                 goto out;
 
         printk("Enabling non-boot CPUs ...\n");
+
+        arch_enable_nonboot_cpus_begin();
+
         for_each_cpu(cpu, frozen_cpus) {
                 error = _cpu_up(cpu, 1);
                 if (!error) {
@@ -432,6 +444,9 @@ void __ref enable_nonboot_cpus(void)
                 }
                 printk(KERN_WARNING "Error taking CPU%d up: %d\n", cpu, error);
         }
+
+        arch_enable_nonboot_cpus_end();
+
         cpumask_clear(frozen_cpus);
 out:
         cpu_maps_update_done();
diff --git a/kernel/cred.c b/kernel/cred.c
index 1bb4d7e5d616..d7f7a01082eb 100644
--- a/kernel/cred.c
+++ b/kernel/cred.c
@@ -18,6 +18,18 @@
 #include <linux/cn_proc.h>
 #include "cred-internals.h"
 
+#if 0
+#define kdebug(FMT, ...) \
+        printk("[%-5.5s%5u] "FMT"\n", current->comm, current->pid ,##__VA_ARGS__)
+#else
+static inline __attribute__((format(printf, 1, 2)))
+void no_printk(const char *fmt, ...)
+{
+}
+#define kdebug(FMT, ...) \
+        no_printk("[%-5.5s%5u] "FMT"\n", current->comm, current->pid ,##__VA_ARGS__)
+#endif
+
 static struct kmem_cache *cred_jar;
 
 /*
@@ -36,6 +48,10 @@ static struct thread_group_cred init_tgcred = {
  */
 struct cred init_cred = {
         .usage                  = ATOMIC_INIT(4),
+#ifdef CONFIG_DEBUG_CREDENTIALS
+        .subscribers            = ATOMIC_INIT(2),
+        .magic                  = CRED_MAGIC,
+#endif
         .securebits             = SECUREBITS_DEFAULT,
         .cap_inheritable        = CAP_INIT_INH_SET,
         .cap_permitted          = CAP_FULL_SET,
@@ -48,6 +64,31 @@ struct cred init_cred = {
 #endif
 };
 
+static inline void set_cred_subscribers(struct cred *cred, int n)
+{
+#ifdef CONFIG_DEBUG_CREDENTIALS
+        atomic_set(&cred->subscribers, n);
+#endif
+}
+
+static inline int read_cred_subscribers(const struct cred *cred)
+{
+#ifdef CONFIG_DEBUG_CREDENTIALS
+        return atomic_read(&cred->subscribers);
+#else
+        return 0;
+#endif
+}
+
+static inline void alter_cred_subscribers(const struct cred *_cred, int n)
+{
+#ifdef CONFIG_DEBUG_CREDENTIALS
+        struct cred *cred = (struct cred *) _cred;
+
+        atomic_add(n, &cred->subscribers);
+#endif
+}
+
 /*
  * Dispose of the shared task group credentials
  */
@@ -85,15 +126,29 @@ static void put_cred_rcu(struct rcu_head *rcu)
 {
         struct cred *cred = container_of(rcu, struct cred, rcu);
 
+        kdebug("put_cred_rcu(%p)", cred);
+
+#ifdef CONFIG_DEBUG_CREDENTIALS
+        if (cred->magic != CRED_MAGIC_DEAD ||
+            atomic_read(&cred->usage) != 0 ||
+            read_cred_subscribers(cred) != 0)
+                panic("CRED: put_cred_rcu() sees %p with"
+                      " mag %x, put %p, usage %d, subscr %d\n",
+                      cred, cred->magic, cred->put_addr,
+                      atomic_read(&cred->usage),
+                      read_cred_subscribers(cred));
+#else
         if (atomic_read(&cred->usage) != 0)
                 panic("CRED: put_cred_rcu() sees %p with usage %d\n",
                       cred, atomic_read(&cred->usage));
+#endif
 
         security_cred_free(cred);
         key_put(cred->thread_keyring);
         key_put(cred->request_key_auth);
         release_tgcred(cred);
-        put_group_info(cred->group_info);
+        if (cred->group_info)
+                put_group_info(cred->group_info);
         free_uid(cred->user);
         kmem_cache_free(cred_jar, cred);
 }
@@ -106,12 +161,90 @@ static void put_cred_rcu(struct rcu_head *rcu)
  */
 void __put_cred(struct cred *cred)
 {
+        kdebug("__put_cred(%p{%d,%d})", cred,
+               atomic_read(&cred->usage),
+               read_cred_subscribers(cred));
+
         BUG_ON(atomic_read(&cred->usage) != 0);
+#ifdef CONFIG_DEBUG_CREDENTIALS
+        BUG_ON(read_cred_subscribers(cred) != 0);
+        cred->magic = CRED_MAGIC_DEAD;
+        cred->put_addr = __builtin_return_address(0);
+#endif
+        BUG_ON(cred == current->cred);
+        BUG_ON(cred == current->real_cred);
 
         call_rcu(&cred->rcu, put_cred_rcu);
 }
 EXPORT_SYMBOL(__put_cred);
 
+/*
+ * Clean up a task's credentials when it exits
+ */
+void exit_creds(struct task_struct *tsk)
+{
+        struct cred *cred;
+
+        kdebug("exit_creds(%u,%p,%p,{%d,%d})", tsk->pid, tsk->real_cred, tsk->cred,
+               atomic_read(&tsk->cred->usage),
+               read_cred_subscribers(tsk->cred));
+
+        cred = (struct cred *) tsk->real_cred;
+        tsk->real_cred = NULL;
+        validate_creds(cred);
+        alter_cred_subscribers(cred, -1);
+        put_cred(cred);
+
+        cred = (struct cred *) tsk->cred;
+        tsk->cred = NULL;
+        validate_creds(cred);
+        alter_cred_subscribers(cred, -1);
+        put_cred(cred);
+
+        cred = (struct cred *) tsk->replacement_session_keyring;
+        if (cred) {
+                tsk->replacement_session_keyring = NULL;
+                validate_creds(cred);
+                put_cred(cred);
+        }
+}
+
+/*
+ * Allocate blank credentials, such that the credentials can be filled in at a
+ * later date without risk of ENOMEM.
+ */
+struct cred *cred_alloc_blank(void)
+{
+        struct cred *new;
+
+        new = kmem_cache_zalloc(cred_jar, GFP_KERNEL);
+        if (!new)
+                return NULL;
+
+#ifdef CONFIG_KEYS
+        new->tgcred = kzalloc(sizeof(*new->tgcred), GFP_KERNEL);
+        if (!new->tgcred) {
+                kfree(new);
+                return NULL;
+        }
+        atomic_set(&new->tgcred->usage, 1);
+#endif
+
+        atomic_set(&new->usage, 1);
+
+        if (security_cred_alloc_blank(new, GFP_KERNEL) < 0)
+                goto error;
+
+#ifdef CONFIG_DEBUG_CREDENTIALS
+        new->magic = CRED_MAGIC;
+#endif
+        return new;
+
+error:
+        abort_creds(new);
+        return NULL;
+}
+
 /**
  * prepare_creds - Prepare a new set of credentials for modification
  *
@@ -132,16 +265,19 @@ struct cred *prepare_creds(void)
         const struct cred *old;
         struct cred *new;
 
-        BUG_ON(atomic_read(&task->real_cred->usage) < 1);
+        validate_process_creds();
 
         new = kmem_cache_alloc(cred_jar, GFP_KERNEL);
         if (!new)
                 return NULL;
 
+        kdebug("prepare_creds() alloc %p", new);
+
         old = task->cred;
         memcpy(new, old, sizeof(struct cred));
 
         atomic_set(&new->usage, 1);
+        set_cred_subscribers(new, 0);
         get_group_info(new->group_info);
         get_uid(new->user);
 
@@ -157,6 +293,7 @@ struct cred *prepare_creds(void)
 
         if (security_prepare_creds(new, old, GFP_KERNEL) < 0)
                 goto error;
+        validate_creds(new);
         return new;
 
 error:
@@ -229,9 +366,12 @@ struct cred *prepare_usermodehelper_creds(void)
         if (!new)
                 return NULL;
 
+        kdebug("prepare_usermodehelper_creds() alloc %p", new);
+
         memcpy(new, &init_cred, sizeof(struct cred));
 
         atomic_set(&new->usage, 1);
+        set_cred_subscribers(new, 0);
         get_group_info(new->group_info);
         get_uid(new->user);
 
@@ -250,6 +390,7 @@ struct cred *prepare_usermodehelper_creds(void)
 #endif
         if (security_prepare_creds(new, &init_cred, GFP_ATOMIC) < 0)
                 goto error;
+        validate_creds(new);
 
         BUG_ON(atomic_read(&new->usage) != 1);
         return new;
@@ -286,6 +427,10 @@ int copy_creds(struct task_struct *p, unsigned long clone_flags)
             ) {
                 p->real_cred = get_cred(p->cred);
                 get_cred(p->cred);
+                alter_cred_subscribers(p->cred, 2);
+                kdebug("share_creds(%p{%d,%d})",
+                       p->cred, atomic_read(&p->cred->usage),
+                       read_cred_subscribers(p->cred));
                 atomic_inc(&p->cred->user->processes);
                 return 0;
         }
@@ -331,6 +476,8 @@ int copy_creds(struct task_struct *p, unsigned long clone_flags)
 
         atomic_inc(&new->user->processes);
         p->cred = p->real_cred = get_cred(new);
+        alter_cred_subscribers(new, 2);
+        validate_creds(new);
         return 0;
 
 error_put:
@@ -355,13 +502,20 @@ error_put:
 int commit_creds(struct cred *new)
 {
         struct task_struct *task = current;
-        const struct cred *old;
+        const struct cred *old = task->real_cred;
 
-        BUG_ON(task->cred != task->real_cred);
-        BUG_ON(atomic_read(&task->real_cred->usage) < 2);
+        kdebug("commit_creds(%p{%d,%d})", new,
+               atomic_read(&new->usage),
+               read_cred_subscribers(new));
+
+        BUG_ON(task->cred != old);
+#ifdef CONFIG_DEBUG_CREDENTIALS
+        BUG_ON(read_cred_subscribers(old) < 2);
+        validate_creds(old);
+        validate_creds(new);
+#endif
         BUG_ON(atomic_read(&new->usage) < 1);
 
-        old = task->real_cred;
         security_commit_creds(new, old);
 
         get_cred(new); /* we will require a ref for the subj creds too */
@@ -390,12 +544,14 @@ int commit_creds(struct cred *new)
          *   cheaply with the new uid cache, so if it matters
          *   we should be checking for it.  -DaveM
          */
+        alter_cred_subscribers(new, 2);
         if (new->user != old->user)
                 atomic_inc(&new->user->processes);
         rcu_assign_pointer(task->real_cred, new);
         rcu_assign_pointer(task->cred, new);
         if (new->user != old->user)
                 atomic_dec(&old->user->processes);
+        alter_cred_subscribers(old, -2);
 
         sched_switch_user(task);
 
@@ -428,6 +584,13 @@ EXPORT_SYMBOL(commit_creds);
  */
 void abort_creds(struct cred *new)
 {
+        kdebug("abort_creds(%p{%d,%d})", new,
+               atomic_read(&new->usage),
+               read_cred_subscribers(new));
+
+#ifdef CONFIG_DEBUG_CREDENTIALS
+        BUG_ON(read_cred_subscribers(new) != 0);
+#endif
         BUG_ON(atomic_read(&new->usage) < 1);
         put_cred(new);
 }
@@ -444,7 +607,20 @@ const struct cred *override_creds(const struct cred *new)
 {
         const struct cred *old = current->cred;
 
-        rcu_assign_pointer(current->cred, get_cred(new));
+        kdebug("override_creds(%p{%d,%d})", new,
+               atomic_read(&new->usage),
+               read_cred_subscribers(new));
+
+        validate_creds(old);
+        validate_creds(new);
+        get_cred(new);
+        alter_cred_subscribers(new, 1);
+        rcu_assign_pointer(current->cred, new);
+        alter_cred_subscribers(old, -1);
+
+        kdebug("override_creds() = %p{%d,%d}", old,
+               atomic_read(&old->usage),
+               read_cred_subscribers(old));
         return old;
 }
 EXPORT_SYMBOL(override_creds);
@@ -460,7 +636,15 @@ void revert_creds(const struct cred *old)
 {
         const struct cred *override = current->cred;
 
+        kdebug("revert_creds(%p{%d,%d})", old,
+               atomic_read(&old->usage),
+               read_cred_subscribers(old));
+
+        validate_creds(old);
+        validate_creds(override);
+        alter_cred_subscribers(old, 1);
         rcu_assign_pointer(current->cred, old);
+        alter_cred_subscribers(override, -1);
         put_cred(override);
 }
 EXPORT_SYMBOL(revert_creds);
@@ -502,11 +686,15 @@ struct cred *prepare_kernel_cred(struct task_struct *daemon)
         if (!new)
                 return NULL;
 
+        kdebug("prepare_kernel_cred() alloc %p", new);
+
         if (daemon)
                 old = get_task_cred(daemon);
         else
                 old = get_cred(&init_cred);
 
+        validate_creds(old);
+
         *new = *old;
         get_uid(new->user);
         get_group_info(new->group_info);
@@ -526,7 +714,9 @@ struct cred *prepare_kernel_cred(struct task_struct *daemon)
                 goto error;
 
         atomic_set(&new->usage, 1);
+        set_cred_subscribers(new, 0);
         put_cred(old);
+        validate_creds(new);
         return new;
 
 error:
@@ -589,3 +779,95 @@ int set_create_files_as(struct cred *new, struct inode *inode)
         return security_kernel_create_files_as(new, inode);
 }
 EXPORT_SYMBOL(set_create_files_as);
+
+#ifdef CONFIG_DEBUG_CREDENTIALS
+
+/*
+ * dump invalid credentials
+ */
+static void dump_invalid_creds(const struct cred *cred, const char *label,
+                               const struct task_struct *tsk)
+{
+        printk(KERN_ERR "CRED: %s credentials: %p %s%s%s\n",
+               label, cred,
+               cred == &init_cred ? "[init]" : "",
+               cred == tsk->real_cred ? "[real]" : "",
+               cred == tsk->cred ? "[eff]" : "");
+        printk(KERN_ERR "CRED: ->magic=%x, put_addr=%p\n",
+               cred->magic, cred->put_addr);
+        printk(KERN_ERR "CRED: ->usage=%d, subscr=%d\n",
+               atomic_read(&cred->usage),
+               read_cred_subscribers(cred));
+        printk(KERN_ERR "CRED: ->*uid = { %d,%d,%d,%d }\n",
+               cred->uid, cred->euid, cred->suid, cred->fsuid);
+        printk(KERN_ERR "CRED: ->*gid = { %d,%d,%d,%d }\n",
+               cred->gid, cred->egid, cred->sgid, cred->fsgid);
+#ifdef CONFIG_SECURITY
+        printk(KERN_ERR "CRED: ->security is %p\n", cred->security);
+        if ((unsigned long) cred->security >= PAGE_SIZE &&
+            (((unsigned long) cred->security & 0xffffff00) !=
+             (POISON_FREE << 24 | POISON_FREE << 16 | POISON_FREE << 8)))
+                printk(KERN_ERR "CRED: ->security {%x, %x}\n",
+                       ((u32*)cred->security)[0],
+                       ((u32*)cred->security)[1]);
+#endif
+}
+
+/*
+ * report use of invalid credentials
+ */
+void __invalid_creds(const struct cred *cred, const char *file, unsigned line)
+{
+        printk(KERN_ERR "CRED: Invalid credentials\n");
+        printk(KERN_ERR "CRED: At %s:%u\n", file, line);
+        dump_invalid_creds(cred, "Specified", current);
+        BUG();
+}
+EXPORT_SYMBOL(__invalid_creds);
+
+/*
+ * check the credentials on a process
+ */
+void __validate_process_creds(struct task_struct *tsk,
+                              const char *file, unsigned line)
+{
+        if (tsk->cred == tsk->real_cred) {
+                if (unlikely(read_cred_subscribers(tsk->cred) < 2 ||
+                             creds_are_invalid(tsk->cred)))
+                        goto invalid_creds;
+        } else {
+                if (unlikely(read_cred_subscribers(tsk->real_cred) < 1 ||
+                             read_cred_subscribers(tsk->cred) < 1 ||
+                             creds_are_invalid(tsk->real_cred) ||
+                             creds_are_invalid(tsk->cred)))
+                        goto invalid_creds;
+        }
+        return;
+
+invalid_creds:
+        printk(KERN_ERR "CRED: Invalid process credentials\n");
+        printk(KERN_ERR "CRED: At %s:%u\n", file, line);
+
+        dump_invalid_creds(tsk->real_cred, "Real", tsk);
+        if (tsk->cred != tsk->real_cred)
+                dump_invalid_creds(tsk->cred, "Effective", tsk);
+        else
+                printk(KERN_ERR "CRED: Effective creds == Real creds\n");
+        BUG();
+}
+EXPORT_SYMBOL(__validate_process_creds);
+
+/*
+ * check creds for do_exit()
+ */
+void validate_creds_for_do_exit(struct task_struct *tsk)
+{
+        kdebug("validate_creds_for_do_exit(%p,%p{%d,%d})",
+               tsk->real_cred, tsk->cred,
+               atomic_read(&tsk->cred->usage),
+               read_cred_subscribers(tsk->cred));
+
+        __validate_process_creds(tsk, __FILE__, __LINE__);
+}
+
+#endif /* CONFIG_DEBUG_CREDENTIALS */
diff --git a/kernel/dma-coherent.c b/kernel/dma-coherent.c
deleted file mode 100644
index 962a3b574f21..000000000000
--- a/kernel/dma-coherent.c
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Coherent per-device memory handling.
- * Borrowed from i386
- */
-#include <linux/kernel.h>
-#include <linux/dma-mapping.h>
-
-struct dma_coherent_mem {
-        void            *virt_base;
-        u32             device_base;
-        int             size;
-        int             flags;
-        unsigned long   *bitmap;
-};
-
-int dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr,
-                                dma_addr_t device_addr, size_t size, int flags)
-{
-        void __iomem *mem_base = NULL;
-        int pages = size >> PAGE_SHIFT;
-        int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long);
-
-        if ((flags & (DMA_MEMORY_MAP | DMA_MEMORY_IO)) == 0)
-                goto out;
-        if (!size)
-                goto out;
-        if (dev->dma_mem)
-                goto out;
-
-        /* FIXME: this routine just ignores DMA_MEMORY_INCLUDES_CHILDREN */
-
-        mem_base = ioremap(bus_addr, size);
-        if (!mem_base)
-                goto out;
-
-        dev->dma_mem = kzalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL);
-        if (!dev->dma_mem)
-                goto out;
-        dev->dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
-        if (!dev->dma_mem->bitmap)
-                goto free1_out;
-
-        dev->dma_mem->virt_base = mem_base;
-        dev->dma_mem->device_base = device_addr;
-        dev->dma_mem->size = pages;
-        dev->dma_mem->flags = flags;
-
-        if (flags & DMA_MEMORY_MAP)
-                return DMA_MEMORY_MAP;
-
-        return DMA_MEMORY_IO;
-
- free1_out:
-        kfree(dev->dma_mem);
- out:
-        if (mem_base)
-                iounmap(mem_base);
-        return 0;
-}
-EXPORT_SYMBOL(dma_declare_coherent_memory);
-
-void dma_release_declared_memory(struct device *dev)
-{
-        struct dma_coherent_mem *mem = dev->dma_mem;
-
-        if (!mem)
-                return;
-        dev->dma_mem = NULL;
-        iounmap(mem->virt_base);
-        kfree(mem->bitmap);
-        kfree(mem);
-}
-EXPORT_SYMBOL(dma_release_declared_memory);
-
-void *dma_mark_declared_memory_occupied(struct device *dev,
-                                        dma_addr_t device_addr, size_t size)
-{
-        struct dma_coherent_mem *mem = dev->dma_mem;
-        int pos, err;
-
-        size += device_addr & ~PAGE_MASK;
-
-        if (!mem)
-                return ERR_PTR(-EINVAL);
-
-        pos = (device_addr - mem->device_base) >> PAGE_SHIFT;
-        err = bitmap_allocate_region(mem->bitmap, pos, get_order(size));
-        if (err != 0)
-                return ERR_PTR(err);
-        return mem->virt_base + (pos << PAGE_SHIFT);
-}
-EXPORT_SYMBOL(dma_mark_declared_memory_occupied);
-
-/**
- * dma_alloc_from_coherent() - try to allocate memory from the per-device coherent area
- *
- * @dev:        device from which we allocate memory
- * @size:       size of requested memory area
- * @dma_handle: This will be filled with the correct dma handle
- * @ret:        This pointer will be filled with the virtual address
- *              to allocated area.
- *
- * This function should be only called from per-arch dma_alloc_coherent()
- * to support allocation from per-device coherent memory pools.
- *
- * Returns 0 if dma_alloc_coherent should continue with allocating from
- * generic memory areas, or !0 if dma_alloc_coherent should return @ret.
- */
-int dma_alloc_from_coherent(struct device *dev, ssize_t size,
-                                       dma_addr_t *dma_handle, void **ret)
-{
-        struct dma_coherent_mem *mem;
-        int order = get_order(size);
-        int pageno;
-
-        if (!dev)
-                return 0;
-        mem = dev->dma_mem;
-        if (!mem)
-                return 0;
-
-        *ret = NULL;
-
-        if (unlikely(size > (mem->size << PAGE_SHIFT)))
-                goto err;
-
-        pageno = bitmap_find_free_region(mem->bitmap, mem->size, order);
-        if (unlikely(pageno < 0))
-                goto err;
-
-        /*
-         * Memory was found in the per-device area.
-         */
-        *dma_handle = mem->device_base + (pageno << PAGE_SHIFT);
-        *ret = mem->virt_base + (pageno << PAGE_SHIFT);
-        memset(*ret, 0, size);
-
-        return 1;
-
-err:
-        /*
-         * In the case where the allocation can not be satisfied from the
-         * per-device area, try to fall back to generic memory if the
-         * constraints allow it.
-         */
-        return mem->flags & DMA_MEMORY_EXCLUSIVE;
-}
-EXPORT_SYMBOL(dma_alloc_from_coherent);
-
-/**
- * dma_release_from_coherent() - try to free the memory allocated from per-device coherent memory pool
- * @dev:        device from which the memory was allocated
- * @order:      the order of pages allocated
- * @vaddr:      virtual address of allocated pages
- *
- * This checks whether the memory was allocated from the per-device
- * coherent memory pool and if so, releases that memory.
- *
- * Returns 1 if we correctly released the memory, or 0 if
- * dma_release_coherent() should proceed with releasing memory from
- * generic pools.
- */
-int dma_release_from_coherent(struct device *dev, int order, void *vaddr)
-{
-        struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
-
-        if (mem && vaddr >= mem->virt_base && vaddr <
-                   (mem->virt_base + (mem->size << PAGE_SHIFT))) {
-                int page = (vaddr - mem->virt_base) >> PAGE_SHIFT;
-
-                bitmap_release_region(mem->bitmap, page, order);
-                return 1;
-        }
-        return 0;
-}
-EXPORT_SYMBOL(dma_release_from_coherent);
diff --git a/kernel/exit.c b/kernel/exit.c
index 869dc221733e..ae5d8660ddff 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -901,6 +901,8 @@ NORET_TYPE void do_exit(long code)
 
         tracehook_report_exit(&code);
 
+        validate_creds_for_do_exit(tsk);
+
         /*
          * We're taking recursive faults here in do_exit. Safest is to just
          * leave this task alone and wait for reboot.
@@ -1009,7 +1011,10 @@ NORET_TYPE void do_exit(long code)
         if (tsk->splice_pipe)
                 __free_pipe_info(tsk->splice_pipe);
 
+        validate_creds_for_do_exit(tsk);
+
         preempt_disable();
+        exit_rcu();
         /* causes final put_task_struct in finish_task_switch(). */
         tsk->state = TASK_DEAD;
         schedule();
diff --git a/kernel/fork.c b/kernel/fork.c
index e6c04d462ab2..bfee931ee3fb 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -152,8 +152,7 @@ void __put_task_struct(struct task_struct *tsk)
         WARN_ON(atomic_read(&tsk->usage));
         WARN_ON(tsk == current);
 
-        put_cred(tsk->real_cred);
-        put_cred(tsk->cred);
+        exit_creds(tsk);
         delayacct_tsk_free(tsk);
 
         if (!profile_handoff_task(tsk))
@@ -1008,10 +1007,7 @@ static struct task_struct *copy_process(unsigned long clone_flags,
         copy_flags(clone_flags, p);
         INIT_LIST_HEAD(&p->children);
         INIT_LIST_HEAD(&p->sibling);
-#ifdef CONFIG_PREEMPT_RCU
-        p->rcu_read_lock_nesting = 0;
-        p->rcu_flipctr_idx = 0;
-#endif /* #ifdef CONFIG_PREEMPT_RCU */
+        rcu_copy_process(p);
         p->vfork_done = NULL;
         spin_lock_init(&p->alloc_lock);
 
@@ -1297,8 +1293,7 @@ bad_fork_cleanup_put_domain:
         module_put(task_thread_info(p)->exec_domain->module);
 bad_fork_cleanup_count:
         atomic_dec(&p->cred->user->processes);
-        put_cred(p->real_cred);
-        put_cred(p->cred);
+        exit_creds(p);
 bad_fork_free:
         free_task(p);
 fork_out:
diff --git a/kernel/futex.c b/kernel/futex.c
index e18cfbdc7190..248dd119a86e 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -115,6 +115,9 @@ struct futex_q {
         /* rt_waiter storage for requeue_pi: */
         struct rt_mutex_waiter *rt_waiter;
 
+        /* The expected requeue pi target futex key: */
+        union futex_key *requeue_pi_key;
+
         /* Bitset for the optional bitmasked wakeup */
         u32 bitset;
 };
@@ -1089,6 +1092,10 @@ static int futex_proxy_trylock_atomic(u32 __user *pifutex,
         if (!top_waiter)
                 return 0;
 
+        /* Ensure we requeue to the expected futex. */
+        if (!match_futex(top_waiter->requeue_pi_key, key2))
+                return -EINVAL;
+
         /*
          * Try to take the lock for top_waiter.  Set the FUTEX_WAITERS bit in
          * the contended case or if set_waiters is 1.  The pi_state is returned
@@ -1276,6 +1283,12 @@ retry_private:
                         continue;
                 }
 
+                /* Ensure we requeue to the expected futex for requeue_pi. */
+                if (requeue_pi && !match_futex(this->requeue_pi_key, &key2)) {
+                        ret = -EINVAL;
+                        break;
+                }
+
                 /*
                  * Requeue nr_requeue waiters and possibly one more in the case
                  * of requeue_pi if we couldn't acquire the lock atomically.
@@ -1751,6 +1764,7 @@ static int futex_wait(u32 __user *uaddr, int fshared,
         q.pi_state = NULL;
         q.bitset = bitset;
         q.rt_waiter = NULL;
+        q.requeue_pi_key = NULL;
 
         if (abs_time) {
                 to = &timeout;
@@ -1858,6 +1872,7 @@ static int futex_lock_pi(u32 __user *uaddr, int fshared,
 
         q.pi_state = NULL;
         q.rt_waiter = NULL;
+        q.requeue_pi_key = NULL;
 retry:
         q.key = FUTEX_KEY_INIT;
         ret = get_futex_key(uaddr, fshared, &q.key, VERIFY_WRITE);
@@ -2118,11 +2133,11 @@ int handle_early_requeue_pi_wakeup(struct futex_hash_bucket *hb,
  * We call schedule in futex_wait_queue_me() when we enqueue and return there
  * via the following:
  * 1) wakeup on uaddr2 after an atomic lock acquisition by futex_requeue()
- * 2) wakeup on uaddr2 after a requeue and subsequent unlock
- * 3) signal (before or after requeue)
- * 4) timeout (before or after requeue)
+ * 2) wakeup on uaddr2 after a requeue
+ * 3) signal
+ * 4) timeout
  *
- * If 3, we setup a restart_block with futex_wait_requeue_pi() as the function.
+ * If 3, cleanup and return -ERESTARTNOINTR.
  *
  * If 2, we may then block on trying to take the rt_mutex and return via:
  * 5) successful lock
@@ -2130,7 +2145,7 @@ int handle_early_requeue_pi_wakeup(struct futex_hash_bucket *hb,
  * 7) timeout
  * 8) other lock acquisition failure
  *
- * If 6, we setup a restart_block with futex_lock_pi() as the function.
+ * If 6, return -EWOULDBLOCK (restarting the syscall would do the same).
  *
  * If 4 or 7, we cleanup and return with -ETIMEDOUT.
  *
@@ -2169,15 +2184,16 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, int fshared,
         debug_rt_mutex_init_waiter(&rt_waiter);
         rt_waiter.task = NULL;
 
-        q.pi_state = NULL;
-        q.bitset = bitset;
-        q.rt_waiter = &rt_waiter;
-
         key2 = FUTEX_KEY_INIT;
         ret = get_futex_key(uaddr2, fshared, &key2, VERIFY_WRITE);
         if (unlikely(ret != 0))
                 goto out;
 
+        q.pi_state = NULL;
+        q.bitset = bitset;
+        q.rt_waiter = &rt_waiter;
+        q.requeue_pi_key = &key2;
+
         /* Prepare to wait on uaddr. */
         ret = futex_wait_setup(uaddr, val, fshared, &q, &hb);
         if (ret)
@@ -2248,14 +2264,11 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, int fshared,
                         rt_mutex_unlock(pi_mutex);
         } else if (ret == -EINTR) {
                 /*
-                 * We've already been requeued, but we have no way to
-                 * restart by calling futex_lock_pi() directly. We
-                 * could restart the syscall, but that will look at
-                 * the user space value and return right away. So we
-                 * drop back with EWOULDBLOCK to tell user space that
-                 * "val" has been changed. That's the same what the
-                 * restart of the syscall would do in
-                 * futex_wait_setup().
+                 * We've already been requeued, but cannot restart by calling
+                 * futex_lock_pi() directly. We could restart this syscall, but
+                 * it would detect that the user space "val" changed and return
+                 * -EWOULDBLOCK.  Save the overhead of the restart and return
+                 * -EWOULDBLOCK directly.
                  */
                 ret = -EWOULDBLOCK;
         }
diff --git a/kernel/gcov/Kconfig b/kernel/gcov/Kconfig
index 22e9dcfaa3d3..654efd09f6a9 100644
--- a/kernel/gcov/Kconfig
+++ b/kernel/gcov/Kconfig
@@ -34,7 +34,7 @@ config GCOV_KERNEL
 config GCOV_PROFILE_ALL
         bool "Profile entire Kernel"
         depends on GCOV_KERNEL
-        depends on S390 || X86
+        depends on S390 || X86 || (PPC && EXPERIMENTAL)
         default n
         ---help---
         This options activates profiling for the entire kernel.
diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c
index 49da79ab8486..05071bf6a37b 100644
--- a/kernel/hrtimer.c
+++ b/kernel/hrtimer.c
@@ -485,6 +485,7 @@ void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t clock_id,
         debug_object_init_on_stack(timer, &hrtimer_debug_descr);
         __hrtimer_init(timer, clock_id, mode);
 }
+EXPORT_SYMBOL_GPL(hrtimer_init_on_stack);
 
 void destroy_hrtimer_on_stack(struct hrtimer *timer)
 {
@@ -1477,6 +1478,7 @@ void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task)
         sl->timer.function = hrtimer_wakeup;
         sl->task = task;
 }
+EXPORT_SYMBOL_GPL(hrtimer_init_sleeper);
 
 static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode)
 {
diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c
index 13c68e71b726..c1660194d115 100644
--- a/kernel/irq/chip.c
+++ b/kernel/irq/chip.c
@@ -222,6 +222,34 @@ int set_irq_chip_data(unsigned int irq, void *data)
 }
 EXPORT_SYMBOL(set_irq_chip_data);
 
+/**
+ *      set_irq_nested_thread - Set/Reset the IRQ_NESTED_THREAD flag of an irq
+ *
+ *      @irq:   Interrupt number
+ *      @nest:  0 to clear / 1 to set the IRQ_NESTED_THREAD flag
+ *
+ *      The IRQ_NESTED_THREAD flag indicates that on
+ *      request_threaded_irq() no separate interrupt thread should be
+ *      created for the irq as the handler are called nested in the
+ *      context of a demultiplexing interrupt handler thread.
+ */
+void set_irq_nested_thread(unsigned int irq, int nest)
+{
+        struct irq_desc *desc = irq_to_desc(irq);
+        unsigned long flags;
+
+        if (!desc)
+                return;
+
+        spin_lock_irqsave(&desc->lock, flags);
+        if (nest)
+                desc->status |= IRQ_NESTED_THREAD;
+        else
+                desc->status &= ~IRQ_NESTED_THREAD;
+        spin_unlock_irqrestore(&desc->lock, flags);
+}
+EXPORT_SYMBOL_GPL(set_irq_nested_thread);
+
 /*
  * default enable function
  */
@@ -299,6 +327,45 @@ static inline void mask_ack_irq(struct irq_desc *desc, int irq)
         }
 }
 
+/*
+ *      handle_nested_irq - Handle a nested irq from a irq thread
+ *      @irq:   the interrupt number
+ *
+ *      Handle interrupts which are nested into a threaded interrupt
+ *      handler. The handler function is called inside the calling
+ *      threads context.
+ */
+void handle_nested_irq(unsigned int irq)
+{
+        struct irq_desc *desc = irq_to_desc(irq);
+        struct irqaction *action;
+        irqreturn_t action_ret;
+
+        might_sleep();
+
+        spin_lock_irq(&desc->lock);
+
+        kstat_incr_irqs_this_cpu(irq, desc);
+
+        action = desc->action;
+        if (unlikely(!action || (desc->status & IRQ_DISABLED)))
+                goto out_unlock;
+
+        desc->status |= IRQ_INPROGRESS;
+        spin_unlock_irq(&desc->lock);
+
+        action_ret = action->thread_fn(action->irq, action->dev_id);
+        if (!noirqdebug)
+                note_interrupt(irq, desc, action_ret);
+
+        spin_lock_irq(&desc->lock);
+        desc->status &= ~IRQ_INPROGRESS;
+
+out_unlock:
+        spin_unlock_irq(&desc->lock);
+}
+EXPORT_SYMBOL_GPL(handle_nested_irq);
+
 /**
  *      handle_simple_irq - Simple and software-decoded IRQs.
  *      @irq:   the interrupt number
@@ -382,7 +449,10 @@ handle_level_irq(unsigned int irq, struct irq_desc *desc)
 
         spin_lock(&desc->lock);
         desc->status &= ~IRQ_INPROGRESS;
-        if (!(desc->status & IRQ_DISABLED) && desc->chip->unmask)
+
+        if (unlikely(desc->status & IRQ_ONESHOT))
+                desc->status |= IRQ_MASKED;
+        else if (!(desc->status & IRQ_DISABLED) && desc->chip->unmask)
                 desc->chip->unmask(irq);
 out_unlock:
         spin_unlock(&desc->lock);
@@ -572,6 +642,7 @@ __set_irq_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
                 desc->chip = &dummy_irq_chip;
         }
 
+        chip_bus_lock(irq, desc);
         spin_lock_irqsave(&desc->lock, flags);
 
         /* Uninstall? */
@@ -591,6 +662,7 @@ __set_irq_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
                 desc->chip->startup(irq);
         }
         spin_unlock_irqrestore(&desc->lock, flags);
+        chip_bus_sync_unlock(irq, desc);
 }
 EXPORT_SYMBOL_GPL(__set_irq_handler);
 
diff --git a/kernel/irq/handle.c b/kernel/irq/handle.c
index 065205bdd920..a81cf80554db 100644
--- a/kernel/irq/handle.c
+++ b/kernel/irq/handle.c
@@ -161,7 +161,7 @@ int __init early_irq_init(void)
 
         desc = irq_desc_legacy;
         legacy_count = ARRAY_SIZE(irq_desc_legacy);
-        node = first_online_node;
+        node = first_online_node;
 
         /* allocate irq_desc_ptrs array based on nr_irqs */
         irq_desc_ptrs = kcalloc(nr_irqs, sizeof(void *), GFP_NOWAIT);
@@ -172,6 +172,9 @@ int __init early_irq_init(void)
 
         for (i = 0; i < legacy_count; i++) {
                 desc[i].irq = i;
+#ifdef CONFIG_SMP
+                desc[i].node = node;
+#endif
                 desc[i].kstat_irqs = kstat_irqs_legacy + i * nr_cpu_ids;
                 lockdep_set_class(&desc[i].lock, &irq_desc_lock_class);
                 alloc_desc_masks(&desc[i], node, true);
diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h
index e70ed5592eb9..1b5d742c6a77 100644
--- a/kernel/irq/internals.h
+++ b/kernel/irq/internals.h
@@ -44,6 +44,19 @@ extern int irq_select_affinity_usr(unsigned int irq);
 
 extern void irq_set_thread_affinity(struct irq_desc *desc);
 
+/* Inline functions for support of irq chips on slow busses */
+static inline void chip_bus_lock(unsigned int irq, struct irq_desc *desc)
+{
+        if (unlikely(desc->chip->bus_lock))
+                desc->chip->bus_lock(irq);
+}
+
+static inline void chip_bus_sync_unlock(unsigned int irq, struct irq_desc *desc)
+{
+        if (unlikely(desc->chip->bus_sync_unlock))
+                desc->chip->bus_sync_unlock(irq);
+}
+
 /*
  * Debugging printout:
  */
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c
index 0ec9ed831737..bde4c667d24d 100644
--- a/kernel/irq/manage.c
+++ b/kernel/irq/manage.c
@@ -230,9 +230,11 @@ void disable_irq_nosync(unsigned int irq)
         if (!desc)
                 return;
 
+        chip_bus_lock(irq, desc);
         spin_lock_irqsave(&desc->lock, flags);
         __disable_irq(desc, irq, false);
         spin_unlock_irqrestore(&desc->lock, flags);
+        chip_bus_sync_unlock(irq, desc);
 }
 EXPORT_SYMBOL(disable_irq_nosync);
 
@@ -294,7 +296,8 @@ void __enable_irq(struct irq_desc *desc, unsigned int irq, bool resume)
  *      matches the last disable, processing of interrupts on this
  *      IRQ line is re-enabled.
  *
- *      This function may be called from IRQ context.
+ *      This function may be called from IRQ context only when
+ *      desc->chip->bus_lock and desc->chip->bus_sync_unlock are NULL !
  */
 void enable_irq(unsigned int irq)
 {
@@ -304,9 +307,11 @@ void enable_irq(unsigned int irq)
         if (!desc)
                 return;
 
+        chip_bus_lock(irq, desc);
         spin_lock_irqsave(&desc->lock, flags);
         __enable_irq(desc, irq, false);
         spin_unlock_irqrestore(&desc->lock, flags);
+        chip_bus_sync_unlock(irq, desc);
 }
 EXPORT_SYMBOL(enable_irq);
 
@@ -436,6 +441,26 @@ int __irq_set_trigger(struct irq_desc *desc, unsigned int irq,
         return ret;
 }
 
+/*
+ * Default primary interrupt handler for threaded interrupts. Is
+ * assigned as primary handler when request_threaded_irq is called
+ * with handler == NULL. Useful for oneshot interrupts.
+ */
+static irqreturn_t irq_default_primary_handler(int irq, void *dev_id)
+{
+        return IRQ_WAKE_THREAD;
+}
+
+/*
+ * Primary handler for nested threaded interrupts. Should never be
+ * called.
+ */
+static irqreturn_t irq_nested_primary_handler(int irq, void *dev_id)
+{
+        WARN(1, "Primary handler called for nested irq %d\n", irq);
+        return IRQ_NONE;
+}
+
 static int irq_wait_for_interrupt(struct irqaction *action)
 {
         while (!kthread_should_stop()) {
@@ -451,6 +476,23 @@ static int irq_wait_for_interrupt(struct irqaction *action)
         return -1;
 }
 
+/*
+ * Oneshot interrupts keep the irq line masked until the threaded
+ * handler finished. unmask if the interrupt has not been disabled and
+ * is marked MASKED.
+ */
+static void irq_finalize_oneshot(unsigned int irq, struct irq_desc *desc)
+{
+        chip_bus_lock(irq, desc);
+        spin_lock_irq(&desc->lock);
+        if (!(desc->status & IRQ_DISABLED) && (desc->status & IRQ_MASKED)) {
+                desc->status &= ~IRQ_MASKED;
+                desc->chip->unmask(irq);
+        }
+        spin_unlock_irq(&desc->lock);
+        chip_bus_sync_unlock(irq, desc);
+}
+
 #ifdef CONFIG_SMP
 /*
  * Check whether we need to change the affinity of the interrupt thread.
@@ -492,7 +534,7 @@ static int irq_thread(void *data)
         struct sched_param param = { .sched_priority = MAX_USER_RT_PRIO/2, };
         struct irqaction *action = data;
         struct irq_desc *desc = irq_to_desc(action->irq);
-        int wake;
+        int wake, oneshot = desc->status & IRQ_ONESHOT;
 
         sched_setscheduler(current, SCHED_FIFO, &param);
         current->irqaction = action;
@@ -518,6 +560,9 @@ static int irq_thread(void *data)
                         spin_unlock_irq(&desc->lock);
 
                         action->thread_fn(action->irq, action->dev_id);
+
+                        if (oneshot)
+                                irq_finalize_oneshot(action->irq, desc);
                 }
 
                 wake = atomic_dec_and_test(&desc->threads_active);
@@ -565,7 +610,7 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
         struct irqaction *old, **old_ptr;
         const char *old_name = NULL;
         unsigned long flags;
-        int shared = 0;
+        int nested, shared = 0;
         int ret;
 
         if (!desc)
@@ -590,10 +635,32 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
                 rand_initialize_irq(irq);
         }
 
+        /* Oneshot interrupts are not allowed with shared */
+        if ((new->flags & IRQF_ONESHOT) && (new->flags & IRQF_SHARED))
+                return -EINVAL;
+
+        /*
+         * Check whether the interrupt nests into another interrupt
+         * thread.
+         */
+        nested = desc->status & IRQ_NESTED_THREAD;
+        if (nested) {
+                if (!new->thread_fn)
+                        return -EINVAL;
+                /*
+                 * Replace the primary handler which was provided from
+                 * the driver for non nested interrupt handling by the
+                 * dummy function which warns when called.
+                 */
+                new->handler = irq_nested_primary_handler;
+        }
+
         /*
-         * Threaded handler ?
+         * Create a handler thread when a thread function is supplied
+         * and the interrupt does not nest into another interrupt
+         * thread.
          */
-        if (new->thread_fn) {
+        if (new->thread_fn && !nested) {
                 struct task_struct *t;
 
                 t = kthread_create(irq_thread, new, "irq/%d-%s", irq,
@@ -662,9 +729,12 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
                         desc->status |= IRQ_PER_CPU;
 #endif
 
-                desc->status &= ~(IRQ_AUTODETECT | IRQ_WAITING |
+                desc->status &= ~(IRQ_AUTODETECT | IRQ_WAITING | IRQ_ONESHOT |
                                   IRQ_INPROGRESS | IRQ_SPURIOUS_DISABLED);
 
+                if (new->flags & IRQF_ONESHOT)
+                        desc->status |= IRQ_ONESHOT;
+
                 if (!(desc->status & IRQ_NOAUTOEN)) {
                         desc->depth = 0;
                         desc->status &= ~IRQ_DISABLED;
@@ -875,7 +945,14 @@ EXPORT_SYMBOL_GPL(remove_irq);
  */
 void free_irq(unsigned int irq, void *dev_id)
 {
+        struct irq_desc *desc = irq_to_desc(irq);
+
+        if (!desc)
+                return;
+
+        chip_bus_lock(irq, desc);
         kfree(__free_irq(irq, dev_id));
+        chip_bus_sync_unlock(irq, desc);
 }
 EXPORT_SYMBOL(free_irq);
 
@@ -884,6 +961,8 @@ EXPORT_SYMBOL(free_irq);
  *      @irq: Interrupt line to allocate
  *      @handler: Function to be called when the IRQ occurs.
  *                Primary handler for threaded interrupts
+ *                If NULL and thread_fn != NULL the default
+ *                primary handler is installed
  *      @thread_fn: Function called from the irq handler thread
  *                  If NULL, no irq thread is created
  *      @irqflags: Interrupt type flags
@@ -963,8 +1042,12 @@ int request_threaded_irq(unsigned int irq, irq_handler_t handler,
 
         if (desc->status & IRQ_NOREQUEST)
                 return -EINVAL;
-        if (!handler)
-                return -EINVAL;
+
+        if (!handler) {
+                if (!thread_fn)
+                        return -EINVAL;
+                handler = irq_default_primary_handler;
+        }
 
         action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
         if (!action)
@@ -976,7 +1059,10 @@ int request_threaded_irq(unsigned int irq, irq_handler_t handler,
         action->name = devname;
         action->dev_id = dev_id;
 
+        chip_bus_lock(irq, desc);
         retval = __setup_irq(irq, desc, action);
+        chip_bus_sync_unlock(irq, desc);
+
         if (retval)
                 kfree(action);
 
diff --git a/kernel/irq/pm.c b/kernel/irq/pm.c
index 638d8bedec14..a0bb09e79867 100644
--- a/kernel/irq/pm.c
+++ b/kernel/irq/pm.c
@@ -15,10 +15,10 @@
 /**
  * suspend_device_irqs - disable all currently enabled interrupt lines
  *
- * During system-wide suspend or hibernation device interrupts need to be
- * disabled at the chip level and this function is provided for this purpose.
- * It disables all interrupt lines that are enabled at the moment and sets the
- * IRQ_SUSPENDED flag for them.
+ * During system-wide suspend or hibernation device drivers need to be prevented
+ * from receiving interrupts and this function is provided for this purpose.
+ * It marks all interrupt lines in use, except for the timer ones, as disabled
+ * and sets the IRQ_SUSPENDED flag for each of them.
  */
 void suspend_device_irqs(void)
 {
diff --git a/kernel/irq/resend.c b/kernel/irq/resend.c
index 89c7117acf2b..090c3763f3a2 100644
--- a/kernel/irq/resend.c
+++ b/kernel/irq/resend.c
@@ -70,8 +70,7 @@ void check_irq_resend(struct irq_desc *desc, unsigned int irq)
         if ((status & (IRQ_LEVEL | IRQ_PENDING | IRQ_REPLAY)) == IRQ_PENDING) {
                 desc->status = (status & ~IRQ_PENDING) | IRQ_REPLAY;
 
-                if (!desc->chip || !desc->chip->retrigger ||
-                                        !desc->chip->retrigger(irq)) {
+                if (!desc->chip->retrigger || !desc->chip->retrigger(irq)) {
 #ifdef CONFIG_HARDIRQS_SW_RESEND
                         /* Set it pending and activate the softirq: */
                         set_bit(irq, irqs_resend);
diff --git a/kernel/irq/spurious.c b/kernel/irq/spurious.c
index 4d568294de3e..114e704760fe 100644
--- a/kernel/irq/spurious.c
+++ b/kernel/irq/spurious.c
@@ -297,7 +297,6 @@ static int __init irqfixup_setup(char *str)
 
 __setup("irqfixup", irqfixup_setup);
 module_param(irqfixup, int, 0644);
-MODULE_PARM_DESC("irqfixup", "0: No fixup, 1: irqfixup mode, 2: irqpoll mode");
 
 static int __init irqpoll_setup(char *str)
 {
diff --git a/kernel/kmod.c b/kernel/kmod.c
index a92280870e30..9fcb53a11f87 100644
--- a/kernel/kmod.c
+++ b/kernel/kmod.c
@@ -80,6 +80,10 @@ int __request_module(bool wait, const char *fmt, ...)
 #define MAX_KMOD_CONCURRENT 50  /* Completely arbitrary value - KAO */
         static int kmod_loop_msg;
 
+        ret = security_kernel_module_request();
+        if (ret)
+                return ret;
+
         va_start(args, fmt);
         ret = vsnprintf(module_name, MODULE_NAME_LEN, fmt, args);
         va_end(args);
@@ -466,6 +470,7 @@ int call_usermodehelper_exec(struct subprocess_info *sub_info,
         int retval = 0;
 
         BUG_ON(atomic_read(&sub_info->cred->usage) != 1);
+        validate_creds(sub_info->cred);
 
         helper_lock();
         if (sub_info->path[0] == '\0')
diff --git a/kernel/kthread.c b/kernel/kthread.c
index eb8751aa0418..5fe709982caa 100644
--- a/kernel/kthread.c
+++ b/kernel/kthread.c
@@ -16,8 +16,6 @@
 #include <linux/mutex.h>
 #include <trace/events/sched.h>
 
-#define KTHREAD_NICE_LEVEL (-5)
-
 static DEFINE_SPINLOCK(kthread_create_lock);
 static LIST_HEAD(kthread_create_list);
 struct task_struct *kthreadd_task;
@@ -145,7 +143,6 @@ struct task_struct *kthread_create(int (*threadfn)(void *data),
                  * The kernel thread should not inherit these properties.
                  */
                 sched_setscheduler_nocheck(create.result, SCHED_NORMAL, &param);
-                set_user_nice(create.result, KTHREAD_NICE_LEVEL);
                 set_cpus_allowed_ptr(create.result, cpu_all_mask);
         }
         return create.result;
@@ -221,7 +218,6 @@ int kthreadd(void *unused)
         /* Setup a clean context for our children to inherit. */
         set_task_comm(tsk, "kthreadd");
         ignore_signals(tsk);
-        set_user_nice(tsk, KTHREAD_NICE_LEVEL);
         set_cpus_allowed_ptr(tsk, cpu_all_mask);
         set_mems_allowed(node_possible_map);
 
diff --git a/kernel/lockdep.c b/kernel/lockdep.c
index 8bbeef996c76..f74d2d7aa605 100644
--- a/kernel/lockdep.c
+++ b/kernel/lockdep.c
@@ -42,6 +42,7 @@
 #include <linux/hash.h>
 #include <linux/ftrace.h>
 #include <linux/stringify.h>
+#include <linux/bitops.h>
 
 #include <asm/sections.h>
 
@@ -366,11 +367,21 @@ static int save_trace(struct stack_trace *trace)
 
         save_stack_trace(trace);
 
+        /*
+         * Some daft arches put -1 at the end to indicate its a full trace.
+         *
+         * <rant> this is buggy anyway, since it takes a whole extra entry so a
+         * complete trace that maxes out the entries provided will be reported
+         * as incomplete, friggin useless </rant>
+         */
+        if (trace->entries[trace->nr_entries-1] == ULONG_MAX)
+                trace->nr_entries--;
+
         trace->max_entries = trace->nr_entries;
 
         nr_stack_trace_entries += trace->nr_entries;
 
-        if (nr_stack_trace_entries == MAX_STACK_TRACE_ENTRIES) {
+        if (nr_stack_trace_entries >= MAX_STACK_TRACE_ENTRIES-1) {
                 if (!debug_locks_off_graph_unlock())
                         return 0;
 
@@ -388,20 +399,6 @@ unsigned int nr_hardirq_chains;
 unsigned int nr_softirq_chains;
 unsigned int nr_process_chains;
 unsigned int max_lockdep_depth;
-unsigned int max_recursion_depth;
-
-static unsigned int lockdep_dependency_gen_id;
-
-static bool lockdep_dependency_visit(struct lock_class *source,
-                                     unsigned int depth)
-{
-        if (!depth)
-                lockdep_dependency_gen_id++;
-        if (source->dep_gen_id == lockdep_dependency_gen_id)
-                return true;
-        source->dep_gen_id = lockdep_dependency_gen_id;
-        return false;
-}
 
 #ifdef CONFIG_DEBUG_LOCKDEP
 /*
@@ -431,11 +428,8 @@ atomic_t redundant_softirqs_on;
 atomic_t redundant_softirqs_off;
 atomic_t nr_unused_locks;
 atomic_t nr_cyclic_checks;
-atomic_t nr_cyclic_check_recursions;
 atomic_t nr_find_usage_forwards_checks;
-atomic_t nr_find_usage_forwards_recursions;
 atomic_t nr_find_usage_backwards_checks;
-atomic_t nr_find_usage_backwards_recursions;
 #endif
 
 /*
@@ -551,58 +545,6 @@ static void lockdep_print_held_locks(struct task_struct *curr)
         }
 }
 
-static void print_lock_class_header(struct lock_class *class, int depth)
-{
-        int bit;
-
-        printk("%*s->", depth, "");
-        print_lock_name(class);
-        printk(" ops: %lu", class->ops);
-        printk(" {\n");
-
-        for (bit = 0; bit < LOCK_USAGE_STATES; bit++) {
-                if (class->usage_mask & (1 << bit)) {
-                        int len = depth;
-
-                        len += printk("%*s   %s", depth, "", usage_str[bit]);
-                        len += printk(" at:\n");
-                        print_stack_trace(class->usage_traces + bit, len);
-                }
-        }
-        printk("%*s }\n", depth, "");
-
-        printk("%*s ... key      at: ",depth,"");
-        print_ip_sym((unsigned long)class->key);
-}
-
-/*
- * printk all lock dependencies starting at <entry>:
- */
-static void __used
-print_lock_dependencies(struct lock_class *class, int depth)
-{
-        struct lock_list *entry;
-
-        if (lockdep_dependency_visit(class, depth))
-                return;
-
-        if (DEBUG_LOCKS_WARN_ON(depth >= 20))
-                return;
-
-        print_lock_class_header(class, depth);
-
-        list_for_each_entry(entry, &class->locks_after, entry) {
-                if (DEBUG_LOCKS_WARN_ON(!entry->class))
-                        return;
-
-                print_lock_dependencies(entry->class, depth + 1);
-
-                printk("%*s ... acquired at:\n",depth,"");
-                print_stack_trace(&entry->trace, 2);
-                printk("\n");
-        }
-}
-
 static void print_kernel_version(void)
 {
         printk("%s %.*s\n", init_utsname()->release,
@@ -898,22 +840,203 @@ static int add_lock_to_list(struct lock_class *class, struct lock_class *this,
 }
 
 /*
+ * For good efficiency of modular, we use power of 2
+ */
+#define MAX_CIRCULAR_QUEUE_SIZE         4096UL
+#define CQ_MASK                         (MAX_CIRCULAR_QUEUE_SIZE-1)
+
+/*
+ * The circular_queue and helpers is used to implement the
+ * breadth-first search(BFS)algorithem, by which we can build
+ * the shortest path from the next lock to be acquired to the
+ * previous held lock if there is a circular between them.
+ */
+struct circular_queue {
+        unsigned long element[MAX_CIRCULAR_QUEUE_SIZE];
+        unsigned int  front, rear;
+};
+
+static struct circular_queue lock_cq;
+
+unsigned int max_bfs_queue_depth;
+
+static unsigned int lockdep_dependency_gen_id;
+
+static inline void __cq_init(struct circular_queue *cq)
+{
+        cq->front = cq->rear = 0;
+        lockdep_dependency_gen_id++;
+}
+
+static inline int __cq_empty(struct circular_queue *cq)
+{
+        return (cq->front == cq->rear);
+}
+
+static inline int __cq_full(struct circular_queue *cq)
+{
+        return ((cq->rear + 1) & CQ_MASK) == cq->front;
+}
+
+static inline int __cq_enqueue(struct circular_queue *cq, unsigned long elem)
+{
+        if (__cq_full(cq))
+                return -1;
+
+        cq->element[cq->rear] = elem;
+        cq->rear = (cq->rear + 1) & CQ_MASK;
+        return 0;
+}
+
+static inline int __cq_dequeue(struct circular_queue *cq, unsigned long *elem)
+{
+        if (__cq_empty(cq))
+                return -1;
+
+        *elem = cq->element[cq->front];
+        cq->front = (cq->front + 1) & CQ_MASK;
+        return 0;
+}
+
+static inline unsigned int  __cq_get_elem_count(struct circular_queue *cq)
+{
+        return (cq->rear - cq->front) & CQ_MASK;
+}
+
+static inline void mark_lock_accessed(struct lock_list *lock,
+                                        struct lock_list *parent)
+{
+        unsigned long nr;
+
+        nr = lock - list_entries;
+        WARN_ON(nr >= nr_list_entries);
+        lock->parent = parent;
+        lock->class->dep_gen_id = lockdep_dependency_gen_id;
+}
+
+static inline unsigned long lock_accessed(struct lock_list *lock)
+{
+        unsigned long nr;
+
+        nr = lock - list_entries;
+        WARN_ON(nr >= nr_list_entries);
+        return lock->class->dep_gen_id == lockdep_dependency_gen_id;
+}
+
+static inline struct lock_list *get_lock_parent(struct lock_list *child)
+{
+        return child->parent;
+}
+
+static inline int get_lock_depth(struct lock_list *child)
+{
+        int depth = 0;
+        struct lock_list *parent;
+
+        while ((parent = get_lock_parent(child))) {
+                child = parent;
+                depth++;
+        }
+        return depth;
+}
+
+static int __bfs(struct lock_list *source_entry,
+                 void *data,
+                 int (*match)(struct lock_list *entry, void *data),
+                 struct lock_list **target_entry,
+                 int forward)
+{
+        struct lock_list *entry;
+        struct list_head *head;
+        struct circular_queue *cq = &lock_cq;
+        int ret = 1;
+
+        if (match(source_entry, data)) {
+                *target_entry = source_entry;
+                ret = 0;
+                goto exit;
+        }
+
+        if (forward)
+                head = &source_entry->class->locks_after;
+        else
+                head = &source_entry->class->locks_before;
+
+        if (list_empty(head))
+                goto exit;
+
+        __cq_init(cq);
+        __cq_enqueue(cq, (unsigned long)source_entry);
+
+        while (!__cq_empty(cq)) {
+                struct lock_list *lock;
+
+                __cq_dequeue(cq, (unsigned long *)&lock);
+
+                if (!lock->class) {
+                        ret = -2;
+                        goto exit;
+                }
+
+                if (forward)
+                        head = &lock->class->locks_after;
+                else
+                        head = &lock->class->locks_before;
+
+                list_for_each_entry(entry, head, entry) {
+                        if (!lock_accessed(entry)) {
+                                unsigned int cq_depth;
+                                mark_lock_accessed(entry, lock);
+                                if (match(entry, data)) {
+                                        *target_entry = entry;
+                                        ret = 0;
+                                        goto exit;
+                                }
+
+                                if (__cq_enqueue(cq, (unsigned long)entry)) {
+                                        ret = -1;
+                                        goto exit;
+                                }
+                                cq_depth = __cq_get_elem_count(cq);
+                                if (max_bfs_queue_depth < cq_depth)
+                                        max_bfs_queue_depth = cq_depth;
+                        }
+                }
+        }
+exit:
+        return ret;
+}
+
+static inline int __bfs_forwards(struct lock_list *src_entry,
+                        void *data,
+                        int (*match)(struct lock_list *entry, void *data),
+                        struct lock_list **target_entry)
+{
+        return __bfs(src_entry, data, match, target_entry, 1);
+
+}
+
+static inline int __bfs_backwards(struct lock_list *src_entry,
+                        void *data,
+                        int (*match)(struct lock_list *entry, void *data),
+                        struct lock_list **target_entry)
+{
+        return __bfs(src_entry, data, match, target_entry, 0);
+
+}
+
+/*
  * Recursive, forwards-direction lock-dependency checking, used for
  * both noncyclic checking and for hardirq-unsafe/softirq-unsafe
  * checking.
- *
- * (to keep the stackframe of the recursive functions small we
- *  use these global variables, and we also mark various helper
- *  functions as noinline.)
  */
-static struct held_lock *check_source, *check_target;
 
 /*
  * Print a dependency chain entry (this is only done when a deadlock
  * has been detected):
  */
 static noinline int
-print_circular_bug_entry(struct lock_list *target, unsigned int depth)
+print_circular_bug_entry(struct lock_list *target, int depth)
 {
         if (debug_locks_silent)
                 return 0;
@@ -930,11 +1053,13 @@ print_circular_bug_entry(struct lock_list *target, unsigned int depth)
  * header first:
  */
 static noinline int
-print_circular_bug_header(struct lock_list *entry, unsigned int depth)
+print_circular_bug_header(struct lock_list *entry, unsigned int depth,
+                        struct held_lock *check_src,
+                        struct held_lock *check_tgt)
 {
         struct task_struct *curr = current;
 
-        if (!debug_locks_off_graph_unlock() || debug_locks_silent)
+        if (debug_locks_silent)
                 return 0;
 
         printk("\n=======================================================\n");
@@ -943,9 +1068,9 @@ print_circular_bug_header(struct lock_list *entry, unsigned int depth)
         printk(  "-------------------------------------------------------\n");
         printk("%s/%d is trying to acquire lock:\n",
                 curr->comm, task_pid_nr(curr));
-        print_lock(check_source);
+        print_lock(check_src);
         printk("\nbut task is already holding lock:\n");
-        print_lock(check_target);
+        print_lock(check_tgt);
         printk("\nwhich lock already depends on the new lock.\n\n");
         printk("\nthe existing dependency chain (in reverse order) is:\n");
 
@@ -954,19 +1079,36 @@ print_circular_bug_header(struct lock_list *entry, unsigned int depth)
         return 0;
 }
 
-static noinline int print_circular_bug_tail(void)
+static inline int class_equal(struct lock_list *entry, void *data)
+{
+        return entry->class == data;
+}
+
+static noinline int print_circular_bug(struct lock_list *this,
+                                struct lock_list *target,
+                                struct held_lock *check_src,
+                                struct held_lock *check_tgt)
 {
         struct task_struct *curr = current;
-        struct lock_list this;
+        struct lock_list *parent;
+        int depth;
 
-        if (debug_locks_silent)
+        if (!debug_locks_off_graph_unlock() || debug_locks_silent)
                 return 0;
 
-        this.class = hlock_class(check_source);
-        if (!save_trace(&this.trace))
+        if (!save_trace(&this->trace))
                 return 0;
 
-        print_circular_bug_entry(&this, 0);
+        depth = get_lock_depth(target);
+
+        print_circular_bug_header(target, depth, check_src, check_tgt);
+
+        parent = get_lock_parent(target);
+
+        while (parent) {
+                print_circular_bug_entry(parent, --depth);
+                parent = get_lock_parent(parent);
+        }
 
         printk("\nother info that might help us debug this:\n\n");
         lockdep_print_held_locks(curr);
@@ -977,73 +1119,69 @@ static noinline int print_circular_bug_tail(void)
         return 0;
 }
 
-#define RECURSION_LIMIT 40
-
-static int noinline print_infinite_recursion_bug(void)
+static noinline int print_bfs_bug(int ret)
 {
         if (!debug_locks_off_graph_unlock())
                 return 0;
 
-        WARN_ON(1);
+        WARN(1, "lockdep bfs error:%d\n", ret);
 
         return 0;
 }
 
-unsigned long __lockdep_count_forward_deps(struct lock_class *class,
-                                           unsigned int depth)
+static int noop_count(struct lock_list *entry, void *data)
 {
-        struct lock_list *entry;
-        unsigned long ret = 1;
+        (*(unsigned long *)data)++;
+        return 0;
+}
 
-        if (lockdep_dependency_visit(class, depth))
-                return 0;
+unsigned long __lockdep_count_forward_deps(struct lock_list *this)
+{
+        unsigned long  count = 0;
+        struct lock_list *uninitialized_var(target_entry);
 
-        /*
-         * Recurse this class's dependency list:
-         */
-        list_for_each_entry(entry, &class->locks_after, entry)
-                ret += __lockdep_count_forward_deps(entry->class, depth + 1);
+        __bfs_forwards(this, (void *)&count, noop_count, &target_entry);
 
-        return ret;
+        return count;
 }
-
 unsigned long lockdep_count_forward_deps(struct lock_class *class)
 {
         unsigned long ret, flags;
+        struct lock_list this;
+
+        this.parent = NULL;
+        this.class = class;
 
         local_irq_save(flags);
         __raw_spin_lock(&lockdep_lock);
-        ret = __lockdep_count_forward_deps(class, 0);
+        ret = __lockdep_count_forward_deps(&this);
         __raw_spin_unlock(&lockdep_lock);
         local_irq_restore(flags);
 
         return ret;
 }
 
-unsigned long __lockdep_count_backward_deps(struct lock_class *class,
-                                            unsigned int depth)
+unsigned long __lockdep_count_backward_deps(struct lock_list *this)
 {
-        struct lock_list *entry;
-        unsigned long ret = 1;
+        unsigned long  count = 0;
+        struct lock_list *uninitialized_var(target_entry);
 
-        if (lockdep_dependency_visit(class, depth))
-                return 0;
-        /*
-         * Recurse this class's dependency list:
-         */
-        list_for_each_entry(entry, &class->locks_before, entry)
-                ret += __lockdep_count_backward_deps(entry->class, depth + 1);
+        __bfs_backwards(this, (void *)&count, noop_count, &target_entry);
 
-        return ret;
+        return count;
 }
 
 unsigned long lockdep_count_backward_deps(struct lock_class *class)
 {
         unsigned long ret, flags;
+        struct lock_list this;
+
+        this.parent = NULL;
+        this.class = class;
 
         local_irq_save(flags);
         __raw_spin_lock(&lockdep_lock);
-        ret = __lockdep_count_backward_deps(class, 0);
+        ret = __lockdep_count_backward_deps(&this);
         __raw_spin_unlock(&lockdep_lock);
         local_irq_restore(flags);
 
@@ -1055,29 +1193,16 @@ unsigned long lockdep_count_backward_deps(struct lock_class *class)
  * lead to <target>. Print an error and return 0 if it does.
  */
 static noinline int
-check_noncircular(struct lock_class *source, unsigned int depth)
+check_noncircular(struct lock_list *root, struct lock_class *target,
+                struct lock_list **target_entry)
 {
-        struct lock_list *entry;
+        int result;
 
-        if (lockdep_dependency_visit(source, depth))
-                return 1;
+        debug_atomic_inc(&nr_cyclic_checks);
 
-        debug_atomic_inc(&nr_cyclic_check_recursions);
-        if (depth > max_recursion_depth)
-                max_recursion_depth = depth;
-        if (depth >= RECURSION_LIMIT)
-                return print_infinite_recursion_bug();
-        /*
-         * Check this lock's dependency list:
-         */
-        list_for_each_entry(entry, &source->locks_after, entry) {
-                if (entry->class == hlock_class(check_target))
-                        return print_circular_bug_header(entry, depth+1);
-                debug_atomic_inc(&nr_cyclic_checks);
-                if (!check_noncircular(entry->class, depth+1))
-                        return print_circular_bug_entry(entry, depth+1);
-        }
-        return 1;
+        result = __bfs_forwards(root, target, class_equal, target_entry);
+
+        return result;
 }
 
 #if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_PROVE_LOCKING)
@@ -1086,103 +1211,121 @@ check_noncircular(struct lock_class *source, unsigned int depth)
  * proving that two subgraphs can be connected by a new dependency
  * without creating any illegal irq-safe -> irq-unsafe lock dependency.
  */
-static enum lock_usage_bit find_usage_bit;
-static struct lock_class *forwards_match, *backwards_match;
+
+static inline int usage_match(struct lock_list *entry, void *bit)
+{
+        return entry->class->usage_mask & (1 << (enum lock_usage_bit)bit);
+}
+
+
 
 /*
  * Find a node in the forwards-direction dependency sub-graph starting
- * at <source> that matches <find_usage_bit>.
+ * at @root->class that matches @bit.
  *
- * Return 2 if such a node exists in the subgraph, and put that node
- * into <forwards_match>.
+ * Return 0 if such a node exists in the subgraph, and put that node
+ * into *@target_entry.
  *
- * Return 1 otherwise and keep <forwards_match> unchanged.
- * Return 0 on error.
+ * Return 1 otherwise and keep *@target_entry unchanged.
+ * Return <0 on error.
  */
-static noinline int
-find_usage_forwards(struct lock_class *source, unsigned int depth)
+static int
+find_usage_forwards(struct lock_list *root, enum lock_usage_bit bit,
+                        struct lock_list **target_entry)
 {
-        struct lock_list *entry;
-        int ret;
-
-        if (lockdep_dependency_visit(source, depth))
-                return 1;
-
-        if (depth > max_recursion_depth)
-                max_recursion_depth = depth;
-        if (depth >= RECURSION_LIMIT)
-                return print_infinite_recursion_bug();
+        int result;
 
         debug_atomic_inc(&nr_find_usage_forwards_checks);
-        if (source->usage_mask & (1 << find_usage_bit)) {
-                forwards_match = source;
-                return 2;
-        }
 
-        /*
-         * Check this lock's dependency list:
-         */
-        list_for_each_entry(entry, &source->locks_after, entry) {
-                debug_atomic_inc(&nr_find_usage_forwards_recursions);
-                ret = find_usage_forwards(entry->class, depth+1);
-                if (ret == 2 || ret == 0)
-                        return ret;
-        }
-        return 1;
+        result = __bfs_forwards(root, (void *)bit, usage_match, target_entry);
+
+        return result;
 }
 
 /*
  * Find a node in the backwards-direction dependency sub-graph starting
- * at <source> that matches <find_usage_bit>.
+ * at @root->class that matches @bit.
  *
- * Return 2 if such a node exists in the subgraph, and put that node
- * into <backwards_match>.
+ * Return 0 if such a node exists in the subgraph, and put that node
+ * into *@target_entry.
  *
- * Return 1 otherwise and keep <backwards_match> unchanged.
- * Return 0 on error.
+ * Return 1 otherwise and keep *@target_entry unchanged.
+ * Return <0 on error.
  */
-static noinline int
-find_usage_backwards(struct lock_class *source, unsigned int depth)
+static int
+find_usage_backwards(struct lock_list *root, enum lock_usage_bit bit,
+                        struct lock_list **target_entry)
 {
-        struct lock_list *entry;
-        int ret;
+        int result;
 
-        if (lockdep_dependency_visit(source, depth))
-                return 1;
+        debug_atomic_inc(&nr_find_usage_backwards_checks);
 
-        if (!__raw_spin_is_locked(&lockdep_lock))
-                return DEBUG_LOCKS_WARN_ON(1);
+        result = __bfs_backwards(root, (void *)bit, usage_match, target_entry);
 
-        if (depth > max_recursion_depth)
-                max_recursion_depth = depth;
-        if (depth >= RECURSION_LIMIT)
-                return print_infinite_recursion_bug();
+        return result;
+}
 
-        debug_atomic_inc(&nr_find_usage_backwards_checks);
-        if (source->usage_mask & (1 << find_usage_bit)) {
-                backwards_match = source;
-                return 2;
-        }
+static void print_lock_class_header(struct lock_class *class, int depth)
+{
+        int bit;
 
-        if (!source && debug_locks_off_graph_unlock()) {
-                WARN_ON(1);
-                return 0;
-        }
+        printk("%*s->", depth, "");
+        print_lock_name(class);
+        printk(" ops: %lu", class->ops);
+        printk(" {\n");
 
-        /*
-         * Check this lock's dependency list:
-         */
-        list_for_each_entry(entry, &source->locks_before, entry) {
-                debug_atomic_inc(&nr_find_usage_backwards_recursions);
-                ret = find_usage_backwards(entry->class, depth+1);
-                if (ret == 2 || ret == 0)
-                        return ret;
+        for (bit = 0; bit < LOCK_USAGE_STATES; bit++) {
+                if (class->usage_mask & (1 << bit)) {
+                        int len = depth;
+
+                        len += printk("%*s   %s", depth, "", usage_str[bit]);
+                        len += printk(" at:\n");
+                        print_stack_trace(class->usage_traces + bit, len);
+                }
         }
-        return 1;
+        printk("%*s }\n", depth, "");
+
+        printk("%*s ... key      at: ",depth,"");
+        print_ip_sym((unsigned long)class->key);
+}
+
+/*
+ * printk the shortest lock dependencies from @start to @end in reverse order:
+ */
+static void __used
+print_shortest_lock_dependencies(struct lock_list *leaf,
+                                struct lock_list *root)
+{
+        struct lock_list *entry = leaf;
+        int depth;
+
+        /*compute depth from generated tree by BFS*/
+        depth = get_lock_depth(leaf);
+
+        do {
+                print_lock_class_header(entry->class, depth);
+                printk("%*s ... acquired at:\n", depth, "");
+                print_stack_trace(&entry->trace, 2);
+                printk("\n");
+
+                if (depth == 0 && (entry != root)) {
+                        printk("lockdep:%s bad BFS generated tree\n", __func__);
+                        break;
+                }
+
+                entry = get_lock_parent(entry);
+                depth--;
+        } while (entry && (depth >= 0));
+
+        return;
 }
 
 static int
 print_bad_irq_dependency(struct task_struct *curr,
+                         struct lock_list *prev_root,
+                         struct lock_list *next_root,
+                         struct lock_list *backwards_entry,
+                         struct lock_list *forwards_entry,
                          struct held_lock *prev,
                          struct held_lock *next,
                          enum lock_usage_bit bit1,
@@ -1215,26 +1358,32 @@ print_bad_irq_dependency(struct task_struct *curr,
 
         printk("\nbut this new dependency connects a %s-irq-safe lock:\n",
                 irqclass);
-        print_lock_name(backwards_match);
+        print_lock_name(backwards_entry->class);
         printk("\n... which became %s-irq-safe at:\n", irqclass);
 
-        print_stack_trace(backwards_match->usage_traces + bit1, 1);
+        print_stack_trace(backwards_entry->class->usage_traces + bit1, 1);
 
         printk("\nto a %s-irq-unsafe lock:\n", irqclass);
-        print_lock_name(forwards_match);
+        print_lock_name(forwards_entry->class);
         printk("\n... which became %s-irq-unsafe at:\n", irqclass);
         printk("...");
 
-        print_stack_trace(forwards_match->usage_traces + bit2, 1);
+        print_stack_trace(forwards_entry->class->usage_traces + bit2, 1);
 
         printk("\nother info that might help us debug this:\n\n");
         lockdep_print_held_locks(curr);
 
-        printk("\nthe %s-irq-safe lock's dependencies:\n", irqclass);
-        print_lock_dependencies(backwards_match, 0);
+        printk("\nthe dependencies between %s-irq-safe lock", irqclass);
+        printk(" and the holding lock:\n");
+        if (!save_trace(&prev_root->trace))
+                return 0;
+        print_shortest_lock_dependencies(backwards_entry, prev_root);
 
-        printk("\nthe %s-irq-unsafe lock's dependencies:\n", irqclass);
-        print_lock_dependencies(forwards_match, 0);
+        printk("\nthe dependencies between the lock to be acquired");
+        printk(" and %s-irq-unsafe lock:\n", irqclass);
+        if (!save_trace(&next_root->trace))
+                return 0;
+        print_shortest_lock_dependencies(forwards_entry, next_root);
 
         printk("\nstack backtrace:\n");
         dump_stack();
@@ -1248,19 +1397,30 @@ check_usage(struct task_struct *curr, struct held_lock *prev,
             enum lock_usage_bit bit_forwards, const char *irqclass)
 {
         int ret;
+        struct lock_list this, that;
+        struct lock_list *uninitialized_var(target_entry);
+        struct lock_list *uninitialized_var(target_entry1);
 
-        find_usage_bit = bit_backwards;
-        /* fills in <backwards_match> */
-        ret = find_usage_backwards(hlock_class(prev), 0);
-        if (!ret || ret == 1)
+        this.parent = NULL;
+
+        this.class = hlock_class(prev);
+        ret = find_usage_backwards(&this, bit_backwards, &target_entry);
+        if (ret < 0)
+                return print_bfs_bug(ret);
+        if (ret == 1)
                 return ret;
 
-        find_usage_bit = bit_forwards;
-        ret = find_usage_forwards(hlock_class(next), 0);
-        if (!ret || ret == 1)
+        that.parent = NULL;
+        that.class = hlock_class(next);
+        ret = find_usage_forwards(&that, bit_forwards, &target_entry1);
+        if (ret < 0)
+                return print_bfs_bug(ret);
+        if (ret == 1)
                 return ret;
-        /* ret == 2 */
-        return print_bad_irq_dependency(curr, prev, next,
+
+        return print_bad_irq_dependency(curr, &this, &that,
+                        target_entry, target_entry1,
+                        prev, next,
                         bit_backwards, bit_forwards, irqclass);
 }
 
@@ -1472,6 +1632,8 @@ check_prev_add(struct task_struct *curr, struct held_lock *prev,
 {
         struct lock_list *entry;
         int ret;
+        struct lock_list this;
+        struct lock_list *uninitialized_var(target_entry);
 
         /*
          * Prove that the new <prev> -> <next> dependency would not
@@ -1482,10 +1644,13 @@ check_prev_add(struct task_struct *curr, struct held_lock *prev,
          * We are using global variables to control the recursion, to
          * keep the stackframe size of the recursive functions low:
          */
-        check_source = next;
-        check_target = prev;
-        if (!(check_noncircular(hlock_class(next), 0)))
-                return print_circular_bug_tail();
+        this.class = hlock_class(next);
+        this.parent = NULL;
+        ret = check_noncircular(&this, hlock_class(prev), &target_entry);
+        if (unlikely(!ret))
+                return print_circular_bug(&this, target_entry, next, prev);
+        else if (unlikely(ret < 0))
+                return print_bfs_bug(ret);
 
         if (!check_prev_add_irq(curr, prev, next))
                 return 0;
@@ -1884,7 +2049,8 @@ static int mark_lock(struct task_struct *curr, struct held_lock *this,
  * print irq inversion bug:
  */
 static int
-print_irq_inversion_bug(struct task_struct *curr, struct lock_class *other,
+print_irq_inversion_bug(struct task_struct *curr,
+                        struct lock_list *root, struct lock_list *other,
                         struct held_lock *this, int forwards,
                         const char *irqclass)
 {
@@ -1902,17 +2068,16 @@ print_irq_inversion_bug(struct task_struct *curr, struct lock_class *other,
                 printk("but this lock took another, %s-unsafe lock in the past:\n", irqclass);
         else
                 printk("but this lock was taken by another, %s-safe lock in the past:\n", irqclass);
-        print_lock_name(other);
+        print_lock_name(other->class);
         printk("\n\nand interrupts could create inverse lock ordering between them.\n\n");
 
         printk("\nother info that might help us debug this:\n");
         lockdep_print_held_locks(curr);
 
-        printk("\nthe first lock's dependencies:\n");
-        print_lock_dependencies(hlock_class(this), 0);
-
-        printk("\nthe second lock's dependencies:\n");
-        print_lock_dependencies(other, 0);
+        printk("\nthe shortest dependencies between 2nd lock and 1st lock:\n");
+        if (!save_trace(&root->trace))
+                return 0;
+        print_shortest_lock_dependencies(other, root);
 
         printk("\nstack backtrace:\n");
         dump_stack();
@@ -1929,14 +2094,19 @@ check_usage_forwards(struct task_struct *curr, struct held_lock *this,
                      enum lock_usage_bit bit, const char *irqclass)
 {
         int ret;
-
-        find_usage_bit = bit;
-        /* fills in <forwards_match> */
-        ret = find_usage_forwards(hlock_class(this), 0);
-        if (!ret || ret == 1)
+        struct lock_list root;
+        struct lock_list *uninitialized_var(target_entry);
+
+        root.parent = NULL;
+        root.class = hlock_class(this);
+        ret = find_usage_forwards(&root, bit, &target_entry);
+        if (ret < 0)
+                return print_bfs_bug(ret);
+        if (ret == 1)
                 return ret;
 
-        return print_irq_inversion_bug(curr, forwards_match, this, 1, irqclass);
+        return print_irq_inversion_bug(curr, &root, target_entry,
+                                        this, 1, irqclass);
 }
 
 /*
@@ -1948,14 +2118,19 @@ check_usage_backwards(struct task_struct *curr, struct held_lock *this,
                       enum lock_usage_bit bit, const char *irqclass)
 {
         int ret;
-
-        find_usage_bit = bit;
-        /* fills in <backwards_match> */
-        ret = find_usage_backwards(hlock_class(this), 0);
-        if (!ret || ret == 1)
+        struct lock_list root;
+        struct lock_list *uninitialized_var(target_entry);
+
+        root.parent = NULL;
+        root.class = hlock_class(this);
+        ret = find_usage_backwards(&root, bit, &target_entry);
+        if (ret < 0)
+                return print_bfs_bug(ret);
+        if (ret == 1)
                 return ret;
 
-        return print_irq_inversion_bug(curr, backwards_match, this, 0, irqclass);
+        return print_irq_inversion_bug(curr, &root, target_entry,
+                                        this, 1, irqclass);
 }
 
 void print_irqtrace_events(struct task_struct *curr)
@@ -2530,13 +2705,15 @@ EXPORT_SYMBOL_GPL(lockdep_init_map);
  */
 static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
                           int trylock, int read, int check, int hardirqs_off,
-                          struct lockdep_map *nest_lock, unsigned long ip)
+                          struct lockdep_map *nest_lock, unsigned long ip,
+                          int references)
 {
         struct task_struct *curr = current;
         struct lock_class *class = NULL;
         struct held_lock *hlock;
         unsigned int depth, id;
         int chain_head = 0;
+        int class_idx;
         u64 chain_key;
 
         if (!prove_locking)
@@ -2584,10 +2761,24 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
         if (DEBUG_LOCKS_WARN_ON(depth >= MAX_LOCK_DEPTH))
                 return 0;
 
+        class_idx = class - lock_classes + 1;
+
+        if (depth) {
+                hlock = curr->held_locks + depth - 1;
+                if (hlock->class_idx == class_idx && nest_lock) {
+                        if (hlock->references)
+                                hlock->references++;
+                        else
+                                hlock->references = 2;
+
+                        return 1;
+                }
+        }
+
         hlock = curr->held_locks + depth;
         if (DEBUG_LOCKS_WARN_ON(!class))
                 return 0;
-        hlock->class_idx = class - lock_classes + 1;
+        hlock->class_idx = class_idx;
         hlock->acquire_ip = ip;
         hlock->instance = lock;
         hlock->nest_lock = nest_lock;
@@ -2595,6 +2786,7 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
         hlock->read = read;
         hlock->check = check;
         hlock->hardirqs_off = !!hardirqs_off;
+        hlock->references = references;
 #ifdef CONFIG_LOCK_STAT
         hlock->waittime_stamp = 0;
         hlock->holdtime_stamp = sched_clock();
@@ -2703,6 +2895,30 @@ static int check_unlock(struct task_struct *curr, struct lockdep_map *lock,
         return 1;
 }
 
+static int match_held_lock(struct held_lock *hlock, struct lockdep_map *lock)
+{
+        if (hlock->instance == lock)
+                return 1;
+
+        if (hlock->references) {
+                struct lock_class *class = lock->class_cache;
+
+                if (!class)
+                        class = look_up_lock_class(lock, 0);
+
+                if (DEBUG_LOCKS_WARN_ON(!class))
+                        return 0;
+
+                if (DEBUG_LOCKS_WARN_ON(!hlock->nest_lock))
+                        return 0;
+
+                if (hlock->class_idx == class - lock_classes + 1)
+                        return 1;
+        }
+
+        return 0;
+}
+
 static int
 __lock_set_class(struct lockdep_map *lock, const char *name,
                  struct lock_class_key *key, unsigned int subclass,
@@ -2726,7 +2942,7 @@ __lock_set_class(struct lockdep_map *lock, const char *name,
                  */
                 if (prev_hlock && prev_hlock->irq_context != hlock->irq_context)
                         break;
-                if (hlock->instance == lock)
+                if (match_held_lock(hlock, lock))
                         goto found_it;
                 prev_hlock = hlock;
         }
@@ -2745,7 +2961,8 @@ found_it:
                 if (!__lock_acquire(hlock->instance,
                         hlock_class(hlock)->subclass, hlock->trylock,
                                 hlock->read, hlock->check, hlock->hardirqs_off,
-                                hlock->nest_lock, hlock->acquire_ip))
+                                hlock->nest_lock, hlock->acquire_ip,
+                                hlock->references))
                         return 0;
         }
 
@@ -2784,20 +3001,34 @@ lock_release_non_nested(struct task_struct *curr,
                  */
                 if (prev_hlock && prev_hlock->irq_context != hlock->irq_context)
                         break;
-                if (hlock->instance == lock)
+                if (match_held_lock(hlock, lock))
                         goto found_it;
                 prev_hlock = hlock;
         }
         return print_unlock_inbalance_bug(curr, lock, ip);
 
 found_it:
-        lock_release_holdtime(hlock);
+        if (hlock->instance == lock)
+                lock_release_holdtime(hlock);
+
+        if (hlock->references) {
+                hlock->references--;
+                if (hlock->references) {
+                        /*
+                         * We had, and after removing one, still have
+                         * references, the current lock stack is still
+                         * valid. We're done!
+                         */
+                        return 1;
+                }
+        }
 
         /*
          * We have the right lock to unlock, 'hlock' points to it.
          * Now we remove it from the stack, and add back the other
          * entries (if any), recalculating the hash along the way:
          */
+
         curr->lockdep_depth = i;
         curr->curr_chain_key = hlock->prev_chain_key;
 
@@ -2806,7 +3037,8 @@ found_it:
                 if (!__lock_acquire(hlock->instance,
                         hlock_class(hlock)->subclass, hlock->trylock,
                                 hlock->read, hlock->check, hlock->hardirqs_off,
-                                hlock->nest_lock, hlock->acquire_ip))
+                                hlock->nest_lock, hlock->acquire_ip,
+                                hlock->references))
                         return 0;
         }
 
@@ -2836,7 +3068,7 @@ static int lock_release_nested(struct task_struct *curr,
         /*
          * Is the unlock non-nested:
          */
-        if (hlock->instance != lock)
+        if (hlock->instance != lock || hlock->references)
                 return lock_release_non_nested(curr, lock, ip);
         curr->lockdep_depth--;
 
@@ -2881,6 +3113,21 @@ __lock_release(struct lockdep_map *lock, int nested, unsigned long ip)
         check_chain_key(curr);
 }
 
+static int __lock_is_held(struct lockdep_map *lock)
+{
+        struct task_struct *curr = current;
+        int i;
+
+        for (i = 0; i < curr->lockdep_depth; i++) {
+                struct held_lock *hlock = curr->held_locks + i;
+
+                if (match_held_lock(hlock, lock))
+                        return 1;
+        }
+
+        return 0;
+}
+
 /*
  * Check whether we follow the irq-flags state precisely:
  */
@@ -2957,7 +3204,7 @@ void lock_acquire(struct lockdep_map *lock, unsigned int subclass,
 
         current->lockdep_recursion = 1;
         __lock_acquire(lock, subclass, trylock, read, check,
-                       irqs_disabled_flags(flags), nest_lock, ip);
+                       irqs_disabled_flags(flags), nest_lock, ip, 0);
         current->lockdep_recursion = 0;
         raw_local_irq_restore(flags);
 }
@@ -2982,6 +3229,26 @@ void lock_release(struct lockdep_map *lock, int nested,
 }
 EXPORT_SYMBOL_GPL(lock_release);
 
+int lock_is_held(struct lockdep_map *lock)
+{
+        unsigned long flags;
+        int ret = 0;
+
+        if (unlikely(current->lockdep_recursion))
+                return ret;
+
+        raw_local_irq_save(flags);
+        check_flags(flags);
+
+        current->lockdep_recursion = 1;
+        ret = __lock_is_held(lock);
+        current->lockdep_recursion = 0;
+        raw_local_irq_restore(flags);
+
+        return ret;
+}
+EXPORT_SYMBOL_GPL(lock_is_held);
+
 void lockdep_set_current_reclaim_state(gfp_t gfp_mask)
 {
         current->lockdep_reclaim_gfp = gfp_mask;
@@ -3041,7 +3308,7 @@ __lock_contended(struct lockdep_map *lock, unsigned long ip)
                  */
                 if (prev_hlock && prev_hlock->irq_context != hlock->irq_context)
                         break;
-                if (hlock->instance == lock)
+                if (match_held_lock(hlock, lock))
                         goto found_it;
                 prev_hlock = hlock;
         }
@@ -3049,6 +3316,9 @@ __lock_contended(struct lockdep_map *lock, unsigned long ip)
         return;
 
 found_it:
+        if (hlock->instance != lock)
+                return;
+
         hlock->waittime_stamp = sched_clock();
 
         contention_point = lock_point(hlock_class(hlock)->contention_point, ip);
@@ -3088,7 +3358,7 @@ __lock_acquired(struct lockdep_map *lock, unsigned long ip)
                  */
                 if (prev_hlock && prev_hlock->irq_context != hlock->irq_context)
                         break;
-                if (hlock->instance == lock)
+                if (match_held_lock(hlock, lock))
                         goto found_it;
                 prev_hlock = hlock;
         }
@@ -3096,6 +3366,9 @@ __lock_acquired(struct lockdep_map *lock, unsigned long ip)
         return;
 
 found_it:
+        if (hlock->instance != lock)
+                return;
+
         cpu = smp_processor_id();
         if (hlock->waittime_stamp) {
                 now = sched_clock();
@@ -3326,7 +3599,12 @@ void __init lockdep_info(void)
                 sizeof(struct list_head) * CLASSHASH_SIZE +
                 sizeof(struct lock_list) * MAX_LOCKDEP_ENTRIES +
                 sizeof(struct lock_chain) * MAX_LOCKDEP_CHAINS +
-                sizeof(struct list_head) * CHAINHASH_SIZE) / 1024);
+                sizeof(struct list_head) * CHAINHASH_SIZE
+#ifdef CONFIG_PROVE_LOCKING
+                + sizeof(struct circular_queue)
+#endif
+                ) / 1024
+                );
 
         printk(" per task-struct memory footprint: %lu bytes\n",
                 sizeof(struct held_lock) * MAX_LOCK_DEPTH);
diff --git a/kernel/lockdep_internals.h b/kernel/lockdep_internals.h
index 699a2ac3a0d7..a2ee95ad1313 100644
--- a/kernel/lockdep_internals.h
+++ b/kernel/lockdep_internals.h
@@ -91,6 +91,8 @@ extern unsigned int nr_process_chains;
 extern unsigned int max_lockdep_depth;
 extern unsigned int max_recursion_depth;
 
+extern unsigned int max_bfs_queue_depth;
+
 #ifdef CONFIG_PROVE_LOCKING
 extern unsigned long lockdep_count_forward_deps(struct lock_class *);
 extern unsigned long lockdep_count_backward_deps(struct lock_class *);
diff --git a/kernel/lockdep_proc.c b/kernel/lockdep_proc.c
index e94caa666dba..d4b3dbc79fdb 100644
--- a/kernel/lockdep_proc.c
+++ b/kernel/lockdep_proc.c
@@ -25,38 +25,12 @@
 
 static void *l_next(struct seq_file *m, void *v, loff_t *pos)
 {
-        struct lock_class *class;
-
-        (*pos)++;
-
-        if (v == SEQ_START_TOKEN)
-                class = m->private;
-        else {
-                class = v;
-
-                if (class->lock_entry.next != &all_lock_classes)
-                        class = list_entry(class->lock_entry.next,
-                                           struct lock_class, lock_entry);
-                else
-                        class = NULL;
-        }
-
-        return class;
+        return seq_list_next(v, &all_lock_classes, pos);
 }
 
 static void *l_start(struct seq_file *m, loff_t *pos)
 {
-        struct lock_class *class;
-        loff_t i = 0;
-
-        if (*pos == 0)
-                return SEQ_START_TOKEN;
-
-        list_for_each_entry(class, &all_lock_classes, lock_entry) {
-                if (++i == *pos)
-                return class;
-        }
-        return NULL;
+        return seq_list_start_head(&all_lock_classes, *pos);
 }
 
 static void l_stop(struct seq_file *m, void *v)
@@ -82,11 +56,11 @@ static void print_name(struct seq_file *m, struct lock_class *class)
 
 static int l_show(struct seq_file *m, void *v)
 {
-        struct lock_class *class = v;
+        struct lock_class *class = list_entry(v, struct lock_class, lock_entry);
         struct lock_list *entry;
         char usage[LOCK_USAGE_CHARS];
 
-        if (v == SEQ_START_TOKEN) {
+        if (v == &all_lock_classes) {
                 seq_printf(m, "all lock classes:\n");
                 return 0;
         }
@@ -128,17 +102,7 @@ static const struct seq_operations lockdep_ops = {
 
 static int lockdep_open(struct inode *inode, struct file *file)
 {
-        int res = seq_open(file, &lockdep_ops);
-        if (!res) {
-                struct seq_file *m = file->private_data;
-
-                if (!list_empty(&all_lock_classes))
-                        m->private = list_entry(all_lock_classes.next,
-                                        struct lock_class, lock_entry);
-                else
-                        m->private = NULL;
-        }
-        return res;
+        return seq_open(file, &lockdep_ops);
 }
 
 static const struct file_operations proc_lockdep_operations = {
@@ -149,37 +113,23 @@ static const struct file_operations proc_lockdep_operations = {
 };
 
 #ifdef CONFIG_PROVE_LOCKING
-static void *lc_next(struct seq_file *m, void *v, loff_t *pos)
-{
-        struct lock_chain *chain;
-
-        (*pos)++;
-
-        if (v == SEQ_START_TOKEN)
-                chain = m->private;
-        else {
-                chain = v;
-
-                if (*pos < nr_lock_chains)
-                        chain = lock_chains + *pos;
-                else
-                        chain = NULL;
-        }
-
-        return chain;
-}
-
 static void *lc_start(struct seq_file *m, loff_t *pos)
 {
         if (*pos == 0)
                 return SEQ_START_TOKEN;
 
-        if (*pos < nr_lock_chains)
-                return lock_chains + *pos;
+        if (*pos - 1 < nr_lock_chains)
+                return lock_chains + (*pos - 1);
 
         return NULL;
 }
 
+static void *lc_next(struct seq_file *m, void *v, loff_t *pos)
+{
+        (*pos)++;
+        return lc_start(m, pos);
+}
+
 static void lc_stop(struct seq_file *m, void *v)
 {
 }
@@ -220,16 +170,7 @@ static const struct seq_operations lockdep_chains_ops = {
 
 static int lockdep_chains_open(struct inode *inode, struct file *file)
 {
-        int res = seq_open(file, &lockdep_chains_ops);
-        if (!res) {
-                struct seq_file *m = file->private_data;
-
-                if (nr_lock_chains)
-                        m->private = lock_chains;
-                else
-                        m->private = NULL;
-        }
-        return res;
+        return seq_open(file, &lockdep_chains_ops);
 }
 
 static const struct file_operations proc_lockdep_chains_operations = {
@@ -258,16 +199,10 @@ static void lockdep_stats_debug_show(struct seq_file *m)
                 debug_atomic_read(&chain_lookup_hits));
         seq_printf(m, " cyclic checks:                 %11u\n",
                 debug_atomic_read(&nr_cyclic_checks));
-        seq_printf(m, " cyclic-check recursions:       %11u\n",
-                debug_atomic_read(&nr_cyclic_check_recursions));
         seq_printf(m, " find-mask forwards checks:     %11u\n",
                 debug_atomic_read(&nr_find_usage_forwards_checks));
-        seq_printf(m, " find-mask forwards recursions: %11u\n",
-                debug_atomic_read(&nr_find_usage_forwards_recursions));
         seq_printf(m, " find-mask backwards checks:    %11u\n",
                 debug_atomic_read(&nr_find_usage_backwards_checks));
-        seq_printf(m, " find-mask backwards recursions:%11u\n",
-                debug_atomic_read(&nr_find_usage_backwards_recursions));
 
         seq_printf(m, " hardirq on events:             %11u\n", hi1);
         seq_printf(m, " hardirq off events:            %11u\n", hi2);
@@ -409,8 +344,10 @@ static int lockdep_stats_show(struct seq_file *m, void *v)
                         nr_unused);
         seq_printf(m, " max locking depth:             %11u\n",
                         max_lockdep_depth);
-        seq_printf(m, " max recursion depth:           %11u\n",
-                        max_recursion_depth);
+#ifdef CONFIG_PROVE_LOCKING
+        seq_printf(m, " max bfs queue depth:           %11u\n",
+                        max_bfs_queue_depth);
+#endif
         lockdep_stats_debug_show(m);
         seq_printf(m, " debug_locks:                   %11u\n",
                         debug_locks);
@@ -438,7 +375,6 @@ struct lock_stat_data {
 };
 
 struct lock_stat_seq {
-        struct lock_stat_data *iter;
         struct lock_stat_data *iter_end;
         struct lock_stat_data stats[MAX_LOCKDEP_KEYS];
 };
@@ -626,34 +562,22 @@ static void seq_header(struct seq_file *m)
 static void *ls_start(struct seq_file *m, loff_t *pos)
 {
         struct lock_stat_seq *data = m->private;
+        struct lock_stat_data *iter;
 
         if (*pos == 0)
                 return SEQ_START_TOKEN;
 
-        data->iter = data->stats + *pos;
-        if (data->iter >= data->iter_end)
-                data->iter = NULL;
+        iter = data->stats + (*pos - 1);
+        if (iter >= data->iter_end)
+                iter = NULL;
 
-        return data->iter;
+        return iter;
 }
 
 static void *ls_next(struct seq_file *m, void *v, loff_t *pos)
 {
-        struct lock_stat_seq *data = m->private;
-
         (*pos)++;
-
-        if (v == SEQ_START_TOKEN)
-                data->iter = data->stats;
-        else {
-                data->iter = v;
-                data->iter++;
-        }
-
-        if (data->iter == data->iter_end)
-                data->iter = NULL;
-
-        return data->iter;
+        return ls_start(m, pos);
 }
 
 static void ls_stop(struct seq_file *m, void *v)
@@ -691,7 +615,6 @@ static int lock_stat_open(struct inode *inode, struct file *file)
                 struct lock_stat_data *iter = data->stats;
                 struct seq_file *m = file->private_data;
 
-                data->iter = iter;
                 list_for_each_entry(class, &all_lock_classes, lock_entry) {
                         iter->class = class;
                         iter->stats = lock_stats(class);
@@ -699,7 +622,7 @@ static int lock_stat_open(struct inode *inode, struct file *file)
                 }
                 data->iter_end = iter;
 
-                sort(data->stats, data->iter_end - data->iter,
+                sort(data->stats, data->iter_end - data->stats,
                                 sizeof(struct lock_stat_data),
                                 lock_stat_cmp, NULL);
 
@@ -734,7 +657,6 @@ static int lock_stat_release(struct inode *inode, struct file *file)
         struct seq_file *seq = file->private_data;
 
         vfree(seq->private);
-        seq->private = NULL;
         return seq_release(inode, file);
 }
 
diff --git a/kernel/module.c b/kernel/module.c
index 46580edff0cb..05ce49ced8f6 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -369,7 +369,7 @@ EXPORT_SYMBOL_GPL(find_module);
 
 #ifdef CONFIG_SMP
 
-#ifdef CONFIG_HAVE_DYNAMIC_PER_CPU_AREA
+#ifndef CONFIG_HAVE_LEGACY_PER_CPU_AREA
 
 static void *percpu_modalloc(unsigned long size, unsigned long align,
                              const char *name)
@@ -394,7 +394,7 @@ static void percpu_modfree(void *freeme)
         free_percpu(freeme);
 }
 
-#else /* ... !CONFIG_HAVE_DYNAMIC_PER_CPU_AREA */
+#else /* ... CONFIG_HAVE_LEGACY_PER_CPU_AREA */
 
 /* Number of blocks used and allocated. */
 static unsigned int pcpu_num_used, pcpu_num_allocated;
@@ -540,7 +540,7 @@ static int percpu_modinit(void)
 }
 __initcall(percpu_modinit);
 
-#endif /* CONFIG_HAVE_DYNAMIC_PER_CPU_AREA */
+#endif /* CONFIG_HAVE_LEGACY_PER_CPU_AREA */
 
 static unsigned int find_pcpusec(Elf_Ehdr *hdr,
                                  Elf_Shdr *sechdrs,
diff --git a/kernel/perf_counter.c b/kernel/perf_counter.c
index d7cbc579fc80..8cb94a52d1bb 100644
--- a/kernel/perf_counter.c
+++ b/kernel/perf_counter.c
@@ -46,12 +46,18 @@ static atomic_t nr_task_counters __read_mostly;
 
 /*
  * perf counter paranoia level:
- *  0 - not paranoid
- *  1 - disallow cpu counters to unpriv
- *  2 - disallow kernel profiling to unpriv
+ *  -1 - not paranoid at all
+ *   0 - disallow raw tracepoint access for unpriv
+ *   1 - disallow cpu counters for unpriv
+ *   2 - disallow kernel profiling for unpriv
  */
 int sysctl_perf_counter_paranoid __read_mostly = 1;
 
+static inline bool perf_paranoid_tracepoint_raw(void)
+{
+        return sysctl_perf_counter_paranoid > -1;
+}
+
 static inline bool perf_paranoid_cpu(void)
 {
         return sysctl_perf_counter_paranoid > 0;
@@ -100,16 +106,16 @@ hw_perf_group_sched_in(struct perf_counter *group_leader,
 
 void __weak perf_counter_print_debug(void)      { }
 
-static DEFINE_PER_CPU(int, disable_count);
+static DEFINE_PER_CPU(int, perf_disable_count);
 
 void __perf_disable(void)
 {
-        __get_cpu_var(disable_count)++;
+        __get_cpu_var(perf_disable_count)++;
 }
 
 bool __perf_enable(void)
 {
-        return !--__get_cpu_var(disable_count);
+        return !--__get_cpu_var(perf_disable_count);
 }
 
 void perf_disable(void)
@@ -469,7 +475,8 @@ static void update_counter_times(struct perf_counter *counter)
         struct perf_counter_context *ctx = counter->ctx;
         u64 run_end;
 
-        if (counter->state < PERF_COUNTER_STATE_INACTIVE)
+        if (counter->state < PERF_COUNTER_STATE_INACTIVE ||
+            counter->group_leader->state < PERF_COUNTER_STATE_INACTIVE)
                 return;
 
         counter->total_time_enabled = ctx->time - counter->tstamp_enabled;
@@ -518,7 +525,7 @@ static void __perf_counter_disable(void *info)
          */
         if (counter->state >= PERF_COUNTER_STATE_INACTIVE) {
                 update_context_time(ctx);
-                update_counter_times(counter);
+                update_group_times(counter);
                 if (counter == counter->group_leader)
                         group_sched_out(counter, cpuctx, ctx);
                 else
@@ -573,7 +580,7 @@ static void perf_counter_disable(struct perf_counter *counter)
          * in, so we can change the state safely.
          */
         if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
-                update_counter_times(counter);
+                update_group_times(counter);
                 counter->state = PERF_COUNTER_STATE_OFF;
         }
 
@@ -851,6 +858,27 @@ retry:
 }
 
 /*
+ * Put a counter into inactive state and update time fields.
+ * Enabling the leader of a group effectively enables all
+ * the group members that aren't explicitly disabled, so we
+ * have to update their ->tstamp_enabled also.
+ * Note: this works for group members as well as group leaders
+ * since the non-leader members' sibling_lists will be empty.
+ */
+static void __perf_counter_mark_enabled(struct perf_counter *counter,
+                                        struct perf_counter_context *ctx)
+{
+        struct perf_counter *sub;
+
+        counter->state = PERF_COUNTER_STATE_INACTIVE;
+        counter->tstamp_enabled = ctx->time - counter->total_time_enabled;
+        list_for_each_entry(sub, &counter->sibling_list, list_entry)
+                if (sub->state >= PERF_COUNTER_STATE_INACTIVE)
+                        sub->tstamp_enabled =
+                                ctx->time - sub->total_time_enabled;
+}
+
+/*
  * Cross CPU call to enable a performance counter
  */
 static void __perf_counter_enable(void *info)
@@ -877,8 +905,7 @@ static void __perf_counter_enable(void *info)
 
         if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
                 goto unlock;
-        counter->state = PERF_COUNTER_STATE_INACTIVE;
-        counter->tstamp_enabled = ctx->time - counter->total_time_enabled;
+        __perf_counter_mark_enabled(counter, ctx);
 
         /*
          * If the counter is in a group and isn't the group leader,
@@ -971,11 +998,9 @@ static void perf_counter_enable(struct perf_counter *counter)
          * Since we have the lock this context can't be scheduled
          * in, so we can change the state safely.
          */
-        if (counter->state == PERF_COUNTER_STATE_OFF) {
-                counter->state = PERF_COUNTER_STATE_INACTIVE;
-                counter->tstamp_enabled =
-                        ctx->time - counter->total_time_enabled;
-        }
+        if (counter->state == PERF_COUNTER_STATE_OFF)
+                __perf_counter_mark_enabled(counter, ctx);
+
  out:
         spin_unlock_irq(&ctx->lock);
 }
@@ -1479,9 +1504,7 @@ static void perf_counter_enable_on_exec(struct task_struct *task)
                 counter->attr.enable_on_exec = 0;
                 if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
                         continue;
-                counter->state = PERF_COUNTER_STATE_INACTIVE;
-                counter->tstamp_enabled =
-                        ctx->time - counter->total_time_enabled;
+                __perf_counter_mark_enabled(counter, ctx);
                 enabled = 1;
         }
 
@@ -1675,6 +1698,11 @@ static void free_counter(struct perf_counter *counter)
                         atomic_dec(&nr_task_counters);
         }
 
+        if (counter->output) {
+                fput(counter->output->filp);
+                counter->output = NULL;
+        }
+
         if (counter->destroy)
                 counter->destroy(counter);
 
@@ -1960,6 +1988,8 @@ unlock:
         return ret;
 }
 
+int perf_counter_set_output(struct perf_counter *counter, int output_fd);
+
 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 {
         struct perf_counter *counter = file->private_data;
@@ -1983,6 +2013,9 @@ static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
         case PERF_COUNTER_IOC_PERIOD:
                 return perf_counter_period(counter, (u64 __user *)arg);
 
+        case PERF_COUNTER_IOC_SET_OUTPUT:
+                return perf_counter_set_output(counter, arg);
+
         default:
                 return -ENOTTY;
         }
@@ -2253,6 +2286,11 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma)
 
         WARN_ON_ONCE(counter->ctx->parent_ctx);
         mutex_lock(&counter->mmap_mutex);
+        if (counter->output) {
+                ret = -EINVAL;
+                goto unlock;
+        }
+
         if (atomic_inc_not_zero(&counter->mmap_count)) {
                 if (nr_pages != counter->data->nr_pages)
                         ret = -EINVAL;
@@ -2638,6 +2676,7 @@ static int perf_output_begin(struct perf_output_handle *handle,
                              struct perf_counter *counter, unsigned int size,
                              int nmi, int sample)
 {
+        struct perf_counter *output_counter;
         struct perf_mmap_data *data;
         unsigned int offset, head;
         int have_lost;
@@ -2647,13 +2686,17 @@ static int perf_output_begin(struct perf_output_handle *handle,
                 u64                      lost;
         } lost_event;
 
+        rcu_read_lock();
         /*
          * For inherited counters we send all the output towards the parent.
          */
         if (counter->parent)
                 counter = counter->parent;
 
-        rcu_read_lock();
+        output_counter = rcu_dereference(counter->output);
+        if (output_counter)
+                counter = output_counter;
+
         data = rcu_dereference(counter->data);
         if (!data)
                 goto out;
@@ -3934,6 +3977,7 @@ static const struct pmu *tp_perf_counter_init(struct perf_counter *counter)
          * have these.
          */
         if ((counter->attr.sample_type & PERF_SAMPLE_RAW) &&
+                        perf_paranoid_tracepoint_raw() &&
                         !capable(CAP_SYS_ADMIN))
                 return ERR_PTR(-EPERM);
 
@@ -4171,6 +4215,7 @@ static int perf_copy_attr(struct perf_counter_attr __user *uattr,
                         if (val)
                                 goto err_size;
                 }
+                size = sizeof(*attr);
         }
 
         ret = copy_from_user(attr, uattr, size);
@@ -4202,6 +4247,57 @@ err_size:
         goto out;
 }
 
+int perf_counter_set_output(struct perf_counter *counter, int output_fd)
+{
+        struct perf_counter *output_counter = NULL;
+        struct file *output_file = NULL;
+        struct perf_counter *old_output;
+        int fput_needed = 0;
+        int ret = -EINVAL;
+
+        if (!output_fd)
+                goto set;
+
+        output_file = fget_light(output_fd, &fput_needed);
+        if (!output_file)
+                return -EBADF;
+
+        if (output_file->f_op != &perf_fops)
+                goto out;
+
+        output_counter = output_file->private_data;
+
+        /* Don't chain output fds */
+        if (output_counter->output)
+                goto out;
+
+        /* Don't set an output fd when we already have an output channel */
+        if (counter->data)
+                goto out;
+
+        atomic_long_inc(&output_file->f_count);
+
+set:
+        mutex_lock(&counter->mmap_mutex);
+        old_output = counter->output;
+        rcu_assign_pointer(counter->output, output_counter);
+        mutex_unlock(&counter->mmap_mutex);
+
+        if (old_output) {
+                /*
+                 * we need to make sure no existing perf_output_*()
+                 * is still referencing this counter.
+                 */
+                synchronize_rcu();
+                fput(old_output->filp);
+        }
+
+        ret = 0;
+out:
+        fput_light(output_file, fput_needed);
+        return ret;
+}
+
 /**
  * sys_perf_counter_open - open a performance counter, associate it to a task/cpu
  *
@@ -4221,15 +4317,15 @@ SYSCALL_DEFINE5(perf_counter_open,
         struct file *group_file = NULL;
         int fput_needed = 0;
         int fput_needed2 = 0;
-        int ret;
+        int err;
 
         /* for future expandability... */
-        if (flags)
+        if (flags & ~(PERF_FLAG_FD_NO_GROUP | PERF_FLAG_FD_OUTPUT))
                 return -EINVAL;
 
-        ret = perf_copy_attr(attr_uptr, &attr);
-        if (ret)
-                return ret;
+        err = perf_copy_attr(attr_uptr, &attr);
+        if (err)
+                return err;
 
         if (!attr.exclude_kernel) {
                 if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
@@ -4252,8 +4348,8 @@ SYSCALL_DEFINE5(perf_counter_open,
          * Look up the group leader (we will attach this counter to it):
          */
         group_leader = NULL;
-        if (group_fd != -1) {
-                ret = -EINVAL;
+        if (group_fd != -1 && !(flags & PERF_FLAG_FD_NO_GROUP)) {
+                err = -EINVAL;
                 group_file = fget_light(group_fd, &fput_needed);
                 if (!group_file)
                         goto err_put_context;
@@ -4282,18 +4378,24 @@ SYSCALL_DEFINE5(perf_counter_open,
 
         counter = perf_counter_alloc(&attr, cpu, ctx, group_leader,
                                      NULL, GFP_KERNEL);
-        ret = PTR_ERR(counter);
+        err = PTR_ERR(counter);
         if (IS_ERR(counter))
                 goto err_put_context;
 
-        ret = anon_inode_getfd("[perf_counter]", &perf_fops, counter, 0);
-        if (ret < 0)
+        err = anon_inode_getfd("[perf_counter]", &perf_fops, counter, 0);
+        if (err < 0)
                 goto err_free_put_context;
 
-        counter_file = fget_light(ret, &fput_needed2);
+        counter_file = fget_light(err, &fput_needed2);
         if (!counter_file)
                 goto err_free_put_context;
 
+        if (flags & PERF_FLAG_FD_OUTPUT) {
+                err = perf_counter_set_output(counter, group_fd);
+                if (err)
+                        goto err_fput_free_put_context;
+        }
+
         counter->filp = counter_file;
         WARN_ON_ONCE(ctx->parent_ctx);
         mutex_lock(&ctx->mutex);
@@ -4307,20 +4409,20 @@ SYSCALL_DEFINE5(perf_counter_open,
         list_add_tail(&counter->owner_entry, &current->perf_counter_list);
         mutex_unlock(&current->perf_counter_mutex);
 
+err_fput_free_put_context:
         fput_light(counter_file, fput_needed2);
 
-out_fput:
-        fput_light(group_file, fput_needed);
-
-        return ret;
-
 err_free_put_context:
-        kfree(counter);
+        if (err < 0)
+                kfree(counter);
 
 err_put_context:
-        put_ctx(ctx);
+        if (err < 0)
+                put_ctx(ctx);
+
+        fput_light(group_file, fput_needed);
 
-        goto out_fput;
+        return err;
 }
 
 /*
diff --git a/kernel/power/Kconfig b/kernel/power/Kconfig
index 72067cbdb37f..91e09d3b2eb2 100644
--- a/kernel/power/Kconfig
+++ b/kernel/power/Kconfig
@@ -208,3 +208,17 @@ config APM_EMULATION
           random kernel OOPSes or reboots that don't seem to be related to
           anything, try disabling/enabling this option (or disabling/enabling
           APM in your BIOS).
+
+config PM_RUNTIME
+        bool "Run-time PM core functionality"
+        depends on PM
+        ---help---
+          Enable functionality allowing I/O devices to be put into energy-saving
+          (low power) states at run time (or autosuspended) after a specified
+          period of inactivity and woken up in response to a hardware-generated
+          wake-up event or a driver's request.
+
+          Hardware support is generally required for this functionality to work
+          and the bus type drivers of the buses the devices are on are
+          responsible for the actual handling of the autosuspend requests and
+          wake-up events.
diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c
index 81d2e7464893..04b3a83d686f 100644
--- a/kernel/power/hibernate.c
+++ b/kernel/power/hibernate.c
@@ -298,8 +298,8 @@ int hibernation_snapshot(int platform_mode)
         if (error)
                 return error;
 
-        /* Free memory before shutting down devices. */
-        error = swsusp_shrink_memory();
+        /* Preallocate image memory before shutting down devices. */
+        error = hibernate_preallocate_memory();
         if (error)
                 goto Close;
 
@@ -315,6 +315,10 @@ int hibernation_snapshot(int platform_mode)
         /* Control returns here after successful restore */
 
  Resume_devices:
+        /* We may need to release the preallocated image pages here. */
+        if (error || !in_suspend)
+                swsusp_free();
+
         dpm_resume_end(in_suspend ?
                 (error ? PMSG_RECOVER : PMSG_THAW) : PMSG_RESTORE);
         resume_console();
@@ -460,11 +464,11 @@ int hibernation_platform_enter(void)
 
         error = hibernation_ops->prepare();
         if (error)
-                goto Platofrm_finish;
+                goto Platform_finish;
 
         error = disable_nonboot_cpus();
         if (error)
-                goto Platofrm_finish;
+                goto Platform_finish;
 
         local_irq_disable();
         sysdev_suspend(PMSG_HIBERNATE);
@@ -476,7 +480,7 @@ int hibernation_platform_enter(void)
          * We don't need to reenable the nonboot CPUs or resume consoles, since
          * the system is going to be halted anyway.
          */
- Platofrm_finish:
+ Platform_finish:
         hibernation_ops->finish();
 
         dpm_suspend_noirq(PMSG_RESTORE);
@@ -578,7 +582,10 @@ int hibernate(void)
                 goto Thaw;
 
         error = hibernation_snapshot(hibernation_mode == HIBERNATION_PLATFORM);
-        if (in_suspend && !error) {
+        if (error)
+                goto Thaw;
+
+        if (in_suspend) {
                 unsigned int flags = 0;
 
                 if (hibernation_mode == HIBERNATION_PLATFORM)
@@ -590,8 +597,8 @@ int hibernate(void)
                         power_down();
         } else {
                 pr_debug("PM: Image restored successfully.\n");
-                swsusp_free();
         }
+
  Thaw:
         thaw_processes();
  Finish:
diff --git a/kernel/power/main.c b/kernel/power/main.c
index f710e36930cc..347d2cc88cd0 100644
--- a/kernel/power/main.c
+++ b/kernel/power/main.c
@@ -11,6 +11,7 @@
 #include <linux/kobject.h>
 #include <linux/string.h>
 #include <linux/resume-trace.h>
+#include <linux/workqueue.h>
 
 #include "power.h"
 
@@ -217,8 +218,24 @@ static struct attribute_group attr_group = {
         .attrs = g,
 };
 
+#ifdef CONFIG_PM_RUNTIME
+struct workqueue_struct *pm_wq;
+
+static int __init pm_start_workqueue(void)
+{
+        pm_wq = create_freezeable_workqueue("pm");
+
+        return pm_wq ? 0 : -ENOMEM;
+}
+#else
+static inline int pm_start_workqueue(void) { return 0; }
+#endif
+
 static int __init pm_init(void)
 {
+        int error = pm_start_workqueue();
+        if (error)
+                return error;
         power_kobj = kobject_create_and_add("power", NULL);
         if (!power_kobj)
                 return -ENOMEM;
diff --git a/kernel/power/power.h b/kernel/power/power.h
index 26d5a26f82e3..46c5a26630a3 100644
--- a/kernel/power/power.h
+++ b/kernel/power/power.h
@@ -74,7 +74,7 @@ extern asmlinkage int swsusp_arch_resume(void);
 
 extern int create_basic_memory_bitmaps(void);
 extern void free_basic_memory_bitmaps(void);
-extern int swsusp_shrink_memory(void);
+extern int hibernate_preallocate_memory(void);
 
 /**
  *      Auxiliary structure used for reading the snapshot image data and
diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c
index 523a451b45d3..97955b0e44f4 100644
--- a/kernel/power/snapshot.c
+++ b/kernel/power/snapshot.c
@@ -233,7 +233,7 @@ static void *chain_alloc(struct chain_allocator *ca, unsigned int size)
 
 #define BM_END_OF_MAP   (~0UL)
 
-#define BM_BITS_PER_BLOCK       (PAGE_SIZE << 3)
+#define BM_BITS_PER_BLOCK       (PAGE_SIZE * BITS_PER_BYTE)
 
 struct bm_block {
         struct list_head hook;  /* hook into a list of bitmap blocks */
@@ -275,7 +275,7 @@ static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free);
 
 /**
  *      create_bm_block_list - create a list of block bitmap objects
- *      @nr_blocks - number of blocks to allocate
+ *      @pages - number of pages to track
  *      @list - list to put the allocated blocks into
  *      @ca - chain allocator to be used for allocating memory
  */
@@ -853,7 +853,7 @@ static unsigned int count_highmem_pages(void)
         struct zone *zone;
         unsigned int n = 0;
 
-        for_each_zone(zone) {
+        for_each_populated_zone(zone) {
                 unsigned long pfn, max_zone_pfn;
 
                 if (!is_highmem(zone))
@@ -916,7 +916,7 @@ static unsigned int count_data_pages(void)
         unsigned long pfn, max_zone_pfn;
         unsigned int n = 0;
 
-        for_each_zone(zone) {
+        for_each_populated_zone(zone) {
                 if (is_highmem(zone))
                         continue;
 
@@ -1010,7 +1010,7 @@ copy_data_pages(struct memory_bitmap *copy_bm, struct memory_bitmap *orig_bm)
         struct zone *zone;
         unsigned long pfn;
 
-        for_each_zone(zone) {
+        for_each_populated_zone(zone) {
                 unsigned long max_zone_pfn;
 
                 mark_free_pages(zone);
@@ -1033,6 +1033,25 @@ copy_data_pages(struct memory_bitmap *copy_bm, struct memory_bitmap *orig_bm)
 static unsigned int nr_copy_pages;
 /* Number of pages needed for saving the original pfns of the image pages */
 static unsigned int nr_meta_pages;
+/*
+ * Numbers of normal and highmem page frames allocated for hibernation image
+ * before suspending devices.
+ */
+unsigned int alloc_normal, alloc_highmem;
+/*
+ * Memory bitmap used for marking saveable pages (during hibernation) or
+ * hibernation image pages (during restore)
+ */
+static struct memory_bitmap orig_bm;
+/*
+ * Memory bitmap used during hibernation for marking allocated page frames that
+ * will contain copies of saveable pages.  During restore it is initially used
+ * for marking hibernation image pages, but then the set bits from it are
+ * duplicated in @orig_bm and it is released.  On highmem systems it is next
+ * used for marking "safe" highmem pages, but it has to be reinitialized for
+ * this purpose.
+ */
+static struct memory_bitmap copy_bm;
 
 /**
  *      swsusp_free - free pages allocated for the suspend.
@@ -1046,7 +1065,7 @@ void swsusp_free(void)
         struct zone *zone;
         unsigned long pfn, max_zone_pfn;
 
-        for_each_zone(zone) {
+        for_each_populated_zone(zone) {
                 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
                 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
                         if (pfn_valid(pfn)) {
@@ -1064,74 +1083,286 @@ void swsusp_free(void)
         nr_meta_pages = 0;
         restore_pblist = NULL;
         buffer = NULL;
+        alloc_normal = 0;
+        alloc_highmem = 0;
 }
 
+/* Helper functions used for the shrinking of memory. */
+
+#define GFP_IMAGE       (GFP_KERNEL | __GFP_NOWARN)
+
 /**
- *      swsusp_shrink_memory -  Try to free as much memory as needed
- *
- *      ... but do not OOM-kill anyone
+ * preallocate_image_pages - Allocate a number of pages for hibernation image
+ * @nr_pages: Number of page frames to allocate.
+ * @mask: GFP flags to use for the allocation.
  *
- *      Notice: all userland should be stopped before it is called, or
- *      livelock is possible.
+ * Return value: Number of page frames actually allocated
+ */
+static unsigned long preallocate_image_pages(unsigned long nr_pages, gfp_t mask)
+{
+        unsigned long nr_alloc = 0;
+
+        while (nr_pages > 0) {
+                struct page *page;
+
+                page = alloc_image_page(mask);
+                if (!page)
+                        break;
+                memory_bm_set_bit(&copy_bm, page_to_pfn(page));
+                if (PageHighMem(page))
+                        alloc_highmem++;
+                else
+                        alloc_normal++;
+                nr_pages--;
+                nr_alloc++;
+        }
+
+        return nr_alloc;
+}
+
+static unsigned long preallocate_image_memory(unsigned long nr_pages)
+{
+        return preallocate_image_pages(nr_pages, GFP_IMAGE);
+}
+
+#ifdef CONFIG_HIGHMEM
+static unsigned long preallocate_image_highmem(unsigned long nr_pages)
+{
+        return preallocate_image_pages(nr_pages, GFP_IMAGE | __GFP_HIGHMEM);
+}
+
+/**
+ *  __fraction - Compute (an approximation of) x * (multiplier / base)
  */
+static unsigned long __fraction(u64 x, u64 multiplier, u64 base)
+{
+        x *= multiplier;
+        do_div(x, base);
+        return (unsigned long)x;
+}
+
+static unsigned long preallocate_highmem_fraction(unsigned long nr_pages,
+                                                unsigned long highmem,
+                                                unsigned long total)
+{
+        unsigned long alloc = __fraction(nr_pages, highmem, total);
 
-#define SHRINK_BITE     10000
-static inline unsigned long __shrink_memory(long tmp)
+        return preallocate_image_pages(alloc, GFP_IMAGE | __GFP_HIGHMEM);
+}
+#else /* CONFIG_HIGHMEM */
+static inline unsigned long preallocate_image_highmem(unsigned long nr_pages)
 {
-        if (tmp > SHRINK_BITE)
-                tmp = SHRINK_BITE;
-        return shrink_all_memory(tmp);
+        return 0;
 }
 
-int swsusp_shrink_memory(void)
+static inline unsigned long preallocate_highmem_fraction(unsigned long nr_pages,
+                                                unsigned long highmem,
+                                                unsigned long total)
+{
+        return 0;
+}
+#endif /* CONFIG_HIGHMEM */
+
+/**
+ * free_unnecessary_pages - Release preallocated pages not needed for the image
+ */
+static void free_unnecessary_pages(void)
+{
+        unsigned long save_highmem, to_free_normal, to_free_highmem;
+
+        to_free_normal = alloc_normal - count_data_pages();
+        save_highmem = count_highmem_pages();
+        if (alloc_highmem > save_highmem) {
+                to_free_highmem = alloc_highmem - save_highmem;
+        } else {
+                to_free_highmem = 0;
+                to_free_normal -= save_highmem - alloc_highmem;
+        }
+
+        memory_bm_position_reset(&copy_bm);
+
+        while (to_free_normal > 0 && to_free_highmem > 0) {
+                unsigned long pfn = memory_bm_next_pfn(&copy_bm);
+                struct page *page = pfn_to_page(pfn);
+
+                if (PageHighMem(page)) {
+                        if (!to_free_highmem)
+                                continue;
+                        to_free_highmem--;
+                        alloc_highmem--;
+                } else {
+                        if (!to_free_normal)
+                                continue;
+                        to_free_normal--;
+                        alloc_normal--;
+                }
+                memory_bm_clear_bit(&copy_bm, pfn);
+                swsusp_unset_page_forbidden(page);
+                swsusp_unset_page_free(page);
+                __free_page(page);
+        }
+}
+
+/**
+ * minimum_image_size - Estimate the minimum acceptable size of an image
+ * @saveable: Number of saveable pages in the system.
+ *
+ * We want to avoid attempting to free too much memory too hard, so estimate the
+ * minimum acceptable size of a hibernation image to use as the lower limit for
+ * preallocating memory.
+ *
+ * We assume that the minimum image size should be proportional to
+ *
+ * [number of saveable pages] - [number of pages that can be freed in theory]
+ *
+ * where the second term is the sum of (1) reclaimable slab pages, (2) active
+ * and (3) inactive anonymouns pages, (4) active and (5) inactive file pages,
+ * minus mapped file pages.
+ */
+static unsigned long minimum_image_size(unsigned long saveable)
+{
+        unsigned long size;
+
+        size = global_page_state(NR_SLAB_RECLAIMABLE)
+                + global_page_state(NR_ACTIVE_ANON)
+                + global_page_state(NR_INACTIVE_ANON)
+                + global_page_state(NR_ACTIVE_FILE)
+                + global_page_state(NR_INACTIVE_FILE)
+                - global_page_state(NR_FILE_MAPPED);
+
+        return saveable <= size ? 0 : saveable - size;
+}
+
+/**
+ * hibernate_preallocate_memory - Preallocate memory for hibernation image
+ *
+ * To create a hibernation image it is necessary to make a copy of every page
+ * frame in use.  We also need a number of page frames to be free during
+ * hibernation for allocations made while saving the image and for device
+ * drivers, in case they need to allocate memory from their hibernation
+ * callbacks (these two numbers are given by PAGES_FOR_IO and SPARE_PAGES,
+ * respectively, both of which are rough estimates).  To make this happen, we
+ * compute the total number of available page frames and allocate at least
+ *
+ * ([page frames total] + PAGES_FOR_IO + [metadata pages]) / 2 + 2 * SPARE_PAGES
+ *
+ * of them, which corresponds to the maximum size of a hibernation image.
+ *
+ * If image_size is set below the number following from the above formula,
+ * the preallocation of memory is continued until the total number of saveable
+ * pages in the system is below the requested image size or the minimum
+ * acceptable image size returned by minimum_image_size(), whichever is greater.
+ */
+int hibernate_preallocate_memory(void)
 {
-        long tmp;
         struct zone *zone;
-        unsigned long pages = 0;
-        unsigned int i = 0;
-        char *p = "-\\|/";
+        unsigned long saveable, size, max_size, count, highmem, pages = 0;
+        unsigned long alloc, save_highmem, pages_highmem;
         struct timeval start, stop;
+        int error;
 
-        printk(KERN_INFO "PM: Shrinking memory...  ");
+        printk(KERN_INFO "PM: Preallocating image memory... ");
         do_gettimeofday(&start);
-        do {
-                long size, highmem_size;
-
-                highmem_size = count_highmem_pages();
-                size = count_data_pages() + PAGES_FOR_IO + SPARE_PAGES;
-                tmp = size;
-                size += highmem_size;
-                for_each_populated_zone(zone) {
-                        tmp += snapshot_additional_pages(zone);
-                        if (is_highmem(zone)) {
-                                highmem_size -=
-                                        zone_page_state(zone, NR_FREE_PAGES);
-                        } else {
-                                tmp -= zone_page_state(zone, NR_FREE_PAGES);
-                                tmp += zone->lowmem_reserve[ZONE_NORMAL];
-                        }
-                }
 
-                if (highmem_size < 0)
-                        highmem_size = 0;
+        error = memory_bm_create(&orig_bm, GFP_IMAGE, PG_ANY);
+        if (error)
+                goto err_out;
 
-                tmp += highmem_size;
-                if (tmp > 0) {
-                        tmp = __shrink_memory(tmp);
-                        if (!tmp)
-                                return -ENOMEM;
-                        pages += tmp;
-                } else if (size > image_size / PAGE_SIZE) {
-                        tmp = __shrink_memory(size - (image_size / PAGE_SIZE));
-                        pages += tmp;
-                }
-                printk("\b%c", p[i++%4]);
-        } while (tmp > 0);
+        error = memory_bm_create(&copy_bm, GFP_IMAGE, PG_ANY);
+        if (error)
+                goto err_out;
+
+        alloc_normal = 0;
+        alloc_highmem = 0;
+
+        /* Count the number of saveable data pages. */
+        save_highmem = count_highmem_pages();
+        saveable = count_data_pages();
+
+        /*
+         * Compute the total number of page frames we can use (count) and the
+         * number of pages needed for image metadata (size).
+         */
+        count = saveable;
+        saveable += save_highmem;
+        highmem = save_highmem;
+        size = 0;
+        for_each_populated_zone(zone) {
+                size += snapshot_additional_pages(zone);
+                if (is_highmem(zone))
+                        highmem += zone_page_state(zone, NR_FREE_PAGES);
+                else
+                        count += zone_page_state(zone, NR_FREE_PAGES);
+        }
+        count += highmem;
+        count -= totalreserve_pages;
+
+        /* Compute the maximum number of saveable pages to leave in memory. */
+        max_size = (count - (size + PAGES_FOR_IO)) / 2 - 2 * SPARE_PAGES;
+        size = DIV_ROUND_UP(image_size, PAGE_SIZE);
+        if (size > max_size)
+                size = max_size;
+        /*
+         * If the maximum is not less than the current number of saveable pages
+         * in memory, allocate page frames for the image and we're done.
+         */
+        if (size >= saveable) {
+                pages = preallocate_image_highmem(save_highmem);
+                pages += preallocate_image_memory(saveable - pages);
+                goto out;
+        }
+
+        /* Estimate the minimum size of the image. */
+        pages = minimum_image_size(saveable);
+        if (size < pages)
+                size = min_t(unsigned long, pages, max_size);
+
+        /*
+         * Let the memory management subsystem know that we're going to need a
+         * large number of page frames to allocate and make it free some memory.
+         * NOTE: If this is not done, performance will be hurt badly in some
+         * test cases.
+         */
+        shrink_all_memory(saveable - size);
+
+        /*
+         * The number of saveable pages in memory was too high, so apply some
+         * pressure to decrease it.  First, make room for the largest possible
+         * image and fail if that doesn't work.  Next, try to decrease the size
+         * of the image as much as indicated by 'size' using allocations from
+         * highmem and non-highmem zones separately.
+         */
+        pages_highmem = preallocate_image_highmem(highmem / 2);
+        alloc = (count - max_size) - pages_highmem;
+        pages = preallocate_image_memory(alloc);
+        if (pages < alloc)
+                goto err_out;
+        size = max_size - size;
+        alloc = size;
+        size = preallocate_highmem_fraction(size, highmem, count);
+        pages_highmem += size;
+        alloc -= size;
+        pages += preallocate_image_memory(alloc);
+        pages += pages_highmem;
+
+        /*
+         * We only need as many page frames for the image as there are saveable
+         * pages in memory, but we have allocated more.  Release the excessive
+         * ones now.
+         */
+        free_unnecessary_pages();
+
+ out:
         do_gettimeofday(&stop);
-        printk("\bdone (%lu pages freed)\n", pages);
-        swsusp_show_speed(&start, &stop, pages, "Freed");
+        printk(KERN_CONT "done (allocated %lu pages)\n", pages);
+        swsusp_show_speed(&start, &stop, pages, "Allocated");
 
         return 0;
+
+ err_out:
+        printk(KERN_CONT "\n");
+        swsusp_free();
+        return -ENOMEM;
 }
 
 #ifdef CONFIG_HIGHMEM
@@ -1142,7 +1373,7 @@ int swsusp_shrink_memory(void)
 
 static unsigned int count_pages_for_highmem(unsigned int nr_highmem)
 {
-        unsigned int free_highmem = count_free_highmem_pages();
+        unsigned int free_highmem = count_free_highmem_pages() + alloc_highmem;
 
         if (free_highmem >= nr_highmem)
                 nr_highmem = 0;
@@ -1164,19 +1395,17 @@ count_pages_for_highmem(unsigned int nr_highmem) { return 0; }
 static int enough_free_mem(unsigned int nr_pages, unsigned int nr_highmem)
 {
         struct zone *zone;
-        unsigned int free = 0, meta = 0;
+        unsigned int free = alloc_normal;
 
-        for_each_zone(zone) {
-                meta += snapshot_additional_pages(zone);
+        for_each_populated_zone(zone)
                 if (!is_highmem(zone))
                         free += zone_page_state(zone, NR_FREE_PAGES);
-        }
 
         nr_pages += count_pages_for_highmem(nr_highmem);
-        pr_debug("PM: Normal pages needed: %u + %u + %u, available pages: %u\n",
-                nr_pages, PAGES_FOR_IO, meta, free);
+        pr_debug("PM: Normal pages needed: %u + %u, available pages: %u\n",
+                nr_pages, PAGES_FOR_IO, free);
 
-        return free > nr_pages + PAGES_FOR_IO + meta;
+        return free > nr_pages + PAGES_FOR_IO;
 }
 
 #ifdef CONFIG_HIGHMEM
@@ -1198,7 +1427,7 @@ static inline int get_highmem_buffer(int safe_needed)
  */
 
 static inline unsigned int
-alloc_highmem_image_pages(struct memory_bitmap *bm, unsigned int nr_highmem)
+alloc_highmem_pages(struct memory_bitmap *bm, unsigned int nr_highmem)
 {
         unsigned int to_alloc = count_free_highmem_pages();
 
@@ -1218,7 +1447,7 @@ alloc_highmem_image_pages(struct memory_bitmap *bm, unsigned int nr_highmem)
 static inline int get_highmem_buffer(int safe_needed) { return 0; }
 
 static inline unsigned int
-alloc_highmem_image_pages(struct memory_bitmap *bm, unsigned int n) { return 0; }
+alloc_highmem_pages(struct memory_bitmap *bm, unsigned int n) { return 0; }
 #endif /* CONFIG_HIGHMEM */
 
 /**
@@ -1237,51 +1466,36 @@ static int
 swsusp_alloc(struct memory_bitmap *orig_bm, struct memory_bitmap *copy_bm,
                 unsigned int nr_pages, unsigned int nr_highmem)
 {
-        int error;
-
-        error = memory_bm_create(orig_bm, GFP_ATOMIC | __GFP_COLD, PG_ANY);
-        if (error)
-                goto Free;
-
-        error = memory_bm_create(copy_bm, GFP_ATOMIC | __GFP_COLD, PG_ANY);
-        if (error)
-                goto Free;
+        int error = 0;
 
         if (nr_highmem > 0) {
                 error = get_highmem_buffer(PG_ANY);
                 if (error)
-                        goto Free;
-
-                nr_pages += alloc_highmem_image_pages(copy_bm, nr_highmem);
+                        goto err_out;
+                if (nr_highmem > alloc_highmem) {
+                        nr_highmem -= alloc_highmem;
+                        nr_pages += alloc_highmem_pages(copy_bm, nr_highmem);
+                }
         }
-        while (nr_pages-- > 0) {
-                struct page *page = alloc_image_page(GFP_ATOMIC | __GFP_COLD);
-
-                if (!page)
-                        goto Free;
+        if (nr_pages > alloc_normal) {
+                nr_pages -= alloc_normal;
+                while (nr_pages-- > 0) {
+                        struct page *page;
 
-                memory_bm_set_bit(copy_bm, page_to_pfn(page));
+                        page = alloc_image_page(GFP_ATOMIC | __GFP_COLD);
+                        if (!page)
+                                goto err_out;
+                        memory_bm_set_bit(copy_bm, page_to_pfn(page));
+                }
         }
+
         return 0;
 
- Free:
+ err_out:
         swsusp_free();
-        return -ENOMEM;
+        return error;
 }
 
-/* Memory bitmap used for marking saveable pages (during suspend) or the
- * suspend image pages (during resume)
- */
-static struct memory_bitmap orig_bm;
-/* Memory bitmap used on suspend for marking allocated pages that will contain
- * the copies of saveable pages.  During resume it is initially used for
- * marking the suspend image pages, but then its set bits are duplicated in
- * @orig_bm and it is released.  Next, on systems with high memory, it may be
- * used for marking "safe" highmem pages, but it has to be reinitialized for
- * this purpose.
- */
-static struct memory_bitmap copy_bm;
-
 asmlinkage int swsusp_save(void)
 {
         unsigned int nr_pages, nr_highmem;
@@ -1474,7 +1688,7 @@ static int mark_unsafe_pages(struct memory_bitmap *bm)
         unsigned long pfn, max_zone_pfn;
 
         /* Clear page flags */
-        for_each_zone(zone) {
+        for_each_populated_zone(zone) {
                 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
                 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
                         if (pfn_valid(pfn))
diff --git a/kernel/printk.c b/kernel/printk.c
index b4d97b54c1ec..602033acd6c7 100644
--- a/kernel/printk.c
+++ b/kernel/printk.c
@@ -37,6 +37,12 @@
 #include <asm/uaccess.h>
 
 /*
+ * for_each_console() allows you to iterate on each console
+ */
+#define for_each_console(con) \
+        for (con = console_drivers; con != NULL; con = con->next)
+
+/*
  * Architectures can override it:
  */
 void asmlinkage __attribute__((weak)) early_printk(const char *fmt, ...)
@@ -61,6 +67,8 @@ int console_printk[4] = {
         DEFAULT_CONSOLE_LOGLEVEL,       /* default_console_loglevel */
 };
 
+static int saved_console_loglevel = -1;
+
 /*
  * Low level drivers may need that to know if they can schedule in
  * their unblank() callback or not. So let's export it.
@@ -372,10 +380,15 @@ int do_syslog(int type, char __user *buf, int len)
                 logged_chars = 0;
                 break;
         case 6:         /* Disable logging to console */
+                if (saved_console_loglevel == -1)
+                        saved_console_loglevel = console_loglevel;
                 console_loglevel = minimum_console_loglevel;
                 break;
         case 7:         /* Enable logging to console */
-                console_loglevel = default_console_loglevel;
+                if (saved_console_loglevel != -1) {
+                        console_loglevel = saved_console_loglevel;
+                        saved_console_loglevel = -1;
+                }
                 break;
         case 8:         /* Set level of messages printed to console */
                 error = -EINVAL;
@@ -384,6 +397,8 @@ int do_syslog(int type, char __user *buf, int len)
                 if (len < minimum_console_loglevel)
                         len = minimum_console_loglevel;
                 console_loglevel = len;
+                /* Implicitly re-enable logging to console */
+                saved_console_loglevel = -1;
                 error = 0;
                 break;
         case 9:         /* Number of chars in the log buffer */
@@ -412,7 +427,7 @@ static void __call_console_drivers(unsigned start, unsigned end)
 {
         struct console *con;
 
-        for (con = console_drivers; con; con = con->next) {
+        for_each_console(con) {
                 if ((con->flags & CON_ENABLED) && con->write &&
                                 (cpu_online(smp_processor_id()) ||
                                 (con->flags & CON_ANYTIME)))
@@ -544,7 +559,7 @@ static int have_callable_console(void)
 {
         struct console *con;
 
-        for (con = console_drivers; con; con = con->next)
+        for_each_console(con)
                 if (con->flags & CON_ANYTIME)
                         return 1;
 
@@ -1060,12 +1075,6 @@ void __sched console_conditional_schedule(void)
 }
 EXPORT_SYMBOL(console_conditional_schedule);
 
-void console_print(const char *s)
-{
-        printk(KERN_EMERG "%s", s);
-}
-EXPORT_SYMBOL(console_print);
-
 void console_unblank(void)
 {
         struct console *c;
@@ -1082,7 +1091,7 @@ void console_unblank(void)
 
         console_locked = 1;
         console_may_schedule = 0;
-        for (c = console_drivers; c != NULL; c = c->next)
+        for_each_console(c)
                 if ((c->flags & CON_ENABLED) && c->unblank)
                         c->unblank();
         release_console_sem();
@@ -1097,7 +1106,7 @@ struct tty_driver *console_device(int *index)
         struct tty_driver *driver = NULL;
 
         acquire_console_sem();
-        for (c = console_drivers; c != NULL; c = c->next) {
+        for_each_console(c) {
                 if (!c->device)
                         continue;
                 driver = c->device(c, index);
@@ -1134,25 +1143,49 @@ EXPORT_SYMBOL(console_start);
  * to register the console printing procedure with printk() and to
  * print any messages that were printed by the kernel before the
  * console driver was initialized.
+ *
+ * This can happen pretty early during the boot process (because of
+ * early_printk) - sometimes before setup_arch() completes - be careful
+ * of what kernel features are used - they may not be initialised yet.
+ *
+ * There are two types of consoles - bootconsoles (early_printk) and
+ * "real" consoles (everything which is not a bootconsole) which are
+ * handled differently.
+ *  - Any number of bootconsoles can be registered at any time.
+ *  - As soon as a "real" console is registered, all bootconsoles
+ *    will be unregistered automatically.
+ *  - Once a "real" console is registered, any attempt to register a
+ *    bootconsoles will be rejected
  */
-void register_console(struct console *console)
+void register_console(struct console *newcon)
 {
         int i;
         unsigned long flags;
-        struct console *bootconsole = NULL;
+        struct console *bcon = NULL;
 
-        if (console_drivers) {
-                if (console->flags & CON_BOOT)
-                        return;
-                if (console_drivers->flags & CON_BOOT)
-                        bootconsole = console_drivers;
+        /*
+         * before we register a new CON_BOOT console, make sure we don't
+         * already have a valid console
+         */
+        if (console_drivers && newcon->flags & CON_BOOT) {
+                /* find the last or real console */
+                for_each_console(bcon) {
+                        if (!(bcon->flags & CON_BOOT)) {
+                                printk(KERN_INFO "Too late to register bootconsole %s%d\n",
+                                        newcon->name, newcon->index);
+                                return;
+                        }
+                }
         }
 
-        if (preferred_console < 0 || bootconsole || !console_drivers)
+        if (console_drivers && console_drivers->flags & CON_BOOT)
+                bcon = console_drivers;
+
+        if (preferred_console < 0 || bcon || !console_drivers)
                 preferred_console = selected_console;
 
-        if (console->early_setup)
-                console->early_setup();
+        if (newcon->early_setup)
+                newcon->early_setup();
 
         /*
          *      See if we want to use this console driver. If we
@@ -1160,13 +1193,13 @@ void register_console(struct console *console)
          *      that registers here.
          */
         if (preferred_console < 0) {
-                if (console->index < 0)
-                        console->index = 0;
-                if (console->setup == NULL ||
-                    console->setup(console, NULL) == 0) {
-                        console->flags |= CON_ENABLED;
-                        if (console->device) {
-                                console->flags |= CON_CONSDEV;
+                if (newcon->index < 0)
+                        newcon->index = 0;
+                if (newcon->setup == NULL ||
+                    newcon->setup(newcon, NULL) == 0) {
+                        newcon->flags |= CON_ENABLED;
+                        if (newcon->device) {
+                                newcon->flags |= CON_CONSDEV;
                                 preferred_console = 0;
                         }
                 }
@@ -1178,64 +1211,62 @@ void register_console(struct console *console)
          */
         for (i = 0; i < MAX_CMDLINECONSOLES && console_cmdline[i].name[0];
                         i++) {
-                if (strcmp(console_cmdline[i].name, console->name) != 0)
+                if (strcmp(console_cmdline[i].name, newcon->name) != 0)
                         continue;
-                if (console->index >= 0 &&
-                    console->index != console_cmdline[i].index)
+                if (newcon->index >= 0 &&
+                    newcon->index != console_cmdline[i].index)
                         continue;
-                if (console->index < 0)
-                        console->index = console_cmdline[i].index;
+                if (newcon->index < 0)
+                        newcon->index = console_cmdline[i].index;
 #ifdef CONFIG_A11Y_BRAILLE_CONSOLE
                 if (console_cmdline[i].brl_options) {
-                        console->flags |= CON_BRL;
-                        braille_register_console(console,
+                        newcon->flags |= CON_BRL;
+                        braille_register_console(newcon,
                                         console_cmdline[i].index,
                                         console_cmdline[i].options,
                                         console_cmdline[i].brl_options);
                         return;
                 }
 #endif
-                if (console->setup &&
-                    console->setup(console, console_cmdline[i].options) != 0)
+                if (newcon->setup &&
+                    newcon->setup(newcon, console_cmdline[i].options) != 0)
                         break;
-                console->flags |= CON_ENABLED;
-                console->index = console_cmdline[i].index;
+                newcon->flags |= CON_ENABLED;
+                newcon->index = console_cmdline[i].index;
                 if (i == selected_console) {
-                        console->flags |= CON_CONSDEV;
+                        newcon->flags |= CON_CONSDEV;
                         preferred_console = selected_console;
                 }
                 break;
         }
 
-        if (!(console->flags & CON_ENABLED))
+        if (!(newcon->flags & CON_ENABLED))
                 return;
 
-        if (bootconsole && (console->flags & CON_CONSDEV)) {
-                printk(KERN_INFO "console handover: boot [%s%d] -> real [%s%d]\n",
-                       bootconsole->name, bootconsole->index,
-                       console->name, console->index);
-                unregister_console(bootconsole);
-                console->flags &= ~CON_PRINTBUFFER;
-        } else {
-                printk(KERN_INFO "console [%s%d] enabled\n",
-                       console->name, console->index);
-        }
+        /*
+         * If we have a bootconsole, and are switching to a real console,
+         * don't print everything out again, since when the boot console, and
+         * the real console are the same physical device, it's annoying to
+         * see the beginning boot messages twice
+         */
+        if (bcon && ((newcon->flags & (CON_CONSDEV | CON_BOOT)) == CON_CONSDEV))
+                newcon->flags &= ~CON_PRINTBUFFER;
 
         /*
          *      Put this console in the list - keep the
          *      preferred driver at the head of the list.
          */
         acquire_console_sem();
-        if ((console->flags & CON_CONSDEV) || console_drivers == NULL) {
-                console->next = console_drivers;
-                console_drivers = console;
-                if (console->next)
-                        console->next->flags &= ~CON_CONSDEV;
+        if ((newcon->flags & CON_CONSDEV) || console_drivers == NULL) {
+                newcon->next = console_drivers;
+                console_drivers = newcon;
+                if (newcon->next)
+                        newcon->next->flags &= ~CON_CONSDEV;
         } else {
-                console->next = console_drivers->next;
-                console_drivers->next = console;
+                newcon->next = console_drivers->next;
+                console_drivers->next = newcon;
         }
-        if (console->flags & CON_PRINTBUFFER) {
+        if (newcon->flags & CON_PRINTBUFFER) {
                 /*
                  * release_console_sem() will print out the buffered messages
                  * for us.
@@ -1245,6 +1276,28 @@ void register_console(struct console *console)
                 spin_unlock_irqrestore(&logbuf_lock, flags);
         }
         release_console_sem();
+
+        /*
+         * By unregistering the bootconsoles after we enable the real console
+         * we get the "console xxx enabled" message on all the consoles -
+         * boot consoles, real consoles, etc - this is to ensure that end
+         * users know there might be something in the kernel's log buffer that
+         * went to the bootconsole (that they do not see on the real console)
+         */
+        if (bcon && ((newcon->flags & (CON_CONSDEV | CON_BOOT)) == CON_CONSDEV)) {
+                /* we need to iterate through twice, to make sure we print
+                 * everything out, before we unregister the console(s)
+                 */
+                printk(KERN_INFO "console [%s%d] enabled, bootconsole disabled\n",
+                        newcon->name, newcon->index);
+                for_each_console(bcon)
+                        if (bcon->flags & CON_BOOT)
+                                unregister_console(bcon);
+        } else {
+                printk(KERN_INFO "%sconsole [%s%d] enabled\n",
+                        (newcon->flags & CON_BOOT) ? "boot" : "" ,
+                        newcon->name, newcon->index);
+        }
 }
 EXPORT_SYMBOL(register_console);
 
@@ -1287,11 +1340,13 @@ EXPORT_SYMBOL(unregister_console);
 
 static int __init disable_boot_consoles(void)
 {
-        if (console_drivers != NULL) {
-                if (console_drivers->flags & CON_BOOT) {
+        struct console *con;
+
+        for_each_console(con) {
+                if (con->flags & CON_BOOT) {
                         printk(KERN_INFO "turn off boot console %s%d\n",
-                                console_drivers->name, console_drivers->index);
-                        return unregister_console(console_drivers);
+                                con->name, con->index);
+                        unregister_console(con);
                 }
         }
         return 0;
diff --git a/kernel/ptrace.c b/kernel/ptrace.c
index 082c320e4dbf..307c285af59e 100644
--- a/kernel/ptrace.c
+++ b/kernel/ptrace.c
@@ -152,7 +152,7 @@ int __ptrace_may_access(struct task_struct *task, unsigned int mode)
         if (!dumpable && !capable(CAP_SYS_PTRACE))
                 return -EPERM;
 
-        return security_ptrace_may_access(task, mode);
+        return security_ptrace_access_check(task, mode);
 }
 
 bool ptrace_may_access(struct task_struct *task, unsigned int mode)
diff --git a/kernel/rcuclassic.c b/kernel/rcuclassic.c
deleted file mode 100644
index 0f2b0b311304..000000000000
--- a/kernel/rcuclassic.c
+++ /dev/null
@@ -1,807 +0,0 @@
-/*
- * Read-Copy Update mechanism for mutual exclusion
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
- *
- * Copyright IBM Corporation, 2001
- *
- * Authors: Dipankar Sarma <dipankar@in.ibm.com>
- *          Manfred Spraul <manfred@colorfullife.com>
- *
- * Based on the original work by Paul McKenney <paulmck@us.ibm.com>
- * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
- * Papers:
- * http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf
- * http://lse.sourceforge.net/locking/rclock_OLS.2001.05.01c.sc.pdf (OLS2001)
- *
- * For detailed explanation of Read-Copy Update mechanism see -
- *              Documentation/RCU
- *
- */
-#include <linux/types.h>
-#include <linux/kernel.h>
-#include <linux/init.h>
-#include <linux/spinlock.h>
-#include <linux/smp.h>
-#include <linux/rcupdate.h>
-#include <linux/interrupt.h>
-#include <linux/sched.h>
-#include <asm/atomic.h>
-#include <linux/bitops.h>
-#include <linux/module.h>
-#include <linux/completion.h>
-#include <linux/moduleparam.h>
-#include <linux/percpu.h>
-#include <linux/notifier.h>
-#include <linux/cpu.h>
-#include <linux/mutex.h>
-#include <linux/time.h>
-
-#ifdef CONFIG_DEBUG_LOCK_ALLOC
-static struct lock_class_key rcu_lock_key;
-struct lockdep_map rcu_lock_map =
-        STATIC_LOCKDEP_MAP_INIT("rcu_read_lock", &rcu_lock_key);
-EXPORT_SYMBOL_GPL(rcu_lock_map);
-#endif
-
-
-/* Definition for rcupdate control block. */
-static struct rcu_ctrlblk rcu_ctrlblk = {
-        .cur = -300,
-        .completed = -300,
-        .pending = -300,
-        .lock = __SPIN_LOCK_UNLOCKED(&rcu_ctrlblk.lock),
-        .cpumask = CPU_BITS_NONE,
-};
-
-static struct rcu_ctrlblk rcu_bh_ctrlblk = {
-        .cur = -300,
-        .completed = -300,
-        .pending = -300,
-        .lock = __SPIN_LOCK_UNLOCKED(&rcu_bh_ctrlblk.lock),
-        .cpumask = CPU_BITS_NONE,
-};
-
-static DEFINE_PER_CPU(struct rcu_data, rcu_data);
-static DEFINE_PER_CPU(struct rcu_data, rcu_bh_data);
-
-/*
- * Increment the quiescent state counter.
- * The counter is a bit degenerated: We do not need to know
- * how many quiescent states passed, just if there was at least
- * one since the start of the grace period. Thus just a flag.
- */
-void rcu_qsctr_inc(int cpu)
-{
-        struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
-        rdp->passed_quiesc = 1;
-}
-
-void rcu_bh_qsctr_inc(int cpu)
-{
-        struct rcu_data *rdp = &per_cpu(rcu_bh_data, cpu);
-        rdp->passed_quiesc = 1;
-}
-
-static int blimit = 10;
-static int qhimark = 10000;
-static int qlowmark = 100;
-
-#ifdef CONFIG_SMP
-static void force_quiescent_state(struct rcu_data *rdp,
-                        struct rcu_ctrlblk *rcp)
-{
-        int cpu;
-        unsigned long flags;
-
-        set_need_resched();
-        spin_lock_irqsave(&rcp->lock, flags);
-        if (unlikely(!rcp->signaled)) {
-                rcp->signaled = 1;
-                /*
-                 * Don't send IPI to itself. With irqs disabled,
-                 * rdp->cpu is the current cpu.
-                 *
-                 * cpu_online_mask is updated by the _cpu_down()
-                 * using __stop_machine(). Since we're in irqs disabled
-                 * section, __stop_machine() is not exectuting, hence
-                 * the cpu_online_mask is stable.
-                 *
-                 * However,  a cpu might have been offlined _just_ before
-                 * we disabled irqs while entering here.
-                 * And rcu subsystem might not yet have handled the CPU_DEAD
-                 * notification, leading to the offlined cpu's bit
-                 * being set in the rcp->cpumask.
-                 *
-                 * Hence cpumask = (rcp->cpumask & cpu_online_mask) to prevent
-                 * sending smp_reschedule() to an offlined CPU.
-                 */
-                for_each_cpu_and(cpu,
-                                  to_cpumask(rcp->cpumask), cpu_online_mask) {
-                        if (cpu != rdp->cpu)
-                                smp_send_reschedule(cpu);
-                }
-        }
-        spin_unlock_irqrestore(&rcp->lock, flags);
-}
-#else
-static inline void force_quiescent_state(struct rcu_data *rdp,
-                        struct rcu_ctrlblk *rcp)
-{
-        set_need_resched();
-}
-#endif
-
-static void __call_rcu(struct rcu_head *head, struct rcu_ctrlblk *rcp,
-                struct rcu_data *rdp)
-{
-        long batch;
-
-        head->next = NULL;
-        smp_mb(); /* Read of rcu->cur must happen after any change by caller. */
-
-        /*
-         * Determine the batch number of this callback.
-         *
-         * Using ACCESS_ONCE to avoid the following error when gcc eliminates
-         * local variable "batch" and emits codes like this:
-         *      1) rdp->batch = rcp->cur + 1 # gets old value
-         *      ......
-         *      2)rcu_batch_after(rcp->cur + 1, rdp->batch) # gets new value
-         * then [*nxttail[0], *nxttail[1]) may contain callbacks
-         * that batch# = rdp->batch, see the comment of struct rcu_data.
-         */
-        batch = ACCESS_ONCE(rcp->cur) + 1;
-
-        if (rdp->nxtlist && rcu_batch_after(batch, rdp->batch)) {
-                /* process callbacks */
-                rdp->nxttail[0] = rdp->nxttail[1];
-                rdp->nxttail[1] = rdp->nxttail[2];
-                if (rcu_batch_after(batch - 1, rdp->batch))
-                        rdp->nxttail[0] = rdp->nxttail[2];
-        }
-
-        rdp->batch = batch;
-        *rdp->nxttail[2] = head;
-        rdp->nxttail[2] = &head->next;
-
-        if (unlikely(++rdp->qlen > qhimark)) {
-                rdp->blimit = INT_MAX;
-                force_quiescent_state(rdp, &rcu_ctrlblk);
-        }
-}
-
-#ifdef CONFIG_RCU_CPU_STALL_DETECTOR
-
-static void record_gp_stall_check_time(struct rcu_ctrlblk *rcp)
-{
-        rcp->gp_start = jiffies;
-        rcp->jiffies_stall = jiffies + RCU_SECONDS_TILL_STALL_CHECK;
-}
-
-static void print_other_cpu_stall(struct rcu_ctrlblk *rcp)
-{
-        int cpu;
-        long delta;
-        unsigned long flags;
-
-        /* Only let one CPU complain about others per time interval. */
-
-        spin_lock_irqsave(&rcp->lock, flags);
-        delta = jiffies - rcp->jiffies_stall;
-        if (delta < 2 || rcp->cur != rcp->completed) {
-                spin_unlock_irqrestore(&rcp->lock, flags);
-                return;
-        }
-        rcp->jiffies_stall = jiffies + RCU_SECONDS_TILL_STALL_RECHECK;
-        spin_unlock_irqrestore(&rcp->lock, flags);
-
-        /* OK, time to rat on our buddy... */
-
-        printk(KERN_ERR "INFO: RCU detected CPU stalls:");
-        for_each_possible_cpu(cpu) {
-                if (cpumask_test_cpu(cpu, to_cpumask(rcp->cpumask)))
-                        printk(" %d", cpu);
-        }
-        printk(" (detected by %d, t=%ld jiffies)\n",
-               smp_processor_id(), (long)(jiffies - rcp->gp_start));
-}
-
-static void print_cpu_stall(struct rcu_ctrlblk *rcp)
-{
-        unsigned long flags;
-
-        printk(KERN_ERR "INFO: RCU detected CPU %d stall (t=%lu/%lu jiffies)\n",
-                        smp_processor_id(), jiffies,
-                        jiffies - rcp->gp_start);
-        dump_stack();
-        spin_lock_irqsave(&rcp->lock, flags);
-        if ((long)(jiffies - rcp->jiffies_stall) >= 0)
-                rcp->jiffies_stall =
-                        jiffies + RCU_SECONDS_TILL_STALL_RECHECK;
-        spin_unlock_irqrestore(&rcp->lock, flags);
-        set_need_resched();  /* kick ourselves to get things going. */
-}
-
-static void check_cpu_stall(struct rcu_ctrlblk *rcp)
-{
-        long delta;
-
-        delta = jiffies - rcp->jiffies_stall;
-        if (cpumask_test_cpu(smp_processor_id(), to_cpumask(rcp->cpumask)) &&
-                delta >= 0) {
-
-                /* We haven't checked in, so go dump stack. */
-                print_cpu_stall(rcp);
-
-        } else if (rcp->cur != rcp->completed && delta >= 2) {
-
-                /* They had two seconds to dump stack, so complain. */
-                print_other_cpu_stall(rcp);
-        }
-}
-
-#else /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
-
-static void record_gp_stall_check_time(struct rcu_ctrlblk *rcp)
-{
-}
-
-static inline void check_cpu_stall(struct rcu_ctrlblk *rcp)
-{
-}
-
-#endif /* #else #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
-
-/**
- * call_rcu - Queue an RCU callback for invocation after a grace period.
- * @head: structure to be used for queueing the RCU updates.
- * @func: actual update function to be invoked after the grace period
- *
- * The update function will be invoked some time after a full grace
- * period elapses, in other words after all currently executing RCU
- * read-side critical sections have completed.  RCU read-side critical
- * sections are delimited by rcu_read_lock() and rcu_read_unlock(),
- * and may be nested.
- */
-void call_rcu(struct rcu_head *head,
-                                void (*func)(struct rcu_head *rcu))
-{
-        unsigned long flags;
-
-        head->func = func;
-        local_irq_save(flags);
-        __call_rcu(head, &rcu_ctrlblk, &__get_cpu_var(rcu_data));
-        local_irq_restore(flags);
-}
-EXPORT_SYMBOL_GPL(call_rcu);
-
-/**
- * call_rcu_bh - Queue an RCU for invocation after a quicker grace period.
- * @head: structure to be used for queueing the RCU updates.
- * @func: actual update function to be invoked after the grace period
- *
- * The update function will be invoked some time after a full grace
- * period elapses, in other words after all currently executing RCU
- * read-side critical sections have completed. call_rcu_bh() assumes
- * that the read-side critical sections end on completion of a softirq
- * handler. This means that read-side critical sections in process
- * context must not be interrupted by softirqs. This interface is to be
- * used when most of the read-side critical sections are in softirq context.
- * RCU read-side critical sections are delimited by rcu_read_lock() and
- * rcu_read_unlock(), * if in interrupt context or rcu_read_lock_bh()
- * and rcu_read_unlock_bh(), if in process context. These may be nested.
- */
-void call_rcu_bh(struct rcu_head *head,
-                                void (*func)(struct rcu_head *rcu))
-{
-        unsigned long flags;
-
-        head->func = func;
-        local_irq_save(flags);
-        __call_rcu(head, &rcu_bh_ctrlblk, &__get_cpu_var(rcu_bh_data));
-        local_irq_restore(flags);
-}
-EXPORT_SYMBOL_GPL(call_rcu_bh);
-
-/*
- * Return the number of RCU batches processed thus far.  Useful
- * for debug and statistics.
- */
-long rcu_batches_completed(void)
-{
-        return rcu_ctrlblk.completed;
-}
-EXPORT_SYMBOL_GPL(rcu_batches_completed);
-
-/*
- * Return the number of RCU batches processed thus far.  Useful
- * for debug and statistics.
- */
-long rcu_batches_completed_bh(void)
-{
-        return rcu_bh_ctrlblk.completed;
-}
-EXPORT_SYMBOL_GPL(rcu_batches_completed_bh);
-
-/* Raises the softirq for processing rcu_callbacks. */
-static inline void raise_rcu_softirq(void)
-{
-        raise_softirq(RCU_SOFTIRQ);
-}
-
-/*
- * Invoke the completed RCU callbacks. They are expected to be in
- * a per-cpu list.
- */
-static void rcu_do_batch(struct rcu_data *rdp)
-{
-        unsigned long flags;
-        struct rcu_head *next, *list;
-        int count = 0;
-
-        list = rdp->donelist;
-        while (list) {
-                next = list->next;
-                prefetch(next);
-                list->func(list);
-                list = next;
-                if (++count >= rdp->blimit)
-                        break;
-        }
-        rdp->donelist = list;
-
-        local_irq_save(flags);
-        rdp->qlen -= count;
-        local_irq_restore(flags);
-        if (rdp->blimit == INT_MAX && rdp->qlen <= qlowmark)
-                rdp->blimit = blimit;
-
-        if (!rdp->donelist)
-                rdp->donetail = &rdp->donelist;
-        else
-                raise_rcu_softirq();
-}
-
-/*
- * Grace period handling:
- * The grace period handling consists out of two steps:
- * - A new grace period is started.
- *   This is done by rcu_start_batch. The start is not broadcasted to
- *   all cpus, they must pick this up by comparing rcp->cur with
- *   rdp->quiescbatch. All cpus are recorded  in the
- *   rcu_ctrlblk.cpumask bitmap.
- * - All cpus must go through a quiescent state.
- *   Since the start of the grace period is not broadcasted, at least two
- *   calls to rcu_check_quiescent_state are required:
- *   The first call just notices that a new grace period is running. The
- *   following calls check if there was a quiescent state since the beginning
- *   of the grace period. If so, it updates rcu_ctrlblk.cpumask. If
- *   the bitmap is empty, then the grace period is completed.
- *   rcu_check_quiescent_state calls rcu_start_batch(0) to start the next grace
- *   period (if necessary).
- */
-
-/*
- * Register a new batch of callbacks, and start it up if there is currently no
- * active batch and the batch to be registered has not already occurred.
- * Caller must hold rcu_ctrlblk.lock.
- */
-static void rcu_start_batch(struct rcu_ctrlblk *rcp)
-{
-        if (rcp->cur != rcp->pending &&
-                        rcp->completed == rcp->cur) {
-                rcp->cur++;
-                record_gp_stall_check_time(rcp);
-
-                /*
-                 * Accessing nohz_cpu_mask before incrementing rcp->cur needs a
-                 * Barrier  Otherwise it can cause tickless idle CPUs to be
-                 * included in rcp->cpumask, which will extend graceperiods
-                 * unnecessarily.
-                 */
-                smp_mb();
-                cpumask_andnot(to_cpumask(rcp->cpumask),
-                               cpu_online_mask, nohz_cpu_mask);
-
-                rcp->signaled = 0;
-        }
-}
-
-/*
- * cpu went through a quiescent state since the beginning of the grace period.
- * Clear it from the cpu mask and complete the grace period if it was the last
- * cpu. Start another grace period if someone has further entries pending
- */
-static void cpu_quiet(int cpu, struct rcu_ctrlblk *rcp)
-{
-        cpumask_clear_cpu(cpu, to_cpumask(rcp->cpumask));
-        if (cpumask_empty(to_cpumask(rcp->cpumask))) {
-                /* batch completed ! */
-                rcp->completed = rcp->cur;
-                rcu_start_batch(rcp);
-        }
-}
-
-/*
- * Check if the cpu has gone through a quiescent state (say context
- * switch). If so and if it already hasn't done so in this RCU
- * quiescent cycle, then indicate that it has done so.
- */
-static void rcu_check_quiescent_state(struct rcu_ctrlblk *rcp,
-                                        struct rcu_data *rdp)
-{
-        unsigned long flags;
-
-        if (rdp->quiescbatch != rcp->cur) {
-                /* start new grace period: */
-                rdp->qs_pending = 1;
-                rdp->passed_quiesc = 0;
-                rdp->quiescbatch = rcp->cur;
-                return;
-        }
-
-        /* Grace period already completed for this cpu?
-         * qs_pending is checked instead of the actual bitmap to avoid
-         * cacheline trashing.
-         */
-        if (!rdp->qs_pending)
-                return;
-
-        /*
-         * Was there a quiescent state since the beginning of the grace
-         * period? If no, then exit and wait for the next call.
-         */
-        if (!rdp->passed_quiesc)
-                return;
-        rdp->qs_pending = 0;
-
-        spin_lock_irqsave(&rcp->lock, flags);
-        /*
-         * rdp->quiescbatch/rcp->cur and the cpu bitmap can come out of sync
-         * during cpu startup. Ignore the quiescent state.
-         */
-        if (likely(rdp->quiescbatch == rcp->cur))
-                cpu_quiet(rdp->cpu, rcp);
-
-        spin_unlock_irqrestore(&rcp->lock, flags);
-}
-
-
-#ifdef CONFIG_HOTPLUG_CPU
-
-/* warning! helper for rcu_offline_cpu. do not use elsewhere without reviewing
- * locking requirements, the list it's pulling from has to belong to a cpu
- * which is dead and hence not processing interrupts.
- */
-static void rcu_move_batch(struct rcu_data *this_rdp, struct rcu_head *list,
-                                struct rcu_head **tail, long batch)
-{
-        unsigned long flags;
-
-        if (list) {
-                local_irq_save(flags);
-                this_rdp->batch = batch;
-                *this_rdp->nxttail[2] = list;
-                this_rdp->nxttail[2] = tail;
-                local_irq_restore(flags);
-        }
-}
-
-static void __rcu_offline_cpu(struct rcu_data *this_rdp,
-                                struct rcu_ctrlblk *rcp, struct rcu_data *rdp)
-{
-        unsigned long flags;
-
-        /*
-         * if the cpu going offline owns the grace period
-         * we can block indefinitely waiting for it, so flush
-         * it here
-         */
-        spin_lock_irqsave(&rcp->lock, flags);
-        if (rcp->cur != rcp->completed)
-                cpu_quiet(rdp->cpu, rcp);
-        rcu_move_batch(this_rdp, rdp->donelist, rdp->donetail, rcp->cur + 1);
-        rcu_move_batch(this_rdp, rdp->nxtlist, rdp->nxttail[2], rcp->cur + 1);
-        spin_unlock(&rcp->lock);
-
-        this_rdp->qlen += rdp->qlen;
-        local_irq_restore(flags);
-}
-
-static void rcu_offline_cpu(int cpu)
-{
-        struct rcu_data *this_rdp = &get_cpu_var(rcu_data);
-        struct rcu_data *this_bh_rdp = &get_cpu_var(rcu_bh_data);
-
-        __rcu_offline_cpu(this_rdp, &rcu_ctrlblk,
-                                        &per_cpu(rcu_data, cpu));
-        __rcu_offline_cpu(this_bh_rdp, &rcu_bh_ctrlblk,
-                                        &per_cpu(rcu_bh_data, cpu));
-        put_cpu_var(rcu_data);
-        put_cpu_var(rcu_bh_data);
-}
-
-#else
-
-static void rcu_offline_cpu(int cpu)
-{
-}
-
-#endif
-
-/*
- * This does the RCU processing work from softirq context.
- */
-static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp,
-                                        struct rcu_data *rdp)
-{
-        unsigned long flags;
-        long completed_snap;
-
-        if (rdp->nxtlist) {
-                local_irq_save(flags);
-                completed_snap = ACCESS_ONCE(rcp->completed);
-
-                /*
-                 * move the other grace-period-completed entries to
-                 * [rdp->nxtlist, *rdp->nxttail[0]) temporarily
-                 */
-                if (!rcu_batch_before(completed_snap, rdp->batch))
-                        rdp->nxttail[0] = rdp->nxttail[1] = rdp->nxttail[2];
-                else if (!rcu_batch_before(completed_snap, rdp->batch - 1))
-                        rdp->nxttail[0] = rdp->nxttail[1];
-
-                /*
-                 * the grace period for entries in
-                 * [rdp->nxtlist, *rdp->nxttail[0]) has completed and
-                 * move these entries to donelist
-                 */
-                if (rdp->nxttail[0] != &rdp->nxtlist) {
-                        *rdp->donetail = rdp->nxtlist;
-                        rdp->donetail = rdp->nxttail[0];
-                        rdp->nxtlist = *rdp->nxttail[0];
-                        *rdp->donetail = NULL;
-
-                        if (rdp->nxttail[1] == rdp->nxttail[0])
-                                rdp->nxttail[1] = &rdp->nxtlist;
-                        if (rdp->nxttail[2] == rdp->nxttail[0])
-                                rdp->nxttail[2] = &rdp->nxtlist;
-                        rdp->nxttail[0] = &rdp->nxtlist;
-                }
-
-                local_irq_restore(flags);
-
-                if (rcu_batch_after(rdp->batch, rcp->pending)) {
-                        unsigned long flags2;
-
-                        /* and start it/schedule start if it's a new batch */
-                        spin_lock_irqsave(&rcp->lock, flags2);
-                        if (rcu_batch_after(rdp->batch, rcp->pending)) {
-                                rcp->pending = rdp->batch;
-                                rcu_start_batch(rcp);
-                        }
-                        spin_unlock_irqrestore(&rcp->lock, flags2);
-                }
-        }
-
-        rcu_check_quiescent_state(rcp, rdp);
-        if (rdp->donelist)
-                rcu_do_batch(rdp);
-}
-
-static void rcu_process_callbacks(struct softirq_action *unused)
-{
-        /*
-         * Memory references from any prior RCU read-side critical sections
-         * executed by the interrupted code must be see before any RCU
-         * grace-period manupulations below.
-         */
-
-        smp_mb(); /* See above block comment. */
-
-        __rcu_process_callbacks(&rcu_ctrlblk, &__get_cpu_var(rcu_data));
-        __rcu_process_callbacks(&rcu_bh_ctrlblk, &__get_cpu_var(rcu_bh_data));
-
-        /*
-         * Memory references from any later RCU read-side critical sections
-         * executed by the interrupted code must be see after any RCU
-         * grace-period manupulations above.
-         */
-
-        smp_mb(); /* See above block comment. */
-}
-
-static int __rcu_pending(struct rcu_ctrlblk *rcp, struct rcu_data *rdp)
-{
-        /* Check for CPU stalls, if enabled. */
-        check_cpu_stall(rcp);
-
-        if (rdp->nxtlist) {
-                long completed_snap = ACCESS_ONCE(rcp->completed);
-
-                /*
-                 * This cpu has pending rcu entries and the grace period
-                 * for them has completed.
-                 */
-                if (!rcu_batch_before(completed_snap, rdp->batch))
-                        return 1;
-                if (!rcu_batch_before(completed_snap, rdp->batch - 1) &&
-                                rdp->nxttail[0] != rdp->nxttail[1])
-                        return 1;
-                if (rdp->nxttail[0] != &rdp->nxtlist)
-                        return 1;
-
-                /*
-                 * This cpu has pending rcu entries and the new batch
-                 * for then hasn't been started nor scheduled start
-                 */
-                if (rcu_batch_after(rdp->batch, rcp->pending))
-                        return 1;
-        }
-
-        /* This cpu has finished callbacks to invoke */
-        if (rdp->donelist)
-                return 1;
-
-        /* The rcu core waits for a quiescent state from the cpu */
-        if (rdp->quiescbatch != rcp->cur || rdp->qs_pending)
-                return 1;
-
-        /* nothing to do */
-        return 0;
-}
-
-/*
- * Check to see if there is any immediate RCU-related work to be done
- * by the current CPU, returning 1 if so.  This function is part of the
- * RCU implementation; it is -not- an exported member of the RCU API.
- */
-int rcu_pending(int cpu)
-{
-        return __rcu_pending(&rcu_ctrlblk, &per_cpu(rcu_data, cpu)) ||
-                __rcu_pending(&rcu_bh_ctrlblk, &per_cpu(rcu_bh_data, cpu));
-}
-
-/*
- * Check to see if any future RCU-related work will need to be done
- * by the current CPU, even if none need be done immediately, returning
- * 1 if so.  This function is part of the RCU implementation; it is -not-
- * an exported member of the RCU API.
- */
-int rcu_needs_cpu(int cpu)
-{
-        struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
-        struct rcu_data *rdp_bh = &per_cpu(rcu_bh_data, cpu);
-
-        return !!rdp->nxtlist || !!rdp_bh->nxtlist || rcu_pending(cpu);
-}
-
-/*
- * Top-level function driving RCU grace-period detection, normally
- * invoked from the scheduler-clock interrupt.  This function simply
- * increments counters that are read only from softirq by this same
- * CPU, so there are no memory barriers required.
- */
-void rcu_check_callbacks(int cpu, int user)
-{
-        if (user ||
-            (idle_cpu(cpu) && rcu_scheduler_active &&
-             !in_softirq() && hardirq_count() <= (1 << HARDIRQ_SHIFT))) {
-
-                /*
-                 * Get here if this CPU took its interrupt from user
-                 * mode or from the idle loop, and if this is not a
-                 * nested interrupt.  In this case, the CPU is in
-                 * a quiescent state, so count it.
-                 *
-                 * Also do a memory barrier.  This is needed to handle
-                 * the case where writes from a preempt-disable section
-                 * of code get reordered into schedule() by this CPU's
-                 * write buffer.  The memory barrier makes sure that
-                 * the rcu_qsctr_inc() and rcu_bh_qsctr_inc() are see
-                 * by other CPUs to happen after any such write.
-                 */
-
-                smp_mb();  /* See above block comment. */
-                rcu_qsctr_inc(cpu);
-                rcu_bh_qsctr_inc(cpu);
-
-        } else if (!in_softirq()) {
-
-                /*
-                 * Get here if this CPU did not take its interrupt from
-                 * softirq, in other words, if it is not interrupting
-                 * a rcu_bh read-side critical section.  This is an _bh
-                 * critical section, so count it.  The memory barrier
-                 * is needed for the same reason as is the above one.
-                 */
-
-                smp_mb();  /* See above block comment. */
-                rcu_bh_qsctr_inc(cpu);
-        }
-        raise_rcu_softirq();
-}
-
-static void __cpuinit rcu_init_percpu_data(int cpu, struct rcu_ctrlblk *rcp,
-                                                struct rcu_data *rdp)
-{
-        unsigned long flags;
-
-        spin_lock_irqsave(&rcp->lock, flags);
-        memset(rdp, 0, sizeof(*rdp));
-        rdp->nxttail[0] = rdp->nxttail[1] = rdp->nxttail[2] = &rdp->nxtlist;
-        rdp->donetail = &rdp->donelist;
-        rdp->quiescbatch = rcp->completed;
-        rdp->qs_pending = 0;
-        rdp->cpu = cpu;
-        rdp->blimit = blimit;
-        spin_unlock_irqrestore(&rcp->lock, flags);
-}
-
-static void __cpuinit rcu_online_cpu(int cpu)
-{
-        struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
-        struct rcu_data *bh_rdp = &per_cpu(rcu_bh_data, cpu);
-
-        rcu_init_percpu_data(cpu, &rcu_ctrlblk, rdp);
-        rcu_init_percpu_data(cpu, &rcu_bh_ctrlblk, bh_rdp);
-        open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
-}
-
-static int __cpuinit rcu_cpu_notify(struct notifier_block *self,
-                                unsigned long action, void *hcpu)
-{
-        long cpu = (long)hcpu;
-
-        switch (action) {
-        case CPU_UP_PREPARE:
-        case CPU_UP_PREPARE_FROZEN:
-                rcu_online_cpu(cpu);
-                break;
-        case CPU_DEAD:
-        case CPU_DEAD_FROZEN:
-                rcu_offline_cpu(cpu);
-                break;
-        default:
-                break;
-        }
-        return NOTIFY_OK;
-}
-
-static struct notifier_block __cpuinitdata rcu_nb = {
-        .notifier_call  = rcu_cpu_notify,
-};
-
-/*
- * Initializes rcu mechanism.  Assumed to be called early.
- * That is before local timer(SMP) or jiffie timer (uniproc) is setup.
- * Note that rcu_qsctr and friends are implicitly
- * initialized due to the choice of ``0'' for RCU_CTR_INVALID.
- */
-void __init __rcu_init(void)
-{
-#ifdef CONFIG_RCU_CPU_STALL_DETECTOR
-        printk(KERN_INFO "RCU-based detection of stalled CPUs is enabled.\n");
-#endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
-        rcu_cpu_notify(&rcu_nb, CPU_UP_PREPARE,
-                        (void *)(long)smp_processor_id());
-        /* Register notifier for non-boot CPUs */
-        register_cpu_notifier(&rcu_nb);
-}
-
-module_param(blimit, int, 0);
-module_param(qhimark, int, 0);
-module_param(qlowmark, int, 0);
diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c
index a967c9feb90a..bd5d5c8e5140 100644
--- a/kernel/rcupdate.c
+++ b/kernel/rcupdate.c
@@ -98,6 +98,30 @@ void synchronize_rcu(void)
 }
 EXPORT_SYMBOL_GPL(synchronize_rcu);
 
+/**
+ * synchronize_rcu_bh - wait until an rcu_bh grace period has elapsed.
+ *
+ * Control will return to the caller some time after a full rcu_bh grace
+ * period has elapsed, in other words after all currently executing rcu_bh
+ * read-side critical sections have completed.  RCU read-side critical
+ * sections are delimited by rcu_read_lock_bh() and rcu_read_unlock_bh(),
+ * and may be nested.
+ */
+void synchronize_rcu_bh(void)
+{
+        struct rcu_synchronize rcu;
+
+        if (rcu_blocking_is_gp())
+                return;
+
+        init_completion(&rcu.completion);
+        /* Will wake me after RCU finished. */
+        call_rcu_bh(&rcu.head, wakeme_after_rcu);
+        /* Wait for it. */
+        wait_for_completion(&rcu.completion);
+}
+EXPORT_SYMBOL_GPL(synchronize_rcu_bh);
+
 static void rcu_barrier_callback(struct rcu_head *notused)
 {
         if (atomic_dec_and_test(&rcu_barrier_cpu_count))
@@ -129,6 +153,7 @@ static void rcu_barrier_func(void *type)
 static inline void wait_migrated_callbacks(void)
 {
         wait_event(rcu_migrate_wq, !atomic_read(&rcu_migrate_type_count));
+        smp_mb(); /* In case we didn't sleep. */
 }
 
 /*
@@ -192,9 +217,13 @@ static void rcu_migrate_callback(struct rcu_head *notused)
                 wake_up(&rcu_migrate_wq);
 }
 
+extern int rcu_cpu_notify(struct notifier_block *self,
+                          unsigned long action, void *hcpu);
+
 static int __cpuinit rcu_barrier_cpu_hotplug(struct notifier_block *self,
                 unsigned long action, void *hcpu)
 {
+        rcu_cpu_notify(self, action, hcpu);
         if (action == CPU_DYING) {
                 /*
                  * preempt_disable() in on_each_cpu() prevents stop_machine(),
@@ -209,7 +238,8 @@ static int __cpuinit rcu_barrier_cpu_hotplug(struct notifier_block *self,
                 call_rcu_bh(rcu_migrate_head, rcu_migrate_callback);
                 call_rcu_sched(rcu_migrate_head + 1, rcu_migrate_callback);
                 call_rcu(rcu_migrate_head + 2, rcu_migrate_callback);
-        } else if (action == CPU_POST_DEAD) {
+        } else if (action == CPU_DOWN_PREPARE) {
+                /* Don't need to wait until next removal operation. */
                 /* rcu_migrate_head is protected by cpu_add_remove_lock */
                 wait_migrated_callbacks();
         }
@@ -219,8 +249,18 @@ static int __cpuinit rcu_barrier_cpu_hotplug(struct notifier_block *self,
 
 void __init rcu_init(void)
 {
+        int i;
+
         __rcu_init();
-        hotcpu_notifier(rcu_barrier_cpu_hotplug, 0);
+        cpu_notifier(rcu_barrier_cpu_hotplug, 0);
+
+        /*
+         * We don't need protection against CPU-hotplug here because
+         * this is called early in boot, before either interrupts
+         * or the scheduler are operational.
+         */
+        for_each_online_cpu(i)
+                rcu_barrier_cpu_hotplug(NULL, CPU_UP_PREPARE, (void *)(long)i);
 }
 
 void rcu_scheduler_starting(void)
diff --git a/kernel/rcupreempt.c b/kernel/rcupreempt.c
deleted file mode 100644
index beb0e659adcc..000000000000
--- a/kernel/rcupreempt.c
+++ /dev/null
@@ -1,1539 +0,0 @@
-/*
- * Read-Copy Update mechanism for mutual exclusion, realtime implementation
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
- *
- * Copyright IBM Corporation, 2006
- *
- * Authors: Paul E. McKenney <paulmck@us.ibm.com>
- *              With thanks to Esben Nielsen, Bill Huey, and Ingo Molnar
- *              for pushing me away from locks and towards counters, and
- *              to Suparna Bhattacharya for pushing me completely away
- *              from atomic instructions on the read side.
- *
- *  - Added handling of Dynamic Ticks
- *      Copyright 2007 - Paul E. Mckenney <paulmck@us.ibm.com>
- *                     - Steven Rostedt <srostedt@redhat.com>
- *
- * Papers:  http://www.rdrop.com/users/paulmck/RCU
- *
- * Design Document: http://lwn.net/Articles/253651/
- *
- * For detailed explanation of Read-Copy Update mechanism see -
- *              Documentation/RCU/ *.txt
- *
- */
-#include <linux/types.h>
-#include <linux/kernel.h>
-#include <linux/init.h>
-#include <linux/spinlock.h>
-#include <linux/smp.h>
-#include <linux/rcupdate.h>
-#include <linux/interrupt.h>
-#include <linux/sched.h>
-#include <asm/atomic.h>
-#include <linux/bitops.h>
-#include <linux/module.h>
-#include <linux/kthread.h>
-#include <linux/completion.h>
-#include <linux/moduleparam.h>
-#include <linux/percpu.h>
-#include <linux/notifier.h>
-#include <linux/cpu.h>
-#include <linux/random.h>
-#include <linux/delay.h>
-#include <linux/cpumask.h>
-#include <linux/rcupreempt_trace.h>
-#include <asm/byteorder.h>
-
-/*
- * PREEMPT_RCU data structures.
- */
-
-/*
- * GP_STAGES specifies the number of times the state machine has
- * to go through the all the rcu_try_flip_states (see below)
- * in a single Grace Period.
- *
- * GP in GP_STAGES stands for Grace Period ;)
- */
-#define GP_STAGES    2
-struct rcu_data {
-        spinlock_t      lock;           /* Protect rcu_data fields. */
-        long            completed;      /* Number of last completed batch. */
-        int             waitlistcount;
-        struct rcu_head *nextlist;
-        struct rcu_head **nexttail;
-        struct rcu_head *waitlist[GP_STAGES];
-        struct rcu_head **waittail[GP_STAGES];
-        struct rcu_head *donelist;      /* from waitlist & waitschedlist */
-        struct rcu_head **donetail;
-        long rcu_flipctr[2];
-        struct rcu_head *nextschedlist;
-        struct rcu_head **nextschedtail;
-        struct rcu_head *waitschedlist;
-        struct rcu_head **waitschedtail;
-        int rcu_sched_sleeping;
-#ifdef CONFIG_RCU_TRACE
-        struct rcupreempt_trace trace;
-#endif /* #ifdef CONFIG_RCU_TRACE */
-};
-
-/*
- * States for rcu_try_flip() and friends.
- */
-
-enum rcu_try_flip_states {
-
-        /*
-         * Stay here if nothing is happening. Flip the counter if somthing
-         * starts happening. Denoted by "I"
-         */
-        rcu_try_flip_idle_state,
-
-        /*
-         * Wait here for all CPUs to notice that the counter has flipped. This
-         * prevents the old set of counters from ever being incremented once
-         * we leave this state, which in turn is necessary because we cannot
-         * test any individual counter for zero -- we can only check the sum.
-         * Denoted by "A".
-         */
-        rcu_try_flip_waitack_state,
-
-        /*
-         * Wait here for the sum of the old per-CPU counters to reach zero.
-         * Denoted by "Z".
-         */
-        rcu_try_flip_waitzero_state,
-
-        /*
-         * Wait here for each of the other CPUs to execute a memory barrier.
-         * This is necessary to ensure that these other CPUs really have
-         * completed executing their RCU read-side critical sections, despite
-         * their CPUs wildly reordering memory. Denoted by "M".
-         */
-        rcu_try_flip_waitmb_state,
-};
-
-/*
- * States for rcu_ctrlblk.rcu_sched_sleep.
- */
-
-enum rcu_sched_sleep_states {
-        rcu_sched_not_sleeping, /* Not sleeping, callbacks need GP.  */
-        rcu_sched_sleep_prep,   /* Thinking of sleeping, rechecking. */
-        rcu_sched_sleeping,     /* Sleeping, awaken if GP needed. */
-};
-
-struct rcu_ctrlblk {
-        spinlock_t      fliplock;       /* Protect state-machine transitions. */
-        long            completed;      /* Number of last completed batch. */
-        enum rcu_try_flip_states rcu_try_flip_state; /* The current state of
-                                                        the rcu state machine */
-        spinlock_t      schedlock;      /* Protect rcu_sched sleep state. */
-        enum rcu_sched_sleep_states sched_sleep; /* rcu_sched state. */
-        wait_queue_head_t sched_wq;     /* Place for rcu_sched to sleep. */
-};
-
-struct rcu_dyntick_sched {
-        int dynticks;
-        int dynticks_snap;
-        int sched_qs;
-        int sched_qs_snap;
-        int sched_dynticks_snap;
-};
-
-static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_dyntick_sched, rcu_dyntick_sched) = {
-        .dynticks = 1,
-};
-
-void rcu_qsctr_inc(int cpu)
-{
-        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
-
-        rdssp->sched_qs++;
-}
-
-#ifdef CONFIG_NO_HZ
-
-void rcu_enter_nohz(void)
-{
-        static DEFINE_RATELIMIT_STATE(rs, 10 * HZ, 1);
-
-        smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */
-        __get_cpu_var(rcu_dyntick_sched).dynticks++;
-        WARN_ON_RATELIMIT(__get_cpu_var(rcu_dyntick_sched).dynticks & 0x1, &rs);
-}
-
-void rcu_exit_nohz(void)
-{
-        static DEFINE_RATELIMIT_STATE(rs, 10 * HZ, 1);
-
-        __get_cpu_var(rcu_dyntick_sched).dynticks++;
-        smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */
-        WARN_ON_RATELIMIT(!(__get_cpu_var(rcu_dyntick_sched).dynticks & 0x1),
-                                &rs);
-}
-
-#endif /* CONFIG_NO_HZ */
-
-
-static DEFINE_PER_CPU(struct rcu_data, rcu_data);
-
-static struct rcu_ctrlblk rcu_ctrlblk = {
-        .fliplock = __SPIN_LOCK_UNLOCKED(rcu_ctrlblk.fliplock),
-        .completed = 0,
-        .rcu_try_flip_state = rcu_try_flip_idle_state,
-        .schedlock = __SPIN_LOCK_UNLOCKED(rcu_ctrlblk.schedlock),
-        .sched_sleep = rcu_sched_not_sleeping,
-        .sched_wq = __WAIT_QUEUE_HEAD_INITIALIZER(rcu_ctrlblk.sched_wq),
-};
-
-static struct task_struct *rcu_sched_grace_period_task;
-
-#ifdef CONFIG_RCU_TRACE
-static char *rcu_try_flip_state_names[] =
-        { "idle", "waitack", "waitzero", "waitmb" };
-#endif /* #ifdef CONFIG_RCU_TRACE */
-
-static DECLARE_BITMAP(rcu_cpu_online_map, NR_CPUS) __read_mostly
-        = CPU_BITS_NONE;
-
-/*
- * Enum and per-CPU flag to determine when each CPU has seen
- * the most recent counter flip.
- */
-
-enum rcu_flip_flag_values {
-        rcu_flip_seen,          /* Steady/initial state, last flip seen. */
-                                /* Only GP detector can update. */
-        rcu_flipped             /* Flip just completed, need confirmation. */
-                                /* Only corresponding CPU can update. */
-};
-static DEFINE_PER_CPU_SHARED_ALIGNED(enum rcu_flip_flag_values, rcu_flip_flag)
-                                                                = rcu_flip_seen;
-
-/*
- * Enum and per-CPU flag to determine when each CPU has executed the
- * needed memory barrier to fence in memory references from its last RCU
- * read-side critical section in the just-completed grace period.
- */
-
-enum rcu_mb_flag_values {
-        rcu_mb_done,            /* Steady/initial state, no mb()s required. */
-                                /* Only GP detector can update. */
-        rcu_mb_needed           /* Flip just completed, need an mb(). */
-                                /* Only corresponding CPU can update. */
-};
-static DEFINE_PER_CPU_SHARED_ALIGNED(enum rcu_mb_flag_values, rcu_mb_flag)
-                                                                = rcu_mb_done;
-
-/*
- * RCU_DATA_ME: find the current CPU's rcu_data structure.
- * RCU_DATA_CPU: find the specified CPU's rcu_data structure.
- */
-#define RCU_DATA_ME()           (&__get_cpu_var(rcu_data))
-#define RCU_DATA_CPU(cpu)       (&per_cpu(rcu_data, cpu))
-
-/*
- * Helper macro for tracing when the appropriate rcu_data is not
- * cached in a local variable, but where the CPU number is so cached.
- */
-#define RCU_TRACE_CPU(f, cpu) RCU_TRACE(f, &(RCU_DATA_CPU(cpu)->trace));
-
-/*
- * Helper macro for tracing when the appropriate rcu_data is not
- * cached in a local variable.
- */
-#define RCU_TRACE_ME(f) RCU_TRACE(f, &(RCU_DATA_ME()->trace));
-
-/*
- * Helper macro for tracing when the appropriate rcu_data is pointed
- * to by a local variable.
- */
-#define RCU_TRACE_RDP(f, rdp) RCU_TRACE(f, &((rdp)->trace));
-
-#define RCU_SCHED_BATCH_TIME (HZ / 50)
-
-/*
- * Return the number of RCU batches processed thus far.  Useful
- * for debug and statistics.
- */
-long rcu_batches_completed(void)
-{
-        return rcu_ctrlblk.completed;
-}
-EXPORT_SYMBOL_GPL(rcu_batches_completed);
-
-void __rcu_read_lock(void)
-{
-        int idx;
-        struct task_struct *t = current;
-        int nesting;
-
-        nesting = ACCESS_ONCE(t->rcu_read_lock_nesting);
-        if (nesting != 0) {
-
-                /* An earlier rcu_read_lock() covers us, just count it. */
-
-                t->rcu_read_lock_nesting = nesting + 1;
-
-        } else {
-                unsigned long flags;
-
-                /*
-                 * We disable interrupts for the following reasons:
-                 * - If we get scheduling clock interrupt here, and we
-                 *   end up acking the counter flip, it's like a promise
-                 *   that we will never increment the old counter again.
-                 *   Thus we will break that promise if that
-                 *   scheduling clock interrupt happens between the time
-                 *   we pick the .completed field and the time that we
-                 *   increment our counter.
-                 *
-                 * - We don't want to be preempted out here.
-                 *
-                 * NMIs can still occur, of course, and might themselves
-                 * contain rcu_read_lock().
-                 */
-
-                local_irq_save(flags);
-
-                /*
-                 * Outermost nesting of rcu_read_lock(), so increment
-                 * the current counter for the current CPU.  Use volatile
-                 * casts to prevent the compiler from reordering.
-                 */
-
-                idx = ACCESS_ONCE(rcu_ctrlblk.completed) & 0x1;
-                ACCESS_ONCE(RCU_DATA_ME()->rcu_flipctr[idx])++;
-
-                /*
-                 * Now that the per-CPU counter has been incremented, we
-                 * are protected from races with rcu_read_lock() invoked
-                 * from NMI handlers on this CPU.  We can therefore safely
-                 * increment the nesting counter, relieving further NMIs
-                 * of the need to increment the per-CPU counter.
-                 */
-
-                ACCESS_ONCE(t->rcu_read_lock_nesting) = nesting + 1;
-
-                /*
-                 * Now that we have preventing any NMIs from storing
-                 * to the ->rcu_flipctr_idx, we can safely use it to
-                 * remember which counter to decrement in the matching
-                 * rcu_read_unlock().
-                 */
-
-                ACCESS_ONCE(t->rcu_flipctr_idx) = idx;
-                local_irq_restore(flags);
-        }
-}
-EXPORT_SYMBOL_GPL(__rcu_read_lock);
-
-void __rcu_read_unlock(void)
-{
-        int idx;
-        struct task_struct *t = current;
-        int nesting;
-
-        nesting = ACCESS_ONCE(t->rcu_read_lock_nesting);
-        if (nesting > 1) {
-
-                /*
-                 * We are still protected by the enclosing rcu_read_lock(),
-                 * so simply decrement the counter.
-                 */
-
-                t->rcu_read_lock_nesting = nesting - 1;
-
-        } else {
-                unsigned long flags;
-
-                /*
-                 * Disable local interrupts to prevent the grace-period
-                 * detection state machine from seeing us half-done.
-                 * NMIs can still occur, of course, and might themselves
-                 * contain rcu_read_lock() and rcu_read_unlock().
-                 */
-
-                local_irq_save(flags);
-
-                /*
-                 * Outermost nesting of rcu_read_unlock(), so we must
-                 * decrement the current counter for the current CPU.
-                 * This must be done carefully, because NMIs can
-                 * occur at any point in this code, and any rcu_read_lock()
-                 * and rcu_read_unlock() pairs in the NMI handlers
-                 * must interact non-destructively with this code.
-                 * Lots of volatile casts, and -very- careful ordering.
-                 *
-                 * Changes to this code, including this one, must be
-                 * inspected, validated, and tested extremely carefully!!!
-                 */
-
-                /*
-                 * First, pick up the index.
-                 */
-
-                idx = ACCESS_ONCE(t->rcu_flipctr_idx);
-
-                /*
-                 * Now that we have fetched the counter index, it is
-                 * safe to decrement the per-task RCU nesting counter.
-                 * After this, any interrupts or NMIs will increment and
-                 * decrement the per-CPU counters.
-                 */
-                ACCESS_ONCE(t->rcu_read_lock_nesting) = nesting - 1;
-
-                /*
-                 * It is now safe to decrement this task's nesting count.
-                 * NMIs that occur after this statement will route their
-                 * rcu_read_lock() calls through this "else" clause, and
-                 * will thus start incrementing the per-CPU counter on
-                 * their own.  They will also clobber ->rcu_flipctr_idx,
-                 * but that is OK, since we have already fetched it.
-                 */
-
-                ACCESS_ONCE(RCU_DATA_ME()->rcu_flipctr[idx])--;
-                local_irq_restore(flags);
-        }
-}
-EXPORT_SYMBOL_GPL(__rcu_read_unlock);
-
-/*
- * If a global counter flip has occurred since the last time that we
- * advanced callbacks, advance them.  Hardware interrupts must be
- * disabled when calling this function.
- */
-static void __rcu_advance_callbacks(struct rcu_data *rdp)
-{
-        int cpu;
-        int i;
-        int wlc = 0;
-
-        if (rdp->completed != rcu_ctrlblk.completed) {
-                if (rdp->waitlist[GP_STAGES - 1] != NULL) {
-                        *rdp->donetail = rdp->waitlist[GP_STAGES - 1];
-                        rdp->donetail = rdp->waittail[GP_STAGES - 1];
-                        RCU_TRACE_RDP(rcupreempt_trace_move2done, rdp);
-                }
-                for (i = GP_STAGES - 2; i >= 0; i--) {
-                        if (rdp->waitlist[i] != NULL) {
-                                rdp->waitlist[i + 1] = rdp->waitlist[i];
-                                rdp->waittail[i + 1] = rdp->waittail[i];
-                                wlc++;
-                        } else {
-                                rdp->waitlist[i + 1] = NULL;
-                                rdp->waittail[i + 1] =
-                                        &rdp->waitlist[i + 1];
-                        }
-                }
-                if (rdp->nextlist != NULL) {
-                        rdp->waitlist[0] = rdp->nextlist;
-                        rdp->waittail[0] = rdp->nexttail;
-                        wlc++;
-                        rdp->nextlist = NULL;
-                        rdp->nexttail = &rdp->nextlist;
-                        RCU_TRACE_RDP(rcupreempt_trace_move2wait, rdp);
-                } else {
-                        rdp->waitlist[0] = NULL;
-                        rdp->waittail[0] = &rdp->waitlist[0];
-                }
-                rdp->waitlistcount = wlc;
-                rdp->completed = rcu_ctrlblk.completed;
-        }
-
-        /*
-         * Check to see if this CPU needs to report that it has seen
-         * the most recent counter flip, thereby declaring that all
-         * subsequent rcu_read_lock() invocations will respect this flip.
-         */
-
-        cpu = raw_smp_processor_id();
-        if (per_cpu(rcu_flip_flag, cpu) == rcu_flipped) {
-                smp_mb();  /* Subsequent counter accesses must see new value */
-                per_cpu(rcu_flip_flag, cpu) = rcu_flip_seen;
-                smp_mb();  /* Subsequent RCU read-side critical sections */
-                           /*  seen -after- acknowledgement. */
-        }
-}
-
-#ifdef CONFIG_NO_HZ
-static DEFINE_PER_CPU(int, rcu_update_flag);
-
-/**
- * rcu_irq_enter - Called from Hard irq handlers and NMI/SMI.
- *
- * If the CPU was idle with dynamic ticks active, this updates the
- * rcu_dyntick_sched.dynticks to let the RCU handling know that the
- * CPU is active.
- */
-void rcu_irq_enter(void)
-{
-        int cpu = smp_processor_id();
-        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
-
-        if (per_cpu(rcu_update_flag, cpu))
-                per_cpu(rcu_update_flag, cpu)++;
-
-        /*
-         * Only update if we are coming from a stopped ticks mode
-         * (rcu_dyntick_sched.dynticks is even).
-         */
-        if (!in_interrupt() &&
-            (rdssp->dynticks & 0x1) == 0) {
-                /*
-                 * The following might seem like we could have a race
-                 * with NMI/SMIs. But this really isn't a problem.
-                 * Here we do a read/modify/write, and the race happens
-                 * when an NMI/SMI comes in after the read and before
-                 * the write. But NMI/SMIs will increment this counter
-                 * twice before returning, so the zero bit will not
-                 * be corrupted by the NMI/SMI which is the most important
-                 * part.
-                 *
-                 * The only thing is that we would bring back the counter
-                 * to a postion that it was in during the NMI/SMI.
-                 * But the zero bit would be set, so the rest of the
-                 * counter would again be ignored.
-                 *
-                 * On return from the IRQ, the counter may have the zero
-                 * bit be 0 and the counter the same as the return from
-                 * the NMI/SMI. If the state machine was so unlucky to
-                 * see that, it still doesn't matter, since all
-                 * RCU read-side critical sections on this CPU would
-                 * have already completed.
-                 */
-                rdssp->dynticks++;
-                /*
-                 * The following memory barrier ensures that any
-                 * rcu_read_lock() primitives in the irq handler
-                 * are seen by other CPUs to follow the above
-                 * increment to rcu_dyntick_sched.dynticks. This is
-                 * required in order for other CPUs to correctly
-                 * determine when it is safe to advance the RCU
-                 * grace-period state machine.
-                 */
-                smp_mb(); /* see above block comment. */
-                /*
-                 * Since we can't determine the dynamic tick mode from
-                 * the rcu_dyntick_sched.dynticks after this routine,
-                 * we use a second flag to acknowledge that we came
-                 * from an idle state with ticks stopped.
-                 */
-                per_cpu(rcu_update_flag, cpu)++;
-                /*
-                 * If we take an NMI/SMI now, they will also increment
-                 * the rcu_update_flag, and will not update the
-                 * rcu_dyntick_sched.dynticks on exit. That is for
-                 * this IRQ to do.
-                 */
-        }
-}
-
-/**
- * rcu_irq_exit - Called from exiting Hard irq context.
- *
- * If the CPU was idle with dynamic ticks active, update the
- * rcu_dyntick_sched.dynticks to put let the RCU handling be
- * aware that the CPU is going back to idle with no ticks.
- */
-void rcu_irq_exit(void)
-{
-        int cpu = smp_processor_id();
-        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
-
-        /*
-         * rcu_update_flag is set if we interrupted the CPU
-         * when it was idle with ticks stopped.
-         * Once this occurs, we keep track of interrupt nesting
-         * because a NMI/SMI could also come in, and we still
-         * only want the IRQ that started the increment of the
-         * rcu_dyntick_sched.dynticks to be the one that modifies
-         * it on exit.
-         */
-        if (per_cpu(rcu_update_flag, cpu)) {
-                if (--per_cpu(rcu_update_flag, cpu))
-                        return;
-
-                /* This must match the interrupt nesting */
-                WARN_ON(in_interrupt());
-
-                /*
-                 * If an NMI/SMI happens now we are still
-                 * protected by the rcu_dyntick_sched.dynticks being odd.
-                 */
-
-                /*
-                 * The following memory barrier ensures that any
-                 * rcu_read_unlock() primitives in the irq handler
-                 * are seen by other CPUs to preceed the following
-                 * increment to rcu_dyntick_sched.dynticks. This
-                 * is required in order for other CPUs to determine
-                 * when it is safe to advance the RCU grace-period
-                 * state machine.
-                 */
-                smp_mb(); /* see above block comment. */
-                rdssp->dynticks++;
-                WARN_ON(rdssp->dynticks & 0x1);
-        }
-}
-
-void rcu_nmi_enter(void)
-{
-        rcu_irq_enter();
-}
-
-void rcu_nmi_exit(void)
-{
-        rcu_irq_exit();
-}
-
-static void dyntick_save_progress_counter(int cpu)
-{
-        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
-
-        rdssp->dynticks_snap = rdssp->dynticks;
-}
-
-static inline int
-rcu_try_flip_waitack_needed(int cpu)
-{
-        long curr;
-        long snap;
-        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
-
-        curr = rdssp->dynticks;
-        snap = rdssp->dynticks_snap;
-        smp_mb(); /* force ordering with cpu entering/leaving dynticks. */
-
-        /*
-         * If the CPU remained in dynticks mode for the entire time
-         * and didn't take any interrupts, NMIs, SMIs, or whatever,
-         * then it cannot be in the middle of an rcu_read_lock(), so
-         * the next rcu_read_lock() it executes must use the new value
-         * of the counter.  So we can safely pretend that this CPU
-         * already acknowledged the counter.
-         */
-
-        if ((curr == snap) && ((curr & 0x1) == 0))
-                return 0;
-
-        /*
-         * If the CPU passed through or entered a dynticks idle phase with
-         * no active irq handlers, then, as above, we can safely pretend
-         * that this CPU already acknowledged the counter.
-         */
-
-        if ((curr - snap) > 2 || (curr & 0x1) == 0)
-                return 0;
-
-        /* We need this CPU to explicitly acknowledge the counter flip. */
-
-        return 1;
-}
-
-static inline int
-rcu_try_flip_waitmb_needed(int cpu)
-{
-        long curr;
-        long snap;
-        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
-
-        curr = rdssp->dynticks;
-        snap = rdssp->dynticks_snap;
-        smp_mb(); /* force ordering with cpu entering/leaving dynticks. */
-
-        /*
-         * If the CPU remained in dynticks mode for the entire time
-         * and didn't take any interrupts, NMIs, SMIs, or whatever,
-         * then it cannot have executed an RCU read-side critical section
-         * during that time, so there is no need for it to execute a
-         * memory barrier.
-         */
-
-        if ((curr == snap) && ((curr & 0x1) == 0))
-                return 0;
-
-        /*
-         * If the CPU either entered or exited an outermost interrupt,
-         * SMI, NMI, or whatever handler, then we know that it executed
-         * a memory barrier when doing so.  So we don't need another one.
-         */
-        if (curr != snap)
-                return 0;
-
-        /* We need the CPU to execute a memory barrier. */
-
-        return 1;
-}
-
-static void dyntick_save_progress_counter_sched(int cpu)
-{
-        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
-
-        rdssp->sched_dynticks_snap = rdssp->dynticks;
-}
-
-static int rcu_qsctr_inc_needed_dyntick(int cpu)
-{
-        long curr;
-        long snap;
-        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
-
-        curr = rdssp->dynticks;
-        snap = rdssp->sched_dynticks_snap;
-        smp_mb(); /* force ordering with cpu entering/leaving dynticks. */
-
-        /*
-         * If the CPU remained in dynticks mode for the entire time
-         * and didn't take any interrupts, NMIs, SMIs, or whatever,
-         * then it cannot be in the middle of an rcu_read_lock(), so
-         * the next rcu_read_lock() it executes must use the new value
-         * of the counter.  Therefore, this CPU has been in a quiescent
-         * state the entire time, and we don't need to wait for it.
-         */
-
-        if ((curr == snap) && ((curr & 0x1) == 0))
-                return 0;
-
-        /*
-         * If the CPU passed through or entered a dynticks idle phase with
-         * no active irq handlers, then, as above, this CPU has already
-         * passed through a quiescent state.
-         */
-
-        if ((curr - snap) > 2 || (snap & 0x1) == 0)
-                return 0;
-
-        /* We need this CPU to go through a quiescent state. */
-
-        return 1;
-}
-
-#else /* !CONFIG_NO_HZ */
-
-# define dyntick_save_progress_counter(cpu)             do { } while (0)
-# define rcu_try_flip_waitack_needed(cpu)               (1)
-# define rcu_try_flip_waitmb_needed(cpu)                (1)
-
-# define dyntick_save_progress_counter_sched(cpu)       do { } while (0)
-# define rcu_qsctr_inc_needed_dyntick(cpu)              (1)
-
-#endif /* CONFIG_NO_HZ */
-
-static void save_qsctr_sched(int cpu)
-{
-        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
-
-        rdssp->sched_qs_snap = rdssp->sched_qs;
-}
-
-static inline int rcu_qsctr_inc_needed(int cpu)
-{
-        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
-
-        /*
-         * If there has been a quiescent state, no more need to wait
-         * on this CPU.
-         */
-
-        if (rdssp->sched_qs != rdssp->sched_qs_snap) {
-                smp_mb(); /* force ordering with cpu entering schedule(). */
-                return 0;
-        }
-
-        /* We need this CPU to go through a quiescent state. */
-
-        return 1;
-}
-
-/*
- * Get here when RCU is idle.  Decide whether we need to
- * move out of idle state, and return non-zero if so.
- * "Straightforward" approach for the moment, might later
- * use callback-list lengths, grace-period duration, or
- * some such to determine when to exit idle state.
- * Might also need a pre-idle test that does not acquire
- * the lock, but let's get the simple case working first...
- */
-
-static int
-rcu_try_flip_idle(void)
-{
-        int cpu;
-
-        RCU_TRACE_ME(rcupreempt_trace_try_flip_i1);
-        if (!rcu_pending(smp_processor_id())) {
-                RCU_TRACE_ME(rcupreempt_trace_try_flip_ie1);
-                return 0;
-        }
-
-        /*
-         * Do the flip.
-         */
-
-        RCU_TRACE_ME(rcupreempt_trace_try_flip_g1);
-        rcu_ctrlblk.completed++;  /* stands in for rcu_try_flip_g2 */
-
-        /*
-         * Need a memory barrier so that other CPUs see the new
-         * counter value before they see the subsequent change of all
-         * the rcu_flip_flag instances to rcu_flipped.
-         */
-
-        smp_mb();       /* see above block comment. */
-
-        /* Now ask each CPU for acknowledgement of the flip. */
-
-        for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map)) {
-                per_cpu(rcu_flip_flag, cpu) = rcu_flipped;
-                dyntick_save_progress_counter(cpu);
-        }
-
-        return 1;
-}
-
-/*
- * Wait for CPUs to acknowledge the flip.
- */
-
-static int
-rcu_try_flip_waitack(void)
-{
-        int cpu;
-
-        RCU_TRACE_ME(rcupreempt_trace_try_flip_a1);
-        for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map))
-                if (rcu_try_flip_waitack_needed(cpu) &&
-                    per_cpu(rcu_flip_flag, cpu) != rcu_flip_seen) {
-                        RCU_TRACE_ME(rcupreempt_trace_try_flip_ae1);
-                        return 0;
-                }
-
-        /*
-         * Make sure our checks above don't bleed into subsequent
-         * waiting for the sum of the counters to reach zero.
-         */
-
-        smp_mb();       /* see above block comment. */
-        RCU_TRACE_ME(rcupreempt_trace_try_flip_a2);
-        return 1;
-}
-
-/*
- * Wait for collective ``last'' counter to reach zero,
- * then tell all CPUs to do an end-of-grace-period memory barrier.
- */
-
-static int
-rcu_try_flip_waitzero(void)
-{
-        int cpu;
-        int lastidx = !(rcu_ctrlblk.completed & 0x1);
-        int sum = 0;
-
-        /* Check to see if the sum of the "last" counters is zero. */
-
-        RCU_TRACE_ME(rcupreempt_trace_try_flip_z1);
-        for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map))
-                sum += RCU_DATA_CPU(cpu)->rcu_flipctr[lastidx];
-        if (sum != 0) {
-                RCU_TRACE_ME(rcupreempt_trace_try_flip_ze1);
-                return 0;
-        }
-
-        /*
-         * This ensures that the other CPUs see the call for
-         * memory barriers -after- the sum to zero has been
-         * detected here
-         */
-        smp_mb();  /*  ^^^^^^^^^^^^ */
-
-        /* Call for a memory barrier from each CPU. */
-        for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map)) {
-                per_cpu(rcu_mb_flag, cpu) = rcu_mb_needed;
-                dyntick_save_progress_counter(cpu);
-        }
-
-        RCU_TRACE_ME(rcupreempt_trace_try_flip_z2);
-        return 1;
-}
-
-/*
- * Wait for all CPUs to do their end-of-grace-period memory barrier.
- * Return 0 once all CPUs have done so.
- */
-
-static int
-rcu_try_flip_waitmb(void)
-{
-        int cpu;
-
-        RCU_TRACE_ME(rcupreempt_trace_try_flip_m1);
-        for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map))
-                if (rcu_try_flip_waitmb_needed(cpu) &&
-                    per_cpu(rcu_mb_flag, cpu) != rcu_mb_done) {
-                        RCU_TRACE_ME(rcupreempt_trace_try_flip_me1);
-                        return 0;
-                }
-
-        smp_mb(); /* Ensure that the above checks precede any following flip. */
-        RCU_TRACE_ME(rcupreempt_trace_try_flip_m2);
-        return 1;
-}
-
-/*
- * Attempt a single flip of the counters.  Remember, a single flip does
- * -not- constitute a grace period.  Instead, the interval between
- * at least GP_STAGES consecutive flips is a grace period.
- *
- * If anyone is nuts enough to run this CONFIG_PREEMPT_RCU implementation
- * on a large SMP, they might want to use a hierarchical organization of
- * the per-CPU-counter pairs.
- */
-static void rcu_try_flip(void)
-{
-        unsigned long flags;
-
-        RCU_TRACE_ME(rcupreempt_trace_try_flip_1);
-        if (unlikely(!spin_trylock_irqsave(&rcu_ctrlblk.fliplock, flags))) {
-                RCU_TRACE_ME(rcupreempt_trace_try_flip_e1);
-                return;
-        }
-
-        /*
-         * Take the next transition(s) through the RCU grace-period
-         * flip-counter state machine.
-         */
-
-        switch (rcu_ctrlblk.rcu_try_flip_state) {
-        case rcu_try_flip_idle_state:
-                if (rcu_try_flip_idle())
-                        rcu_ctrlblk.rcu_try_flip_state =
-                                rcu_try_flip_waitack_state;
-                break;
-        case rcu_try_flip_waitack_state:
-                if (rcu_try_flip_waitack())
-                        rcu_ctrlblk.rcu_try_flip_state =
-                                rcu_try_flip_waitzero_state;
-                break;
-        case rcu_try_flip_waitzero_state:
-                if (rcu_try_flip_waitzero())
-                        rcu_ctrlblk.rcu_try_flip_state =
-                                rcu_try_flip_waitmb_state;
-                break;
-        case rcu_try_flip_waitmb_state:
-                if (rcu_try_flip_waitmb())
-                        rcu_ctrlblk.rcu_try_flip_state =
-                                rcu_try_flip_idle_state;
-        }
-        spin_unlock_irqrestore(&rcu_ctrlblk.fliplock, flags);
-}
-
-/*
- * Check to see if this CPU needs to do a memory barrier in order to
- * ensure that any prior RCU read-side critical sections have committed
- * their counter manipulations and critical-section memory references
- * before declaring the grace period to be completed.
- */
-static void rcu_check_mb(int cpu)
-{
-        if (per_cpu(rcu_mb_flag, cpu) == rcu_mb_needed) {
-                smp_mb();  /* Ensure RCU read-side accesses are visible. */
-                per_cpu(rcu_mb_flag, cpu) = rcu_mb_done;
-        }
-}
-
-void rcu_check_callbacks(int cpu, int user)
-{
-        unsigned long flags;
-        struct rcu_data *rdp = RCU_DATA_CPU(cpu);
-
-        /*
-         * If this CPU took its interrupt from user mode or from the
-         * idle loop, and this is not a nested interrupt, then
-         * this CPU has to have exited all prior preept-disable
-         * sections of code.  So increment the counter to note this.
-         *
-         * The memory barrier is needed to handle the case where
-         * writes from a preempt-disable section of code get reordered
-         * into schedule() by this CPU's write buffer.  So the memory
-         * barrier makes sure that the rcu_qsctr_inc() is seen by other
-         * CPUs to happen after any such write.
-         */
-
-        if (user ||
-            (idle_cpu(cpu) && !in_softirq() &&
-             hardirq_count() <= (1 << HARDIRQ_SHIFT))) {
-                smp_mb();       /* Guard against aggressive schedule(). */
-                rcu_qsctr_inc(cpu);
-        }
-
-        rcu_check_mb(cpu);
-        if (rcu_ctrlblk.completed == rdp->completed)
-                rcu_try_flip();
-        spin_lock_irqsave(&rdp->lock, flags);
-        RCU_TRACE_RDP(rcupreempt_trace_check_callbacks, rdp);
-        __rcu_advance_callbacks(rdp);
-        if (rdp->donelist == NULL) {
-                spin_unlock_irqrestore(&rdp->lock, flags);
-        } else {
-                spin_unlock_irqrestore(&rdp->lock, flags);
-                raise_softirq(RCU_SOFTIRQ);
-        }
-}
-
-/*
- * Needed by dynticks, to make sure all RCU processing has finished
- * when we go idle:
- */
-void rcu_advance_callbacks(int cpu, int user)
-{
-        unsigned long flags;
-        struct rcu_data *rdp = RCU_DATA_CPU(cpu);
-
-        if (rcu_ctrlblk.completed == rdp->completed) {
-                rcu_try_flip();
-                if (rcu_ctrlblk.completed == rdp->completed)
-                        return;
-        }
-        spin_lock_irqsave(&rdp->lock, flags);
-        RCU_TRACE_RDP(rcupreempt_trace_check_callbacks, rdp);
-        __rcu_advance_callbacks(rdp);
-        spin_unlock_irqrestore(&rdp->lock, flags);
-}
-
-#ifdef CONFIG_HOTPLUG_CPU
-#define rcu_offline_cpu_enqueue(srclist, srctail, dstlist, dsttail) do { \
-                *dsttail = srclist; \
-                if (srclist != NULL) { \
-                        dsttail = srctail; \
-                        srclist = NULL; \
-                        srctail = &srclist;\
-                } \
-        } while (0)
-
-void rcu_offline_cpu(int cpu)
-{
-        int i;
-        struct rcu_head *list = NULL;
-        unsigned long flags;
-        struct rcu_data *rdp = RCU_DATA_CPU(cpu);
-        struct rcu_head *schedlist = NULL;
-        struct rcu_head **schedtail = &schedlist;
-        struct rcu_head **tail = &list;
-
-        /*
-         * Remove all callbacks from the newly dead CPU, retaining order.
-         * Otherwise rcu_barrier() will fail
-         */
-
-        spin_lock_irqsave(&rdp->lock, flags);
-        rcu_offline_cpu_enqueue(rdp->donelist, rdp->donetail, list, tail);
-        for (i = GP_STAGES - 1; i >= 0; i--)
-                rcu_offline_cpu_enqueue(rdp->waitlist[i], rdp->waittail[i],
-                                                list, tail);
-        rcu_offline_cpu_enqueue(rdp->nextlist, rdp->nexttail, list, tail);
-        rcu_offline_cpu_enqueue(rdp->waitschedlist, rdp->waitschedtail,
-                                schedlist, schedtail);
-        rcu_offline_cpu_enqueue(rdp->nextschedlist, rdp->nextschedtail,
-                                schedlist, schedtail);
-        rdp->rcu_sched_sleeping = 0;
-        spin_unlock_irqrestore(&rdp->lock, flags);
-        rdp->waitlistcount = 0;
-
-        /* Disengage the newly dead CPU from the grace-period computation. */
-
-        spin_lock_irqsave(&rcu_ctrlblk.fliplock, flags);
-        rcu_check_mb(cpu);
-        if (per_cpu(rcu_flip_flag, cpu) == rcu_flipped) {
-                smp_mb();  /* Subsequent counter accesses must see new value */
-                per_cpu(rcu_flip_flag, cpu) = rcu_flip_seen;
-                smp_mb();  /* Subsequent RCU read-side critical sections */
-                           /*  seen -after- acknowledgement. */
-        }
-
-        RCU_DATA_ME()->rcu_flipctr[0] += RCU_DATA_CPU(cpu)->rcu_flipctr[0];
-        RCU_DATA_ME()->rcu_flipctr[1] += RCU_DATA_CPU(cpu)->rcu_flipctr[1];
-
-        RCU_DATA_CPU(cpu)->rcu_flipctr[0] = 0;
-        RCU_DATA_CPU(cpu)->rcu_flipctr[1] = 0;
-
-        cpumask_clear_cpu(cpu, to_cpumask(rcu_cpu_online_map));
-
-        spin_unlock_irqrestore(&rcu_ctrlblk.fliplock, flags);
-
-        /*
-         * Place the removed callbacks on the current CPU's queue.
-         * Make them all start a new grace period: simple approach,
-         * in theory could starve a given set of callbacks, but
-         * you would need to be doing some serious CPU hotplugging
-         * to make this happen.  If this becomes a problem, adding
-         * a synchronize_rcu() to the hotplug path would be a simple
-         * fix.
-         */
-
-        local_irq_save(flags);  /* disable preempt till we know what lock. */
-        rdp = RCU_DATA_ME();
-        spin_lock(&rdp->lock);
-        *rdp->nexttail = list;
-        if (list)
-                rdp->nexttail = tail;
-        *rdp->nextschedtail = schedlist;
-        if (schedlist)
-                rdp->nextschedtail = schedtail;
-        spin_unlock_irqrestore(&rdp->lock, flags);
-}
-
-#else /* #ifdef CONFIG_HOTPLUG_CPU */
-
-void rcu_offline_cpu(int cpu)
-{
-}
-
-#endif /* #else #ifdef CONFIG_HOTPLUG_CPU */
-
-void __cpuinit rcu_online_cpu(int cpu)
-{
-        unsigned long flags;
-        struct rcu_data *rdp;
-
-        spin_lock_irqsave(&rcu_ctrlblk.fliplock, flags);
-        cpumask_set_cpu(cpu, to_cpumask(rcu_cpu_online_map));
-        spin_unlock_irqrestore(&rcu_ctrlblk.fliplock, flags);
-
-        /*
-         * The rcu_sched grace-period processing might have bypassed
-         * this CPU, given that it was not in the rcu_cpu_online_map
-         * when the grace-period scan started.  This means that the
-         * grace-period task might sleep.  So make sure that if this
-         * should happen, the first callback posted to this CPU will
-         * wake up the grace-period task if need be.
-         */
-
-        rdp = RCU_DATA_CPU(cpu);
-        spin_lock_irqsave(&rdp->lock, flags);
-        rdp->rcu_sched_sleeping = 1;
-        spin_unlock_irqrestore(&rdp->lock, flags);
-}
-
-static void rcu_process_callbacks(struct softirq_action *unused)
-{
-        unsigned long flags;
-        struct rcu_head *next, *list;
-        struct rcu_data *rdp;
-
-        local_irq_save(flags);
-        rdp = RCU_DATA_ME();
-        spin_lock(&rdp->lock);
-        list = rdp->donelist;
-        if (list == NULL) {
-                spin_unlock_irqrestore(&rdp->lock, flags);
-                return;
-        }
-        rdp->donelist = NULL;
-        rdp->donetail = &rdp->donelist;
-        RCU_TRACE_RDP(rcupreempt_trace_done_remove, rdp);
-        spin_unlock_irqrestore(&rdp->lock, flags);
-        while (list) {
-                next = list->next;
-                list->func(list);
-                list = next;
-                RCU_TRACE_ME(rcupreempt_trace_invoke);
-        }
-}
-
-void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
-{
-        unsigned long flags;
-        struct rcu_data *rdp;
-
-        head->func = func;
-        head->next = NULL;
-        local_irq_save(flags);
-        rdp = RCU_DATA_ME();
-        spin_lock(&rdp->lock);
-        __rcu_advance_callbacks(rdp);
-        *rdp->nexttail = head;
-        rdp->nexttail = &head->next;
-        RCU_TRACE_RDP(rcupreempt_trace_next_add, rdp);
-        spin_unlock_irqrestore(&rdp->lock, flags);
-}
-EXPORT_SYMBOL_GPL(call_rcu);
-
-void call_rcu_sched(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
-{
-        unsigned long flags;
-        struct rcu_data *rdp;
-        int wake_gp = 0;
-
-        head->func = func;
-        head->next = NULL;
-        local_irq_save(flags);
-        rdp = RCU_DATA_ME();
-        spin_lock(&rdp->lock);
-        *rdp->nextschedtail = head;
-        rdp->nextschedtail = &head->next;
-        if (rdp->rcu_sched_sleeping) {
-
-                /* Grace-period processing might be sleeping... */
-
-                rdp->rcu_sched_sleeping = 0;
-                wake_gp = 1;
-        }
-        spin_unlock_irqrestore(&rdp->lock, flags);
-        if (wake_gp) {
-
-                /* Wake up grace-period processing, unless someone beat us. */
-
-                spin_lock_irqsave(&rcu_ctrlblk.schedlock, flags);
-                if (rcu_ctrlblk.sched_sleep != rcu_sched_sleeping)
-                        wake_gp = 0;
-                rcu_ctrlblk.sched_sleep = rcu_sched_not_sleeping;
-                spin_unlock_irqrestore(&rcu_ctrlblk.schedlock, flags);
-                if (wake_gp)
-                        wake_up_interruptible(&rcu_ctrlblk.sched_wq);
-        }
-}
-EXPORT_SYMBOL_GPL(call_rcu_sched);
-
-/*
- * Wait until all currently running preempt_disable() code segments
- * (including hardware-irq-disable segments) complete.  Note that
- * in -rt this does -not- necessarily result in all currently executing
- * interrupt -handlers- having completed.
- */
-void __synchronize_sched(void)
-{
-        struct rcu_synchronize rcu;
-
-        if (num_online_cpus() == 1)
-                return;  /* blocking is gp if only one CPU! */
-
-        init_completion(&rcu.completion);
-        /* Will wake me after RCU finished. */
-        call_rcu_sched(&rcu.head, wakeme_after_rcu);
-        /* Wait for it. */
-        wait_for_completion(&rcu.completion);
-}
-EXPORT_SYMBOL_GPL(__synchronize_sched);
-
-/*
- * kthread function that manages call_rcu_sched grace periods.
- */
-static int rcu_sched_grace_period(void *arg)
-{
-        int couldsleep;         /* might sleep after current pass. */
-        int couldsleepnext = 0; /* might sleep after next pass. */
-        int cpu;
-        unsigned long flags;
-        struct rcu_data *rdp;
-        int ret;
-
-        /*
-         * Each pass through the following loop handles one
-         * rcu_sched grace period cycle.
-         */
-        do {
-                /* Save each CPU's current state. */
-
-                for_each_online_cpu(cpu) {
-                        dyntick_save_progress_counter_sched(cpu);
-                        save_qsctr_sched(cpu);
-                }
-
-                /*
-                 * Sleep for about an RCU grace-period's worth to
-                 * allow better batching and to consume less CPU.
-                 */
-                schedule_timeout_interruptible(RCU_SCHED_BATCH_TIME);
-
-                /*
-                 * If there was nothing to do last time, prepare to
-                 * sleep at the end of the current grace period cycle.
-                 */
-                couldsleep = couldsleepnext;
-                couldsleepnext = 1;
-                if (couldsleep) {
-                        spin_lock_irqsave(&rcu_ctrlblk.schedlock, flags);
-                        rcu_ctrlblk.sched_sleep = rcu_sched_sleep_prep;
-                        spin_unlock_irqrestore(&rcu_ctrlblk.schedlock, flags);
-                }
-
-                /*
-                 * Wait on each CPU in turn to have either visited
-                 * a quiescent state or been in dynticks-idle mode.
-                 */
-                for_each_online_cpu(cpu) {
-                        while (rcu_qsctr_inc_needed(cpu) &&
-                               rcu_qsctr_inc_needed_dyntick(cpu)) {
-                                /* resched_cpu(cpu); @@@ */
-                                schedule_timeout_interruptible(1);
-                        }
-                }
-
-                /* Advance callbacks for each CPU.  */
-
-                for_each_online_cpu(cpu) {
-
-                        rdp = RCU_DATA_CPU(cpu);
-                        spin_lock_irqsave(&rdp->lock, flags);
-
-                        /*
-                         * We are running on this CPU irq-disabled, so no
-                         * CPU can go offline until we re-enable irqs.
-                         * The current CPU might have already gone
-                         * offline (between the for_each_offline_cpu and
-                         * the spin_lock_irqsave), but in that case all its
-                         * callback lists will be empty, so no harm done.
-                         *
-                         * Advance the callbacks!  We share normal RCU's
-                         * donelist, since callbacks are invoked the
-                         * same way in either case.
-                         */
-                        if (rdp->waitschedlist != NULL) {
-                                *rdp->donetail = rdp->waitschedlist;
-                                rdp->donetail = rdp->waitschedtail;
-
-                                /*
-                                 * Next rcu_check_callbacks() will
-                                 * do the required raise_softirq().
-                                 */
-                        }
-                        if (rdp->nextschedlist != NULL) {
-                                rdp->waitschedlist = rdp->nextschedlist;
-                                rdp->waitschedtail = rdp->nextschedtail;
-                                couldsleep = 0;
-                                couldsleepnext = 0;
-                        } else {
-                                rdp->waitschedlist = NULL;
-                                rdp->waitschedtail = &rdp->waitschedlist;
-                        }
-                        rdp->nextschedlist = NULL;
-                        rdp->nextschedtail = &rdp->nextschedlist;
-
-                        /* Mark sleep intention. */
-
-                        rdp->rcu_sched_sleeping = couldsleep;
-
-                        spin_unlock_irqrestore(&rdp->lock, flags);
-                }
-
-                /* If we saw callbacks on the last scan, go deal with them. */
-
-                if (!couldsleep)
-                        continue;
-
-                /* Attempt to block... */
-
-                spin_lock_irqsave(&rcu_ctrlblk.schedlock, flags);
-                if (rcu_ctrlblk.sched_sleep != rcu_sched_sleep_prep) {
-
-                        /*
-                         * Someone posted a callback after we scanned.
-                         * Go take care of it.
-                         */
-                        spin_unlock_irqrestore(&rcu_ctrlblk.schedlock, flags);
-                        couldsleepnext = 0;
-                        continue;
-                }
-
-                /* Block until the next person posts a callback. */
-
-                rcu_ctrlblk.sched_sleep = rcu_sched_sleeping;
-                spin_unlock_irqrestore(&rcu_ctrlblk.schedlock, flags);
-                ret = 0; /* unused */
-                __wait_event_interruptible(rcu_ctrlblk.sched_wq,
-                        rcu_ctrlblk.sched_sleep != rcu_sched_sleeping,
-                        ret);
-
-                couldsleepnext = 0;
-
-        } while (!kthread_should_stop());
-
-        return (0);
-}
-
-/*
- * Check to see if any future RCU-related work will need to be done
- * by the current CPU, even if none need be done immediately, returning
- * 1 if so.  Assumes that notifiers would take care of handling any
- * outstanding requests from the RCU core.
- *
- * This function is part of the RCU implementation; it is -not-
- * an exported member of the RCU API.
- */
-int rcu_needs_cpu(int cpu)
-{
-        struct rcu_data *rdp = RCU_DATA_CPU(cpu);
-
-        return (rdp->donelist != NULL ||
-                !!rdp->waitlistcount ||
-                rdp->nextlist != NULL ||
-                rdp->nextschedlist != NULL ||
-                rdp->waitschedlist != NULL);
-}
-
-int rcu_pending(int cpu)
-{
-        struct rcu_data *rdp = RCU_DATA_CPU(cpu);
-
-        /* The CPU has at least one callback queued somewhere. */
-
-        if (rdp->donelist != NULL ||
-            !!rdp->waitlistcount ||
-            rdp->nextlist != NULL ||
-            rdp->nextschedlist != NULL ||
-            rdp->waitschedlist != NULL)
-                return 1;
-
-        /* The RCU core needs an acknowledgement from this CPU. */
-
-        if ((per_cpu(rcu_flip_flag, cpu) == rcu_flipped) ||
-            (per_cpu(rcu_mb_flag, cpu) == rcu_mb_needed))
-                return 1;
-
-        /* This CPU has fallen behind the global grace-period number. */
-
-        if (rdp->completed != rcu_ctrlblk.completed)
-                return 1;
-
-        /* Nothing needed from this CPU. */
-
-        return 0;
-}
-
-static int __cpuinit rcu_cpu_notify(struct notifier_block *self,
-                                unsigned long action, void *hcpu)
-{
-        long cpu = (long)hcpu;
-
-        switch (action) {
-        case CPU_UP_PREPARE:
-        case CPU_UP_PREPARE_FROZEN:
-                rcu_online_cpu(cpu);
-                break;
-        case CPU_UP_CANCELED:
-        case CPU_UP_CANCELED_FROZEN:
-        case CPU_DEAD:
-        case CPU_DEAD_FROZEN:
-                rcu_offline_cpu(cpu);
-                break;
-        default:
-                break;
-        }
-        return NOTIFY_OK;
-}
-
-static struct notifier_block __cpuinitdata rcu_nb = {
-        .notifier_call = rcu_cpu_notify,
-};
-
-void __init __rcu_init(void)
-{
-        int cpu;
-        int i;
-        struct rcu_data *rdp;
-
-        printk(KERN_NOTICE "Preemptible RCU implementation.\n");
-        for_each_possible_cpu(cpu) {
-                rdp = RCU_DATA_CPU(cpu);
-                spin_lock_init(&rdp->lock);
-                rdp->completed = 0;
-                rdp->waitlistcount = 0;
-                rdp->nextlist = NULL;
-                rdp->nexttail = &rdp->nextlist;
-                for (i = 0; i < GP_STAGES; i++) {
-                        rdp->waitlist[i] = NULL;
-                        rdp->waittail[i] = &rdp->waitlist[i];
-                }
-                rdp->donelist = NULL;
-                rdp->donetail = &rdp->donelist;
-                rdp->rcu_flipctr[0] = 0;
-                rdp->rcu_flipctr[1] = 0;
-                rdp->nextschedlist = NULL;
-                rdp->nextschedtail = &rdp->nextschedlist;
-                rdp->waitschedlist = NULL;
-                rdp->waitschedtail = &rdp->waitschedlist;
-                rdp->rcu_sched_sleeping = 0;
-        }
-        register_cpu_notifier(&rcu_nb);
-
-        /*
-         * We don't need protection against CPU-Hotplug here
-         * since
-         * a) If a CPU comes online while we are iterating over the
-         *    cpu_online_mask below, we would only end up making a
-         *    duplicate call to rcu_online_cpu() which sets the corresponding
-         *    CPU's mask in the rcu_cpu_online_map.
-         *
-         * b) A CPU cannot go offline at this point in time since the user
-         *    does not have access to the sysfs interface, nor do we
-         *    suspend the system.
-         */
-        for_each_online_cpu(cpu)
-                rcu_cpu_notify(&rcu_nb, CPU_UP_PREPARE, (void *)(long) cpu);
-
-        open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
-}
-
-/*
- * Late-boot-time RCU initialization that must wait until after scheduler
- * has been initialized.
- */
-void __init rcu_init_sched(void)
-{
-        rcu_sched_grace_period_task = kthread_run(rcu_sched_grace_period,
-                                                  NULL,
-                                                  "rcu_sched_grace_period");
-        WARN_ON(IS_ERR(rcu_sched_grace_period_task));
-}
-
-#ifdef CONFIG_RCU_TRACE
-long *rcupreempt_flipctr(int cpu)
-{
-        return &RCU_DATA_CPU(cpu)->rcu_flipctr[0];
-}
-EXPORT_SYMBOL_GPL(rcupreempt_flipctr);
-
-int rcupreempt_flip_flag(int cpu)
-{
-        return per_cpu(rcu_flip_flag, cpu);
-}
-EXPORT_SYMBOL_GPL(rcupreempt_flip_flag);
-
-int rcupreempt_mb_flag(int cpu)
-{
-        return per_cpu(rcu_mb_flag, cpu);
-}
-EXPORT_SYMBOL_GPL(rcupreempt_mb_flag);
-
-char *rcupreempt_try_flip_state_name(void)
-{
-        return rcu_try_flip_state_names[rcu_ctrlblk.rcu_try_flip_state];
-}
-EXPORT_SYMBOL_GPL(rcupreempt_try_flip_state_name);
-
-struct rcupreempt_trace *rcupreempt_trace_cpu(int cpu)
-{
-        struct rcu_data *rdp = RCU_DATA_CPU(cpu);
-
-        return &rdp->trace;
-}
-EXPORT_SYMBOL_GPL(rcupreempt_trace_cpu);
-
-#endif /* #ifdef RCU_TRACE */
diff --git a/kernel/rcupreempt_trace.c b/kernel/rcupreempt_trace.c
deleted file mode 100644
index 7c2665cac172..000000000000
--- a/kernel/rcupreempt_trace.c
+++ /dev/null
@@ -1,334 +0,0 @@
-/*
- * Read-Copy Update tracing for realtime implementation
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
- *
- * Copyright IBM Corporation, 2006
- *
- * Papers:  http://www.rdrop.com/users/paulmck/RCU
- *
- * For detailed explanation of Read-Copy Update mechanism see -
- *              Documentation/RCU/ *.txt
- *
- */
-#include <linux/types.h>
-#include <linux/kernel.h>
-#include <linux/init.h>
-#include <linux/spinlock.h>
-#include <linux/smp.h>
-#include <linux/rcupdate.h>
-#include <linux/interrupt.h>
-#include <linux/sched.h>
-#include <asm/atomic.h>
-#include <linux/bitops.h>
-#include <linux/module.h>
-#include <linux/completion.h>
-#include <linux/moduleparam.h>
-#include <linux/percpu.h>
-#include <linux/notifier.h>
-#include <linux/cpu.h>
-#include <linux/mutex.h>
-#include <linux/rcupreempt_trace.h>
-#include <linux/debugfs.h>
-
-static struct mutex rcupreempt_trace_mutex;
-static char *rcupreempt_trace_buf;
-#define RCUPREEMPT_TRACE_BUF_SIZE 4096
-
-void rcupreempt_trace_move2done(struct rcupreempt_trace *trace)
-{
-        trace->done_length += trace->wait_length;
-        trace->done_add += trace->wait_length;
-        trace->wait_length = 0;
-}
-void rcupreempt_trace_move2wait(struct rcupreempt_trace *trace)
-{
-        trace->wait_length += trace->next_length;
-        trace->wait_add += trace->next_length;
-        trace->next_length = 0;
-}
-void rcupreempt_trace_try_flip_1(struct rcupreempt_trace *trace)
-{
-        atomic_inc(&trace->rcu_try_flip_1);
-}
-void rcupreempt_trace_try_flip_e1(struct rcupreempt_trace *trace)
-{
-        atomic_inc(&trace->rcu_try_flip_e1);
-}
-void rcupreempt_trace_try_flip_i1(struct rcupreempt_trace *trace)
-{
-        trace->rcu_try_flip_i1++;
-}
-void rcupreempt_trace_try_flip_ie1(struct rcupreempt_trace *trace)
-{
-        trace->rcu_try_flip_ie1++;
-}
-void rcupreempt_trace_try_flip_g1(struct rcupreempt_trace *trace)
-{
-        trace->rcu_try_flip_g1++;
-}
-void rcupreempt_trace_try_flip_a1(struct rcupreempt_trace *trace)
-{
-        trace->rcu_try_flip_a1++;
-}
-void rcupreempt_trace_try_flip_ae1(struct rcupreempt_trace *trace)
-{
-        trace->rcu_try_flip_ae1++;
-}
-void rcupreempt_trace_try_flip_a2(struct rcupreempt_trace *trace)
-{
-        trace->rcu_try_flip_a2++;
-}
-void rcupreempt_trace_try_flip_z1(struct rcupreempt_trace *trace)
-{
-        trace->rcu_try_flip_z1++;
-}
-void rcupreempt_trace_try_flip_ze1(struct rcupreempt_trace *trace)
-{
-        trace->rcu_try_flip_ze1++;
-}
-void rcupreempt_trace_try_flip_z2(struct rcupreempt_trace *trace)
-{
-        trace->rcu_try_flip_z2++;
-}
-void rcupreempt_trace_try_flip_m1(struct rcupreempt_trace *trace)
-{
-        trace->rcu_try_flip_m1++;
-}
-void rcupreempt_trace_try_flip_me1(struct rcupreempt_trace *trace)
-{
-        trace->rcu_try_flip_me1++;
-}
-void rcupreempt_trace_try_flip_m2(struct rcupreempt_trace *trace)
-{
-        trace->rcu_try_flip_m2++;
-}
-void rcupreempt_trace_check_callbacks(struct rcupreempt_trace *trace)
-{
-        trace->rcu_check_callbacks++;
-}
-void rcupreempt_trace_done_remove(struct rcupreempt_trace *trace)
-{
-        trace->done_remove += trace->done_length;
-        trace->done_length = 0;
-}
-void rcupreempt_trace_invoke(struct rcupreempt_trace *trace)
-{
-        atomic_inc(&trace->done_invoked);
-}
-void rcupreempt_trace_next_add(struct rcupreempt_trace *trace)
-{
-        trace->next_add++;
-        trace->next_length++;
-}
-
-static void rcupreempt_trace_sum(struct rcupreempt_trace *sp)
-{
-        struct rcupreempt_trace *cp;
-        int cpu;
-
-        memset(sp, 0, sizeof(*sp));
-        for_each_possible_cpu(cpu) {
-                cp = rcupreempt_trace_cpu(cpu);
-                sp->next_length += cp->next_length;
-                sp->next_add += cp->next_add;
-                sp->wait_length += cp->wait_length;
-                sp->wait_add += cp->wait_add;
-                sp->done_length += cp->done_length;
-                sp->done_add += cp->done_add;
-                sp->done_remove += cp->done_remove;
-                atomic_add(atomic_read(&cp->done_invoked), &sp->done_invoked);
-                sp->rcu_check_callbacks += cp->rcu_check_callbacks;
-                atomic_add(atomic_read(&cp->rcu_try_flip_1),
-                           &sp->rcu_try_flip_1);
-                atomic_add(atomic_read(&cp->rcu_try_flip_e1),
-                           &sp->rcu_try_flip_e1);
-                sp->rcu_try_flip_i1 += cp->rcu_try_flip_i1;
-                sp->rcu_try_flip_ie1 += cp->rcu_try_flip_ie1;
-                sp->rcu_try_flip_g1 += cp->rcu_try_flip_g1;
-                sp->rcu_try_flip_a1 += cp->rcu_try_flip_a1;
-                sp->rcu_try_flip_ae1 += cp->rcu_try_flip_ae1;
-                sp->rcu_try_flip_a2 += cp->rcu_try_flip_a2;
-                sp->rcu_try_flip_z1 += cp->rcu_try_flip_z1;
-                sp->rcu_try_flip_ze1 += cp->rcu_try_flip_ze1;
-                sp->rcu_try_flip_z2 += cp->rcu_try_flip_z2;
-                sp->rcu_try_flip_m1 += cp->rcu_try_flip_m1;
-                sp->rcu_try_flip_me1 += cp->rcu_try_flip_me1;
-                sp->rcu_try_flip_m2 += cp->rcu_try_flip_m2;
-        }
-}
-
-static ssize_t rcustats_read(struct file *filp, char __user *buffer,
-                                size_t count, loff_t *ppos)
-{
-        struct rcupreempt_trace trace;
-        ssize_t bcount;
-        int cnt = 0;
-
-        rcupreempt_trace_sum(&trace);
-        mutex_lock(&rcupreempt_trace_mutex);
-        snprintf(&rcupreempt_trace_buf[cnt], RCUPREEMPT_TRACE_BUF_SIZE - cnt,
-                 "ggp=%ld rcc=%ld\n",
-                 rcu_batches_completed(),
-                 trace.rcu_check_callbacks);
-        snprintf(&rcupreempt_trace_buf[cnt], RCUPREEMPT_TRACE_BUF_SIZE - cnt,
-                 "na=%ld nl=%ld wa=%ld wl=%ld da=%ld dl=%ld dr=%ld di=%d\n"
-                 "1=%d e1=%d i1=%ld ie1=%ld g1=%ld a1=%ld ae1=%ld a2=%ld\n"
-                 "z1=%ld ze1=%ld z2=%ld m1=%ld me1=%ld m2=%ld\n",
-
-                 trace.next_add, trace.next_length,
-                 trace.wait_add, trace.wait_length,
-                 trace.done_add, trace.done_length,
-                 trace.done_remove, atomic_read(&trace.done_invoked),
-                 atomic_read(&trace.rcu_try_flip_1),
-                 atomic_read(&trace.rcu_try_flip_e1),
-                 trace.rcu_try_flip_i1, trace.rcu_try_flip_ie1,
-                 trace.rcu_try_flip_g1,
-                 trace.rcu_try_flip_a1, trace.rcu_try_flip_ae1,
-                         trace.rcu_try_flip_a2,
-                 trace.rcu_try_flip_z1, trace.rcu_try_flip_ze1,
-                         trace.rcu_try_flip_z2,
-                 trace.rcu_try_flip_m1, trace.rcu_try_flip_me1,
-                        trace.rcu_try_flip_m2);
-        bcount = simple_read_from_buffer(buffer, count, ppos,
-                        rcupreempt_trace_buf, strlen(rcupreempt_trace_buf));
-        mutex_unlock(&rcupreempt_trace_mutex);
-        return bcount;
-}
-
-static ssize_t rcugp_read(struct file *filp, char __user *buffer,
-                                size_t count, loff_t *ppos)
-{
-        long oldgp = rcu_batches_completed();
-        ssize_t bcount;
-
-        mutex_lock(&rcupreempt_trace_mutex);
-        synchronize_rcu();
-        snprintf(rcupreempt_trace_buf, RCUPREEMPT_TRACE_BUF_SIZE,
-                "oldggp=%ld  newggp=%ld\n", oldgp, rcu_batches_completed());
-        bcount = simple_read_from_buffer(buffer, count, ppos,
-                        rcupreempt_trace_buf, strlen(rcupreempt_trace_buf));
-        mutex_unlock(&rcupreempt_trace_mutex);
-        return bcount;
-}
-
-static ssize_t rcuctrs_read(struct file *filp, char __user *buffer,
-                                size_t count, loff_t *ppos)
-{
-        int cnt = 0;
-        int cpu;
-        int f = rcu_batches_completed() & 0x1;
-        ssize_t bcount;
-
-        mutex_lock(&rcupreempt_trace_mutex);
-
-        cnt += snprintf(&rcupreempt_trace_buf[cnt], RCUPREEMPT_TRACE_BUF_SIZE,
-                                "CPU last cur F M\n");
-        for_each_online_cpu(cpu) {
-                long *flipctr = rcupreempt_flipctr(cpu);
-                cnt += snprintf(&rcupreempt_trace_buf[cnt],
-                                RCUPREEMPT_TRACE_BUF_SIZE - cnt,
-                                        "%3d %4ld %3ld %d %d\n",
-                               cpu,
-                               flipctr[!f],
-                               flipctr[f],
-                               rcupreempt_flip_flag(cpu),
-                               rcupreempt_mb_flag(cpu));
-        }
-        cnt += snprintf(&rcupreempt_trace_buf[cnt],
-                        RCUPREEMPT_TRACE_BUF_SIZE - cnt,
-                        "ggp = %ld, state = %s\n",
-                        rcu_batches_completed(),
-                        rcupreempt_try_flip_state_name());
-        cnt += snprintf(&rcupreempt_trace_buf[cnt],
-                        RCUPREEMPT_TRACE_BUF_SIZE - cnt,
-                        "\n");
-        bcount = simple_read_from_buffer(buffer, count, ppos,
-                        rcupreempt_trace_buf, strlen(rcupreempt_trace_buf));
-        mutex_unlock(&rcupreempt_trace_mutex);
-        return bcount;
-}
-
-static struct file_operations rcustats_fops = {
-        .owner = THIS_MODULE,
-        .read = rcustats_read,
-};
-
-static struct file_operations rcugp_fops = {
-        .owner = THIS_MODULE,
-        .read = rcugp_read,
-};
-
-static struct file_operations rcuctrs_fops = {
-        .owner = THIS_MODULE,
-        .read = rcuctrs_read,
-};
-
-static struct dentry *rcudir, *statdir, *ctrsdir, *gpdir;
-static int rcupreempt_debugfs_init(void)
-{
-        rcudir = debugfs_create_dir("rcu", NULL);
-        if (!rcudir)
-                goto out;
-        statdir = debugfs_create_file("rcustats", 0444, rcudir,
-                                                NULL, &rcustats_fops);
-        if (!statdir)
-                goto free_out;
-
-        gpdir = debugfs_create_file("rcugp", 0444, rcudir, NULL, &rcugp_fops);
-        if (!gpdir)
-                goto free_out;
-
-        ctrsdir = debugfs_create_file("rcuctrs", 0444, rcudir,
-                                                NULL, &rcuctrs_fops);
-        if (!ctrsdir)
-                goto free_out;
-        return 0;
-free_out:
-        if (statdir)
-                debugfs_remove(statdir);
-        if (gpdir)
-                debugfs_remove(gpdir);
-        debugfs_remove(rcudir);
-out:
-        return 1;
-}
-
-static int __init rcupreempt_trace_init(void)
-{
-        int ret;
-
-        mutex_init(&rcupreempt_trace_mutex);
-        rcupreempt_trace_buf = kmalloc(RCUPREEMPT_TRACE_BUF_SIZE, GFP_KERNEL);
-        if (!rcupreempt_trace_buf)
-                return 1;
-        ret = rcupreempt_debugfs_init();
-        if (ret)
-                kfree(rcupreempt_trace_buf);
-        return ret;
-}
-
-static void __exit rcupreempt_trace_cleanup(void)
-{
-        debugfs_remove(statdir);
-        debugfs_remove(gpdir);
-        debugfs_remove(ctrsdir);
-        debugfs_remove(rcudir);
-        kfree(rcupreempt_trace_buf);
-}
-
-
-module_init(rcupreempt_trace_init);
-module_exit(rcupreempt_trace_cleanup);
diff --git a/kernel/rcutorture.c b/kernel/rcutorture.c
index 9b4a975a4b4a..b33db539a8ad 100644
--- a/kernel/rcutorture.c
+++ b/kernel/rcutorture.c
@@ -257,14 +257,14 @@ struct rcu_torture_ops {
         void (*init)(void);
         void (*cleanup)(void);
         int (*readlock)(void);
-        void (*readdelay)(struct rcu_random_state *rrsp);
+        void (*read_delay)(struct rcu_random_state *rrsp);
         void (*readunlock)(int idx);
         int (*completed)(void);
-        void (*deferredfree)(struct rcu_torture *p);
+        void (*deferred_free)(struct rcu_torture *p);
         void (*sync)(void);
         void (*cb_barrier)(void);
         int (*stats)(char *page);
-        int irqcapable;
+        int irq_capable;
         char *name;
 };
 static struct rcu_torture_ops *cur_ops = NULL;
@@ -320,7 +320,7 @@ rcu_torture_cb(struct rcu_head *p)
                 rp->rtort_mbtest = 0;
                 rcu_torture_free(rp);
         } else
-                cur_ops->deferredfree(rp);
+                cur_ops->deferred_free(rp);
 }
 
 static void rcu_torture_deferred_free(struct rcu_torture *p)
@@ -329,18 +329,18 @@ static void rcu_torture_deferred_free(struct rcu_torture *p)
 }
 
 static struct rcu_torture_ops rcu_ops = {
-        .init = NULL,
-        .cleanup = NULL,
-        .readlock = rcu_torture_read_lock,
-        .readdelay = rcu_read_delay,
-        .readunlock = rcu_torture_read_unlock,
-        .completed = rcu_torture_completed,
-        .deferredfree = rcu_torture_deferred_free,
-        .sync = synchronize_rcu,
-        .cb_barrier = rcu_barrier,
-        .stats = NULL,
-        .irqcapable = 1,
-        .name = "rcu"
+        .init           = NULL,
+        .cleanup        = NULL,
+        .readlock       = rcu_torture_read_lock,
+        .read_delay     = rcu_read_delay,
+        .readunlock     = rcu_torture_read_unlock,
+        .completed      = rcu_torture_completed,
+        .deferred_free  = rcu_torture_deferred_free,
+        .sync           = synchronize_rcu,
+        .cb_barrier     = rcu_barrier,
+        .stats          = NULL,
+        .irq_capable    = 1,
+        .name           = "rcu"
 };
 
 static void rcu_sync_torture_deferred_free(struct rcu_torture *p)
@@ -370,18 +370,18 @@ static void rcu_sync_torture_init(void)
 }
 
 static struct rcu_torture_ops rcu_sync_ops = {
-        .init = rcu_sync_torture_init,
-        .cleanup = NULL,
-        .readlock = rcu_torture_read_lock,
-        .readdelay = rcu_read_delay,
-        .readunlock = rcu_torture_read_unlock,
-        .completed = rcu_torture_completed,
-        .deferredfree = rcu_sync_torture_deferred_free,
-        .sync = synchronize_rcu,
-        .cb_barrier = NULL,
-        .stats = NULL,
-        .irqcapable = 1,
-        .name = "rcu_sync"
+        .init           = rcu_sync_torture_init,
+        .cleanup        = NULL,
+        .readlock       = rcu_torture_read_lock,
+        .read_delay     = rcu_read_delay,
+        .readunlock     = rcu_torture_read_unlock,
+        .completed      = rcu_torture_completed,
+        .deferred_free  = rcu_sync_torture_deferred_free,
+        .sync           = synchronize_rcu,
+        .cb_barrier     = NULL,
+        .stats          = NULL,
+        .irq_capable    = 1,
+        .name           = "rcu_sync"
 };
 
 /*
@@ -432,33 +432,33 @@ static void rcu_bh_torture_synchronize(void)
 }
 
 static struct rcu_torture_ops rcu_bh_ops = {
-        .init = NULL,
-        .cleanup = NULL,
-        .readlock = rcu_bh_torture_read_lock,
-        .readdelay = rcu_read_delay,  /* just reuse rcu's version. */
-        .readunlock = rcu_bh_torture_read_unlock,
-        .completed = rcu_bh_torture_completed,
-        .deferredfree = rcu_bh_torture_deferred_free,
-        .sync = rcu_bh_torture_synchronize,
-        .cb_barrier = rcu_barrier_bh,
-        .stats = NULL,
-        .irqcapable = 1,
-        .name = "rcu_bh"
+        .init           = NULL,
+        .cleanup        = NULL,
+        .readlock       = rcu_bh_torture_read_lock,
+        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
+        .readunlock     = rcu_bh_torture_read_unlock,
+        .completed      = rcu_bh_torture_completed,
+        .deferred_free  = rcu_bh_torture_deferred_free,
+        .sync           = rcu_bh_torture_synchronize,
+        .cb_barrier     = rcu_barrier_bh,
+        .stats          = NULL,
+        .irq_capable    = 1,
+        .name           = "rcu_bh"
 };
 
 static struct rcu_torture_ops rcu_bh_sync_ops = {
-        .init = rcu_sync_torture_init,
-        .cleanup = NULL,
-        .readlock = rcu_bh_torture_read_lock,
-        .readdelay = rcu_read_delay,  /* just reuse rcu's version. */
-        .readunlock = rcu_bh_torture_read_unlock,
-        .completed = rcu_bh_torture_completed,
-        .deferredfree = rcu_sync_torture_deferred_free,
-        .sync = rcu_bh_torture_synchronize,
-        .cb_barrier = NULL,
-        .stats = NULL,
-        .irqcapable = 1,
-        .name = "rcu_bh_sync"
+        .init           = rcu_sync_torture_init,
+        .cleanup        = NULL,
+        .readlock       = rcu_bh_torture_read_lock,
+        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
+        .readunlock     = rcu_bh_torture_read_unlock,
+        .completed      = rcu_bh_torture_completed,
+        .deferred_free  = rcu_sync_torture_deferred_free,
+        .sync           = rcu_bh_torture_synchronize,
+        .cb_barrier     = NULL,
+        .stats          = NULL,
+        .irq_capable    = 1,
+        .name           = "rcu_bh_sync"
 };
 
 /*
@@ -530,17 +530,17 @@ static int srcu_torture_stats(char *page)
 }
 
 static struct rcu_torture_ops srcu_ops = {
-        .init = srcu_torture_init,
-        .cleanup = srcu_torture_cleanup,
-        .readlock = srcu_torture_read_lock,
-        .readdelay = srcu_read_delay,
-        .readunlock = srcu_torture_read_unlock,
-        .completed = srcu_torture_completed,
-        .deferredfree = rcu_sync_torture_deferred_free,
-        .sync = srcu_torture_synchronize,
-        .cb_barrier = NULL,
-        .stats = srcu_torture_stats,
-        .name = "srcu"
+        .init           = srcu_torture_init,
+        .cleanup        = srcu_torture_cleanup,
+        .readlock       = srcu_torture_read_lock,
+        .read_delay     = srcu_read_delay,
+        .readunlock     = srcu_torture_read_unlock,
+        .completed      = srcu_torture_completed,
+        .deferred_free  = rcu_sync_torture_deferred_free,
+        .sync           = srcu_torture_synchronize,
+        .cb_barrier     = NULL,
+        .stats          = srcu_torture_stats,
+        .name           = "srcu"
 };
 
 /*
@@ -574,32 +574,49 @@ static void sched_torture_synchronize(void)
 }
 
 static struct rcu_torture_ops sched_ops = {
-        .init = rcu_sync_torture_init,
-        .cleanup = NULL,
-        .readlock = sched_torture_read_lock,
-        .readdelay = rcu_read_delay,  /* just reuse rcu's version. */
-        .readunlock = sched_torture_read_unlock,
-        .completed = sched_torture_completed,
-        .deferredfree = rcu_sched_torture_deferred_free,
-        .sync = sched_torture_synchronize,
-        .cb_barrier = rcu_barrier_sched,
-        .stats = NULL,
-        .irqcapable = 1,
-        .name = "sched"
+        .init           = rcu_sync_torture_init,
+        .cleanup        = NULL,
+        .readlock       = sched_torture_read_lock,
+        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
+        .readunlock     = sched_torture_read_unlock,
+        .completed      = sched_torture_completed,
+        .deferred_free  = rcu_sched_torture_deferred_free,
+        .sync           = sched_torture_synchronize,
+        .cb_barrier     = rcu_barrier_sched,
+        .stats          = NULL,
+        .irq_capable    = 1,
+        .name           = "sched"
 };
 
 static struct rcu_torture_ops sched_ops_sync = {
-        .init = rcu_sync_torture_init,
-        .cleanup = NULL,
-        .readlock = sched_torture_read_lock,
-        .readdelay = rcu_read_delay,  /* just reuse rcu's version. */
-        .readunlock = sched_torture_read_unlock,
-        .completed = sched_torture_completed,
-        .deferredfree = rcu_sync_torture_deferred_free,
-        .sync = sched_torture_synchronize,
-        .cb_barrier = NULL,
-        .stats = NULL,
-        .name = "sched_sync"
+        .init           = rcu_sync_torture_init,
+        .cleanup        = NULL,
+        .readlock       = sched_torture_read_lock,
+        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
+        .readunlock     = sched_torture_read_unlock,
+        .completed      = sched_torture_completed,
+        .deferred_free  = rcu_sync_torture_deferred_free,
+        .sync           = sched_torture_synchronize,
+        .cb_barrier     = NULL,
+        .stats          = NULL,
+        .name           = "sched_sync"
+};
+
+extern int rcu_expedited_torture_stats(char *page);
+
+static struct rcu_torture_ops sched_expedited_ops = {
+        .init           = rcu_sync_torture_init,
+        .cleanup        = NULL,
+        .readlock       = sched_torture_read_lock,
+        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
+        .readunlock     = sched_torture_read_unlock,
+        .completed      = sched_torture_completed,
+        .deferred_free  = rcu_sync_torture_deferred_free,
+        .sync           = synchronize_sched_expedited,
+        .cb_barrier     = NULL,
+        .stats          = rcu_expedited_torture_stats,
+        .irq_capable    = 1,
+        .name           = "sched_expedited"
 };
 
 /*
@@ -635,7 +652,7 @@ rcu_torture_writer(void *arg)
                                 i = RCU_TORTURE_PIPE_LEN;
                         atomic_inc(&rcu_torture_wcount[i]);
                         old_rp->rtort_pipe_count++;
-                        cur_ops->deferredfree(old_rp);
+                        cur_ops->deferred_free(old_rp);
                 }
                 rcu_torture_current_version++;
                 oldbatch = cur_ops->completed();
@@ -700,7 +717,7 @@ static void rcu_torture_timer(unsigned long unused)
         if (p->rtort_mbtest == 0)
                 atomic_inc(&n_rcu_torture_mberror);
         spin_lock(&rand_lock);
-        cur_ops->readdelay(&rand);
+        cur_ops->read_delay(&rand);
         n_rcu_torture_timers++;
         spin_unlock(&rand_lock);
         preempt_disable();
@@ -738,11 +755,11 @@ rcu_torture_reader(void *arg)
 
         VERBOSE_PRINTK_STRING("rcu_torture_reader task started");
         set_user_nice(current, 19);
-        if (irqreader && cur_ops->irqcapable)
+        if (irqreader && cur_ops->irq_capable)
                 setup_timer_on_stack(&t, rcu_torture_timer, 0);
 
         do {
-                if (irqreader && cur_ops->irqcapable) {
+                if (irqreader && cur_ops->irq_capable) {
                         if (!timer_pending(&t))
                                 mod_timer(&t, 1);
                 }
@@ -757,7 +774,7 @@ rcu_torture_reader(void *arg)
                 }
                 if (p->rtort_mbtest == 0)
                         atomic_inc(&n_rcu_torture_mberror);
-                cur_ops->readdelay(&rand);
+                cur_ops->read_delay(&rand);
                 preempt_disable();
                 pipe_count = p->rtort_pipe_count;
                 if (pipe_count > RCU_TORTURE_PIPE_LEN) {
@@ -778,7 +795,7 @@ rcu_torture_reader(void *arg)
         } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
         VERBOSE_PRINTK_STRING("rcu_torture_reader task stopping");
         rcutorture_shutdown_absorb("rcu_torture_reader");
-        if (irqreader && cur_ops->irqcapable)
+        if (irqreader && cur_ops->irq_capable)
                 del_timer_sync(&t);
         while (!kthread_should_stop())
                 schedule_timeout_uninterruptible(1);
@@ -1078,6 +1095,7 @@ rcu_torture_init(void)
         int firsterr = 0;
         static struct rcu_torture_ops *torture_ops[] =
                 { &rcu_ops, &rcu_sync_ops, &rcu_bh_ops, &rcu_bh_sync_ops,
+                  &sched_expedited_ops,
                   &srcu_ops, &sched_ops, &sched_ops_sync, };
 
         mutex_lock(&fullstop_mutex);
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index 7717b95c2027..6b11b07cfe7f 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -35,6 +35,7 @@
 #include <linux/rcupdate.h>
 #include <linux/interrupt.h>
 #include <linux/sched.h>
+#include <linux/nmi.h>
 #include <asm/atomic.h>
 #include <linux/bitops.h>
 #include <linux/module.h>
@@ -46,6 +47,8 @@
 #include <linux/mutex.h>
 #include <linux/time.h>
 
+#include "rcutree.h"
+
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 static struct lock_class_key rcu_lock_key;
 struct lockdep_map rcu_lock_map =
@@ -72,30 +75,59 @@ EXPORT_SYMBOL_GPL(rcu_lock_map);
         .n_force_qs_ngp = 0, \
 }
 
-struct rcu_state rcu_state = RCU_STATE_INITIALIZER(rcu_state);
-DEFINE_PER_CPU(struct rcu_data, rcu_data);
+struct rcu_state rcu_sched_state = RCU_STATE_INITIALIZER(rcu_sched_state);
+DEFINE_PER_CPU(struct rcu_data, rcu_sched_data);
 
 struct rcu_state rcu_bh_state = RCU_STATE_INITIALIZER(rcu_bh_state);
 DEFINE_PER_CPU(struct rcu_data, rcu_bh_data);
 
+extern long rcu_batches_completed_sched(void);
+static struct rcu_node *rcu_get_root(struct rcu_state *rsp);
+static void cpu_quiet_msk(unsigned long mask, struct rcu_state *rsp,
+                          struct rcu_node *rnp, unsigned long flags);
+static void cpu_quiet_msk_finish(struct rcu_state *rsp, unsigned long flags);
+#ifdef CONFIG_HOTPLUG_CPU
+static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp);
+#endif /* #ifdef CONFIG_HOTPLUG_CPU */
+static void __rcu_process_callbacks(struct rcu_state *rsp,
+                                    struct rcu_data *rdp);
+static void __call_rcu(struct rcu_head *head,
+                       void (*func)(struct rcu_head *rcu),
+                       struct rcu_state *rsp);
+static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp);
+static void __cpuinit rcu_init_percpu_data(int cpu, struct rcu_state *rsp,
+                                           int preemptable);
+
+#include "rcutree_plugin.h"
+
 /*
- * Increment the quiescent state counter.
- * The counter is a bit degenerated: We do not need to know
+ * Note a quiescent state.  Because we do not need to know
  * how many quiescent states passed, just if there was at least
- * one since the start of the grace period. Thus just a flag.
+ * one since the start of the grace period, this just sets a flag.
  */
-void rcu_qsctr_inc(int cpu)
+void rcu_sched_qs(int cpu)
 {
-        struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
+        unsigned long flags;
+        struct rcu_data *rdp;
+
+        local_irq_save(flags);
+        rdp = &per_cpu(rcu_sched_data, cpu);
         rdp->passed_quiesc = 1;
         rdp->passed_quiesc_completed = rdp->completed;
+        rcu_preempt_qs(cpu);
+        local_irq_restore(flags);
 }
 
-void rcu_bh_qsctr_inc(int cpu)
+void rcu_bh_qs(int cpu)
 {
-        struct rcu_data *rdp = &per_cpu(rcu_bh_data, cpu);
+        unsigned long flags;
+        struct rcu_data *rdp;
+
+        local_irq_save(flags);
+        rdp = &per_cpu(rcu_bh_data, cpu);
         rdp->passed_quiesc = 1;
         rdp->passed_quiesc_completed = rdp->completed;
+        local_irq_restore(flags);
 }
 
 #ifdef CONFIG_NO_HZ
@@ -110,15 +142,16 @@ static int qhimark = 10000;	/* If this many pending, ignore blimit. */
 static int qlowmark = 100;      /* Once only this many pending, use blimit. */
 
 static void force_quiescent_state(struct rcu_state *rsp, int relaxed);
+static int rcu_pending(int cpu);
 
 /*
- * Return the number of RCU batches processed thus far for debug & stats.
+ * Return the number of RCU-sched batches processed thus far for debug & stats.
  */
-long rcu_batches_completed(void)
+long rcu_batches_completed_sched(void)
 {
-        return rcu_state.completed;
+        return rcu_sched_state.completed;
 }
-EXPORT_SYMBOL_GPL(rcu_batches_completed);
+EXPORT_SYMBOL_GPL(rcu_batches_completed_sched);
 
 /*
  * Return the number of RCU BH batches processed thus far for debug & stats.
@@ -181,6 +214,10 @@ static int rcu_implicit_offline_qs(struct rcu_data *rdp)
                 return 1;
         }
 
+        /* If preemptable RCU, no point in sending reschedule IPI. */
+        if (rdp->preemptable)
+                return 0;
+
         /* The CPU is online, so send it a reschedule IPI. */
         if (rdp->cpu != smp_processor_id())
                 smp_send_reschedule(rdp->cpu);
@@ -193,7 +230,6 @@ static int rcu_implicit_offline_qs(struct rcu_data *rdp)
 #endif /* #ifdef CONFIG_SMP */
 
 #ifdef CONFIG_NO_HZ
-static DEFINE_RATELIMIT_STATE(rcu_rs, 10 * HZ, 5);
 
 /**
  * rcu_enter_nohz - inform RCU that current CPU is entering nohz
@@ -213,7 +249,7 @@ void rcu_enter_nohz(void)
         rdtp = &__get_cpu_var(rcu_dynticks);
         rdtp->dynticks++;
         rdtp->dynticks_nesting--;
-        WARN_ON_RATELIMIT(rdtp->dynticks & 0x1, &rcu_rs);
+        WARN_ON_ONCE(rdtp->dynticks & 0x1);
         local_irq_restore(flags);
 }
 
@@ -232,7 +268,7 @@ void rcu_exit_nohz(void)
         rdtp = &__get_cpu_var(rcu_dynticks);
         rdtp->dynticks++;
         rdtp->dynticks_nesting++;
-        WARN_ON_RATELIMIT(!(rdtp->dynticks & 0x1), &rcu_rs);
+        WARN_ON_ONCE(!(rdtp->dynticks & 0x1));
         local_irq_restore(flags);
         smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */
 }
@@ -251,7 +287,7 @@ void rcu_nmi_enter(void)
         if (rdtp->dynticks & 0x1)
                 return;
         rdtp->dynticks_nmi++;
-        WARN_ON_RATELIMIT(!(rdtp->dynticks_nmi & 0x1), &rcu_rs);
+        WARN_ON_ONCE(!(rdtp->dynticks_nmi & 0x1));
         smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */
 }
 
@@ -270,7 +306,7 @@ void rcu_nmi_exit(void)
                 return;
         smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */
         rdtp->dynticks_nmi++;
-        WARN_ON_RATELIMIT(rdtp->dynticks_nmi & 0x1, &rcu_rs);
+        WARN_ON_ONCE(rdtp->dynticks_nmi & 0x1);
 }
 
 /**
@@ -286,7 +322,7 @@ void rcu_irq_enter(void)
         if (rdtp->dynticks_nesting++)
                 return;
         rdtp->dynticks++;
-        WARN_ON_RATELIMIT(!(rdtp->dynticks & 0x1), &rcu_rs);
+        WARN_ON_ONCE(!(rdtp->dynticks & 0x1));
         smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */
 }
 
@@ -305,10 +341,10 @@ void rcu_irq_exit(void)
                 return;
         smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */
         rdtp->dynticks++;
-        WARN_ON_RATELIMIT(rdtp->dynticks & 0x1, &rcu_rs);
+        WARN_ON_ONCE(rdtp->dynticks & 0x1);
 
         /* If the interrupt queued a callback, get out of dyntick mode. */
-        if (__get_cpu_var(rcu_data).nxtlist ||
+        if (__get_cpu_var(rcu_sched_data).nxtlist ||
             __get_cpu_var(rcu_bh_data).nxtlist)
                 set_need_resched();
 }
@@ -461,6 +497,7 @@ static void print_other_cpu_stall(struct rcu_state *rsp)
 
         printk(KERN_ERR "INFO: RCU detected CPU stalls:");
         for (; rnp_cur < rnp_end; rnp_cur++) {
+                rcu_print_task_stall(rnp);
                 if (rnp_cur->qsmask == 0)
                         continue;
                 for (cpu = 0; cpu <= rnp_cur->grphi - rnp_cur->grplo; cpu++)
@@ -469,6 +506,8 @@ static void print_other_cpu_stall(struct rcu_state *rsp)
         }
         printk(" (detected by %d, t=%ld jiffies)\n",
                smp_processor_id(), (long)(jiffies - rsp->gp_start));
+        trigger_all_cpu_backtrace();
+
         force_quiescent_state(rsp, 0);  /* Kick them all. */
 }
 
@@ -479,12 +518,14 @@ static void print_cpu_stall(struct rcu_state *rsp)
 
         printk(KERN_ERR "INFO: RCU detected CPU %d stall (t=%lu jiffies)\n",
                         smp_processor_id(), jiffies - rsp->gp_start);
-        dump_stack();
+        trigger_all_cpu_backtrace();
+
         spin_lock_irqsave(&rnp->lock, flags);
         if ((long)(jiffies - rsp->jiffies_stall) >= 0)
                 rsp->jiffies_stall =
                         jiffies + RCU_SECONDS_TILL_STALL_RECHECK;
         spin_unlock_irqrestore(&rnp->lock, flags);
+
         set_need_resched();  /* kick ourselves to get things going. */
 }
 
@@ -674,6 +715,19 @@ rcu_process_gp_end(struct rcu_state *rsp, struct rcu_data *rdp)
 }
 
 /*
+ * Clean up after the prior grace period and let rcu_start_gp() start up
+ * the next grace period if one is needed.  Note that the caller must
+ * hold rnp->lock, as required by rcu_start_gp(), which will release it.
+ */
+static void cpu_quiet_msk_finish(struct rcu_state *rsp, unsigned long flags)
+        __releases(rnp->lock)
+{
+        rsp->completed = rsp->gpnum;
+        rcu_process_gp_end(rsp, rsp->rda[smp_processor_id()]);
+        rcu_start_gp(rsp, flags);  /* releases root node's rnp->lock. */
+}
+
+/*
  * Similar to cpu_quiet(), for which it is a helper function.  Allows
  * a group of CPUs to be quieted at one go, though all the CPUs in the
  * group must be represented by the same leaf rcu_node structure.
@@ -694,7 +748,7 @@ cpu_quiet_msk(unsigned long mask, struct rcu_state *rsp, struct rcu_node *rnp,
                         return;
                 }
                 rnp->qsmask &= ~mask;
-                if (rnp->qsmask != 0) {
+                if (rnp->qsmask != 0 || rcu_preempted_readers(rnp)) {
 
                         /* Other bits still set at this level, so done. */
                         spin_unlock_irqrestore(&rnp->lock, flags);
@@ -714,14 +768,10 @@ cpu_quiet_msk(unsigned long mask, struct rcu_state *rsp, struct rcu_node *rnp,
 
         /*
          * Get here if we are the last CPU to pass through a quiescent
-         * state for this grace period.  Clean up and let rcu_start_gp()
-         * start up the next grace period if one is needed.  Note that
-         * we still hold rnp->lock, as required by rcu_start_gp(), which
-         * will release it.
+         * state for this grace period.  Invoke cpu_quiet_msk_finish()
+         * to clean up and start the next grace period if one is needed.
          */
-        rsp->completed = rsp->gpnum;
-        rcu_process_gp_end(rsp, rsp->rda[smp_processor_id()]);
-        rcu_start_gp(rsp, flags);  /* releases rnp->lock. */
+        cpu_quiet_msk_finish(rsp, flags); /* releases rnp->lock. */
 }
 
 /*
@@ -828,11 +878,12 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)
                 spin_lock(&rnp->lock);          /* irqs already disabled. */
                 rnp->qsmaskinit &= ~mask;
                 if (rnp->qsmaskinit != 0) {
-                        spin_unlock(&rnp->lock); /* irqs already disabled. */
+                        spin_unlock(&rnp->lock); /* irqs remain disabled. */
                         break;
                 }
+                rcu_preempt_offline_tasks(rsp, rnp);
                 mask = rnp->grpmask;
-                spin_unlock(&rnp->lock);        /* irqs already disabled. */
+                spin_unlock(&rnp->lock);        /* irqs remain disabled. */
                 rnp = rnp->parent;
         } while (rnp != NULL);
         lastcomp = rsp->completed;
@@ -845,7 +896,7 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)
         /*
          * Move callbacks from the outgoing CPU to the running CPU.
          * Note that the outgoing CPU is now quiscent, so it is now
-         * (uncharacteristically) safe to access it rcu_data structure.
+         * (uncharacteristically) safe to access its rcu_data structure.
          * Note also that we must carefully retain the order of the
          * outgoing CPU's callbacks in order for rcu_barrier() to work
          * correctly.  Finally, note that we start all the callbacks
@@ -876,8 +927,9 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)
  */
 static void rcu_offline_cpu(int cpu)
 {
-        __rcu_offline_cpu(cpu, &rcu_state);
+        __rcu_offline_cpu(cpu, &rcu_sched_state);
         __rcu_offline_cpu(cpu, &rcu_bh_state);
+        rcu_preempt_offline_cpu(cpu);
 }
 
 #else /* #ifdef CONFIG_HOTPLUG_CPU */
@@ -963,6 +1015,8 @@ static void rcu_do_batch(struct rcu_data *rdp)
  */
 void rcu_check_callbacks(int cpu, int user)
 {
+        if (!rcu_pending(cpu))
+                return; /* if nothing for RCU to do. */
         if (user ||
             (idle_cpu(cpu) && rcu_scheduler_active &&
              !in_softirq() && hardirq_count() <= (1 << HARDIRQ_SHIFT))) {
@@ -971,17 +1025,16 @@ void rcu_check_callbacks(int cpu, int user)
                  * Get here if this CPU took its interrupt from user
                  * mode or from the idle loop, and if this is not a
                  * nested interrupt.  In this case, the CPU is in
-                 * a quiescent state, so count it.
+                 * a quiescent state, so note it.
                  *
                  * No memory barrier is required here because both
-                 * rcu_qsctr_inc() and rcu_bh_qsctr_inc() reference
-                 * only CPU-local variables that other CPUs neither
-                 * access nor modify, at least not while the corresponding
-                 * CPU is online.
+                 * rcu_sched_qs() and rcu_bh_qs() reference only CPU-local
+                 * variables that other CPUs neither access nor modify,
+                 * at least not while the corresponding CPU is online.
                  */
 
-                rcu_qsctr_inc(cpu);
-                rcu_bh_qsctr_inc(cpu);
+                rcu_sched_qs(cpu);
+                rcu_bh_qs(cpu);
 
         } else if (!in_softirq()) {
 
@@ -989,11 +1042,12 @@ void rcu_check_callbacks(int cpu, int user)
                  * Get here if this CPU did not take its interrupt from
                  * softirq, in other words, if it is not interrupting
                  * a rcu_bh read-side critical section.  This is an _bh
-                 * critical section, so count it.
+                 * critical section, so note it.
                  */
 
-                rcu_bh_qsctr_inc(cpu);
+                rcu_bh_qs(cpu);
         }
+        rcu_preempt_check_callbacks(cpu);
         raise_softirq(RCU_SOFTIRQ);
 }
 
@@ -1132,6 +1186,8 @@ __rcu_process_callbacks(struct rcu_state *rsp, struct rcu_data *rdp)
 {
         unsigned long flags;
 
+        WARN_ON_ONCE(rdp->beenonline == 0);
+
         /*
          * If an RCU GP has gone long enough, go check for dyntick
          * idle CPUs and, if needed, send resched IPIs.
@@ -1170,8 +1226,10 @@ static void rcu_process_callbacks(struct softirq_action *unused)
          */
         smp_mb(); /* See above block comment. */
 
-        __rcu_process_callbacks(&rcu_state, &__get_cpu_var(rcu_data));
+        __rcu_process_callbacks(&rcu_sched_state,
+                                &__get_cpu_var(rcu_sched_data));
         __rcu_process_callbacks(&rcu_bh_state, &__get_cpu_var(rcu_bh_data));
+        rcu_preempt_process_callbacks();
 
         /*
          * Memory references from any later RCU read-side critical sections
@@ -1227,13 +1285,13 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
 }
 
 /*
- * Queue an RCU callback for invocation after a grace period.
+ * Queue an RCU-sched callback for invocation after a grace period.
  */
-void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
+void call_rcu_sched(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
 {
-        __call_rcu(head, func, &rcu_state);
+        __call_rcu(head, func, &rcu_sched_state);
 }
-EXPORT_SYMBOL_GPL(call_rcu);
+EXPORT_SYMBOL_GPL(call_rcu_sched);
 
 /*
  * Queue an RCU for invocation after a quicker grace period.
@@ -1305,10 +1363,11 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp)
  * by the current CPU, returning 1 if so.  This function is part of the
  * RCU implementation; it is -not- an exported member of the RCU API.
  */
-int rcu_pending(int cpu)
+static int rcu_pending(int cpu)
 {
-        return __rcu_pending(&rcu_state, &per_cpu(rcu_data, cpu)) ||
-               __rcu_pending(&rcu_bh_state, &per_cpu(rcu_bh_data, cpu));
+        return __rcu_pending(&rcu_sched_state, &per_cpu(rcu_sched_data, cpu)) ||
+               __rcu_pending(&rcu_bh_state, &per_cpu(rcu_bh_data, cpu)) ||
+               rcu_preempt_pending(cpu);
 }
 
 /*
@@ -1320,27 +1379,46 @@ int rcu_pending(int cpu)
 int rcu_needs_cpu(int cpu)
 {
         /* RCU callbacks either ready or pending? */
-        return per_cpu(rcu_data, cpu).nxtlist ||
-               per_cpu(rcu_bh_data, cpu).nxtlist;
+        return per_cpu(rcu_sched_data, cpu).nxtlist ||
+               per_cpu(rcu_bh_data, cpu).nxtlist ||
+               rcu_preempt_needs_cpu(cpu);
 }
 
 /*
- * Initialize a CPU's per-CPU RCU data.  We take this "scorched earth"
- * approach so that we don't have to worry about how long the CPU has
- * been gone, or whether it ever was online previously.  We do trust the
- * ->mynode field, as it is constant for a given struct rcu_data and
- * initialized during early boot.
- *
- * Note that only one online or offline event can be happening at a given
- * time.  Note also that we can accept some slop in the rsp->completed
- * access due to the fact that this CPU cannot possibly have any RCU
- * callbacks in flight yet.
+ * Do boot-time initialization of a CPU's per-CPU RCU data.
  */
-static void __cpuinit
-rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
+static void __init
+rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp)
 {
         unsigned long flags;
         int i;
+        struct rcu_data *rdp = rsp->rda[cpu];
+        struct rcu_node *rnp = rcu_get_root(rsp);
+
+        /* Set up local state, ensuring consistent view of global state. */
+        spin_lock_irqsave(&rnp->lock, flags);
+        rdp->grpmask = 1UL << (cpu - rdp->mynode->grplo);
+        rdp->nxtlist = NULL;
+        for (i = 0; i < RCU_NEXT_SIZE; i++)
+                rdp->nxttail[i] = &rdp->nxtlist;
+        rdp->qlen = 0;
+#ifdef CONFIG_NO_HZ
+        rdp->dynticks = &per_cpu(rcu_dynticks, cpu);
+#endif /* #ifdef CONFIG_NO_HZ */
+        rdp->cpu = cpu;
+        spin_unlock_irqrestore(&rnp->lock, flags);
+}
+
+/*
+ * Initialize a CPU's per-CPU RCU data.  Note that only one online or
+ * offline event can be happening at a given time.  Note also that we
+ * can accept some slop in the rsp->completed access due to the fact
+ * that this CPU cannot possibly have any RCU callbacks in flight yet.
+ */
+static void __cpuinit
+rcu_init_percpu_data(int cpu, struct rcu_state *rsp, int preemptable)
+{
+        unsigned long flags;
         long lastcomp;
         unsigned long mask;
         struct rcu_data *rdp = rsp->rda[cpu];
@@ -1354,17 +1432,9 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
         rdp->passed_quiesc = 0;  /* We could be racing with new GP, */
         rdp->qs_pending = 1;     /*  so set up to respond to current GP. */
         rdp->beenonline = 1;     /* We have now been online. */
+        rdp->preemptable = preemptable;
         rdp->passed_quiesc_completed = lastcomp - 1;
-        rdp->grpmask = 1UL << (cpu - rdp->mynode->grplo);
-        rdp->nxtlist = NULL;
-        for (i = 0; i < RCU_NEXT_SIZE; i++)
-                rdp->nxttail[i] = &rdp->nxtlist;
-        rdp->qlen = 0;
         rdp->blimit = blimit;
-#ifdef CONFIG_NO_HZ
-        rdp->dynticks = &per_cpu(rcu_dynticks, cpu);
-#endif /* #ifdef CONFIG_NO_HZ */
-        rdp->cpu = cpu;
         spin_unlock(&rnp->lock);                /* irqs remain disabled. */
 
         /*
@@ -1405,16 +1475,16 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
 
 static void __cpuinit rcu_online_cpu(int cpu)
 {
-        rcu_init_percpu_data(cpu, &rcu_state);
-        rcu_init_percpu_data(cpu, &rcu_bh_state);
-        open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
+        rcu_init_percpu_data(cpu, &rcu_sched_state, 0);
+        rcu_init_percpu_data(cpu, &rcu_bh_state, 0);
+        rcu_preempt_init_percpu_data(cpu);
 }
 
 /*
- * Handle CPU online/offline notifcation events.
+ * Handle CPU online/offline notification events.
  */
-static int __cpuinit rcu_cpu_notify(struct notifier_block *self,
-                                unsigned long action, void *hcpu)
+int __cpuinit rcu_cpu_notify(struct notifier_block *self,
+                             unsigned long action, void *hcpu)
 {
         long cpu = (long)hcpu;
 
@@ -1486,6 +1556,7 @@ static void __init rcu_init_one(struct rcu_state *rsp)
                 rnp = rsp->level[i];
                 for (j = 0; j < rsp->levelcnt[i]; j++, rnp++) {
                         spin_lock_init(&rnp->lock);
+                        rnp->gpnum = 0;
                         rnp->qsmask = 0;
                         rnp->qsmaskinit = 0;
                         rnp->grplo = j * cpustride;
@@ -1503,16 +1574,20 @@ static void __init rcu_init_one(struct rcu_state *rsp)
                                               j / rsp->levelspread[i - 1];
                         }
                         rnp->level = i;
+                        INIT_LIST_HEAD(&rnp->blocked_tasks[0]);
+                        INIT_LIST_HEAD(&rnp->blocked_tasks[1]);
                 }
         }
 }
 
 /*
- * Helper macro for __rcu_init().  To be used nowhere else!
- * Assigns leaf node pointers into each CPU's rcu_data structure.
+ * Helper macro for __rcu_init() and __rcu_init_preempt().  To be used
+ * nowhere else!  Assigns leaf node pointers into each CPU's rcu_data
+ * structure.
  */
-#define RCU_DATA_PTR_INIT(rsp, rcu_data) \
+#define RCU_INIT_FLAVOR(rsp, rcu_data) \
 do { \
+        rcu_init_one(rsp); \
         rnp = (rsp)->level[NUM_RCU_LVLS - 1]; \
         j = 0; \
         for_each_possible_cpu(i) { \
@@ -1520,32 +1595,43 @@ do { \
                         j++; \
                 per_cpu(rcu_data, i).mynode = &rnp[j]; \
                 (rsp)->rda[i] = &per_cpu(rcu_data, i); \
+                rcu_boot_init_percpu_data(i, rsp); \
         } \
 } while (0)
 
-static struct notifier_block __cpuinitdata rcu_nb = {
-        .notifier_call  = rcu_cpu_notify,
-};
+#ifdef CONFIG_TREE_PREEMPT_RCU
+
+void __init __rcu_init_preempt(void)
+{
+        int i;                  /* All used by RCU_INIT_FLAVOR(). */
+        int j;
+        struct rcu_node *rnp;
+
+        RCU_INIT_FLAVOR(&rcu_preempt_state, rcu_preempt_data);
+}
+
+#else /* #ifdef CONFIG_TREE_PREEMPT_RCU */
+
+void __init __rcu_init_preempt(void)
+{
+}
+
+#endif /* #else #ifdef CONFIG_TREE_PREEMPT_RCU */
 
 void __init __rcu_init(void)
 {
-        int i;                  /* All used by RCU_DATA_PTR_INIT(). */
+        int i;                  /* All used by RCU_INIT_FLAVOR(). */
         int j;
         struct rcu_node *rnp;
 
-        printk(KERN_INFO "Hierarchical RCU implementation.\n");
+        rcu_bootup_announce();
 #ifdef CONFIG_RCU_CPU_STALL_DETECTOR
         printk(KERN_INFO "RCU-based detection of stalled CPUs is enabled.\n");
 #endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
-        rcu_init_one(&rcu_state);
-        RCU_DATA_PTR_INIT(&rcu_state, rcu_data);
-        rcu_init_one(&rcu_bh_state);
-        RCU_DATA_PTR_INIT(&rcu_bh_state, rcu_bh_data);
-
-        for_each_online_cpu(i)
-                rcu_cpu_notify(&rcu_nb, CPU_UP_PREPARE, (void *)(long)i);
-        /* Register notifier for non-boot CPUs */
-        register_cpu_notifier(&rcu_nb);
+        RCU_INIT_FLAVOR(&rcu_sched_state, rcu_sched_data);
+        RCU_INIT_FLAVOR(&rcu_bh_state, rcu_bh_data);
+        __rcu_init_preempt();
+        open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
 }
 
 module_param(blimit, int, 0);
diff --git a/kernel/rcutree.h b/kernel/rcutree.h
index 5e872bbf07f5..bf8a6f9f134d 100644
--- a/kernel/rcutree.h
+++ b/kernel/rcutree.h
@@ -1,10 +1,259 @@
+/*
+ * Read-Copy Update mechanism for mutual exclusion (tree-based version)
+ * Internal non-public definitions.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Copyright IBM Corporation, 2008
+ *
+ * Author: Ingo Molnar <mingo@elte.hu>
+ *         Paul E. McKenney <paulmck@linux.vnet.ibm.com>
+ */
+
+#include <linux/cache.h>
+#include <linux/spinlock.h>
+#include <linux/threads.h>
+#include <linux/cpumask.h>
+#include <linux/seqlock.h>
+
+/*
+ * Define shape of hierarchy based on NR_CPUS and CONFIG_RCU_FANOUT.
+ * In theory, it should be possible to add more levels straightforwardly.
+ * In practice, this has not been tested, so there is probably some
+ * bug somewhere.
+ */
+#define MAX_RCU_LVLS 3
+#define RCU_FANOUT            (CONFIG_RCU_FANOUT)
+#define RCU_FANOUT_SQ         (RCU_FANOUT * RCU_FANOUT)
+#define RCU_FANOUT_CUBE       (RCU_FANOUT_SQ * RCU_FANOUT)
+
+#if NR_CPUS <= RCU_FANOUT
+#  define NUM_RCU_LVLS        1
+#  define NUM_RCU_LVL_0       1
+#  define NUM_RCU_LVL_1       (NR_CPUS)
+#  define NUM_RCU_LVL_2       0
+#  define NUM_RCU_LVL_3       0
+#elif NR_CPUS <= RCU_FANOUT_SQ
+#  define NUM_RCU_LVLS        2
+#  define NUM_RCU_LVL_0       1
+#  define NUM_RCU_LVL_1       (((NR_CPUS) + RCU_FANOUT - 1) / RCU_FANOUT)
+#  define NUM_RCU_LVL_2       (NR_CPUS)
+#  define NUM_RCU_LVL_3       0
+#elif NR_CPUS <= RCU_FANOUT_CUBE
+#  define NUM_RCU_LVLS        3
+#  define NUM_RCU_LVL_0       1
+#  define NUM_RCU_LVL_1       (((NR_CPUS) + RCU_FANOUT_SQ - 1) / RCU_FANOUT_SQ)
+#  define NUM_RCU_LVL_2       (((NR_CPUS) + (RCU_FANOUT) - 1) / (RCU_FANOUT))
+#  define NUM_RCU_LVL_3       NR_CPUS
+#else
+# error "CONFIG_RCU_FANOUT insufficient for NR_CPUS"
+#endif /* #if (NR_CPUS) <= RCU_FANOUT */
+
+#define RCU_SUM (NUM_RCU_LVL_0 + NUM_RCU_LVL_1 + NUM_RCU_LVL_2 + NUM_RCU_LVL_3)
+#define NUM_RCU_NODES (RCU_SUM - NR_CPUS)
+
+/*
+ * Dynticks per-CPU state.
+ */
+struct rcu_dynticks {
+        int dynticks_nesting;   /* Track nesting level, sort of. */
+        int dynticks;           /* Even value for dynticks-idle, else odd. */
+        int dynticks_nmi;       /* Even value for either dynticks-idle or */
+                                /*  not in nmi handler, else odd.  So this */
+                                /*  remains even for nmi from irq handler. */
+};
+
+/*
+ * Definition for node within the RCU grace-period-detection hierarchy.
+ */
+struct rcu_node {
+        spinlock_t lock;
+        long    gpnum;          /* Current grace period for this node. */
+                                /*  This will either be equal to or one */
+                                /*  behind the root rcu_node's gpnum. */
+        unsigned long qsmask;   /* CPUs or groups that need to switch in */
+                                /*  order for current grace period to proceed.*/
+        unsigned long qsmaskinit;
+                                /* Per-GP initialization for qsmask. */
+        unsigned long grpmask;  /* Mask to apply to parent qsmask. */
+        int     grplo;          /* lowest-numbered CPU or group here. */
+        int     grphi;          /* highest-numbered CPU or group here. */
+        u8      grpnum;         /* CPU/group number for next level up. */
+        u8      level;          /* root is at level 0. */
+        struct rcu_node *parent;
+        struct list_head blocked_tasks[2];
+                                /* Tasks blocked in RCU read-side critsect. */
+} ____cacheline_internodealigned_in_smp;
+
+/* Index values for nxttail array in struct rcu_data. */
+#define RCU_DONE_TAIL           0       /* Also RCU_WAIT head. */
+#define RCU_WAIT_TAIL           1       /* Also RCU_NEXT_READY head. */
+#define RCU_NEXT_READY_TAIL     2       /* Also RCU_NEXT head. */
+#define RCU_NEXT_TAIL           3
+#define RCU_NEXT_SIZE           4
+
+/* Per-CPU data for read-copy update. */
+struct rcu_data {
+        /* 1) quiescent-state and grace-period handling : */
+        long            completed;      /* Track rsp->completed gp number */
+                                        /*  in order to detect GP end. */
+        long            gpnum;          /* Highest gp number that this CPU */
+                                        /*  is aware of having started. */
+        long            passed_quiesc_completed;
+                                        /* Value of completed at time of qs. */
+        bool            passed_quiesc;  /* User-mode/idle loop etc. */
+        bool            qs_pending;     /* Core waits for quiesc state. */
+        bool            beenonline;     /* CPU online at least once. */
+        bool            preemptable;    /* Preemptable RCU? */
+        struct rcu_node *mynode;        /* This CPU's leaf of hierarchy */
+        unsigned long grpmask;          /* Mask to apply to leaf qsmask. */
+
+        /* 2) batch handling */
+        /*
+         * If nxtlist is not NULL, it is partitioned as follows.
+         * Any of the partitions might be empty, in which case the
+         * pointer to that partition will be equal to the pointer for
+         * the following partition.  When the list is empty, all of
+         * the nxttail elements point to nxtlist, which is NULL.
+         *
+         * [*nxttail[RCU_NEXT_READY_TAIL], NULL = *nxttail[RCU_NEXT_TAIL]):
+         *      Entries that might have arrived after current GP ended
+         * [*nxttail[RCU_WAIT_TAIL], *nxttail[RCU_NEXT_READY_TAIL]):
+         *      Entries known to have arrived before current GP ended
+         * [*nxttail[RCU_DONE_TAIL], *nxttail[RCU_WAIT_TAIL]):
+         *      Entries that batch # <= ->completed - 1: waiting for current GP
+         * [nxtlist, *nxttail[RCU_DONE_TAIL]):
+         *      Entries that batch # <= ->completed
+         *      The grace period for these entries has completed, and
+         *      the other grace-period-completed entries may be moved
+         *      here temporarily in rcu_process_callbacks().
+         */
+        struct rcu_head *nxtlist;
+        struct rcu_head **nxttail[RCU_NEXT_SIZE];
+        long            qlen;           /* # of queued callbacks */
+        long            blimit;         /* Upper limit on a processed batch */
+
+#ifdef CONFIG_NO_HZ
+        /* 3) dynticks interface. */
+        struct rcu_dynticks *dynticks;  /* Shared per-CPU dynticks state. */
+        int dynticks_snap;              /* Per-GP tracking for dynticks. */
+        int dynticks_nmi_snap;          /* Per-GP tracking for dynticks_nmi. */
+#endif /* #ifdef CONFIG_NO_HZ */
+
+        /* 4) reasons this CPU needed to be kicked by force_quiescent_state */
+#ifdef CONFIG_NO_HZ
+        unsigned long dynticks_fqs;     /* Kicked due to dynticks idle. */
+#endif /* #ifdef CONFIG_NO_HZ */
+        unsigned long offline_fqs;      /* Kicked due to being offline. */
+        unsigned long resched_ipi;      /* Sent a resched IPI. */
+
+        /* 5) __rcu_pending() statistics. */
+        long n_rcu_pending;             /* rcu_pending() calls since boot. */
+        long n_rp_qs_pending;
+        long n_rp_cb_ready;
+        long n_rp_cpu_needs_gp;
+        long n_rp_gp_completed;
+        long n_rp_gp_started;
+        long n_rp_need_fqs;
+        long n_rp_need_nothing;
+
+        int cpu;
+};
+
+/* Values for signaled field in struct rcu_state. */
+#define RCU_GP_INIT             0       /* Grace period being initialized. */
+#define RCU_SAVE_DYNTICK        1       /* Need to scan dyntick state. */
+#define RCU_FORCE_QS            2       /* Need to force quiescent state. */
+#ifdef CONFIG_NO_HZ
+#define RCU_SIGNAL_INIT         RCU_SAVE_DYNTICK
+#else /* #ifdef CONFIG_NO_HZ */
+#define RCU_SIGNAL_INIT         RCU_FORCE_QS
+#endif /* #else #ifdef CONFIG_NO_HZ */
+
+#define RCU_JIFFIES_TILL_FORCE_QS        3      /* for rsp->jiffies_force_qs */
+#ifdef CONFIG_RCU_CPU_STALL_DETECTOR
+#define RCU_SECONDS_TILL_STALL_CHECK   (10 * HZ)  /* for rsp->jiffies_stall */
+#define RCU_SECONDS_TILL_STALL_RECHECK (30 * HZ)  /* for rsp->jiffies_stall */
+#define RCU_STALL_RAT_DELAY             2         /* Allow other CPUs time */
+                                                  /*  to take at least one */
+                                                  /*  scheduling clock irq */
+                                                  /*  before ratting on them. */
+
+#endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
+
+/*
+ * RCU global state, including node hierarchy.  This hierarchy is
+ * represented in "heap" form in a dense array.  The root (first level)
+ * of the hierarchy is in ->node[0] (referenced by ->level[0]), the second
+ * level in ->node[1] through ->node[m] (->node[1] referenced by ->level[1]),
+ * and the third level in ->node[m+1] and following (->node[m+1] referenced
+ * by ->level[2]).  The number of levels is determined by the number of
+ * CPUs and by CONFIG_RCU_FANOUT.  Small systems will have a "hierarchy"
+ * consisting of a single rcu_node.
+ */
+struct rcu_state {
+        struct rcu_node node[NUM_RCU_NODES];    /* Hierarchy. */
+        struct rcu_node *level[NUM_RCU_LVLS];   /* Hierarchy levels. */
+        u32 levelcnt[MAX_RCU_LVLS + 1];         /* # nodes in each level. */
+        u8 levelspread[NUM_RCU_LVLS];           /* kids/node in each level. */
+        struct rcu_data *rda[NR_CPUS];          /* array of rdp pointers. */
+
+        /* The following fields are guarded by the root rcu_node's lock. */
+
+        u8      signaled ____cacheline_internodealigned_in_smp;
+                                                /* Force QS state. */
+        long    gpnum;                          /* Current gp number. */
+        long    completed;                      /* # of last completed gp. */
+        spinlock_t onofflock;                   /* exclude on/offline and */
+                                                /*  starting new GP. */
+        spinlock_t fqslock;                     /* Only one task forcing */
+                                                /*  quiescent states. */
+        unsigned long jiffies_force_qs;         /* Time at which to invoke */
+                                                /*  force_quiescent_state(). */
+        unsigned long n_force_qs;               /* Number of calls to */
+                                                /*  force_quiescent_state(). */
+        unsigned long n_force_qs_lh;            /* ~Number of calls leaving */
+                                                /*  due to lock unavailable. */
+        unsigned long n_force_qs_ngp;           /* Number of calls leaving */
+                                                /*  due to no GP active. */
+#ifdef CONFIG_RCU_CPU_STALL_DETECTOR
+        unsigned long gp_start;                 /* Time at which GP started, */
+                                                /*  but in jiffies. */
+        unsigned long jiffies_stall;            /* Time at which to check */
+                                                /*  for CPU stalls. */
+#endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
+#ifdef CONFIG_NO_HZ
+        long dynticks_completed;                /* Value of completed @ snap. */
+#endif /* #ifdef CONFIG_NO_HZ */
+};
+
+#ifdef RCU_TREE_NONCORE
 
 /*
  * RCU implementation internal declarations:
  */
-extern struct rcu_state rcu_state;
-DECLARE_PER_CPU(struct rcu_data, rcu_data);
+extern struct rcu_state rcu_sched_state;
+DECLARE_PER_CPU(struct rcu_data, rcu_sched_data);
 
 extern struct rcu_state rcu_bh_state;
 DECLARE_PER_CPU(struct rcu_data, rcu_bh_data);
 
+#ifdef CONFIG_TREE_PREEMPT_RCU
+extern struct rcu_state rcu_preempt_state;
+DECLARE_PER_CPU(struct rcu_data, rcu_preempt_data);
+#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
+
+#endif /* #ifdef RCU_TREE_NONCORE */
+
diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h
new file mode 100644
index 000000000000..47789369ea59
--- /dev/null
+++ b/kernel/rcutree_plugin.h
@@ -0,0 +1,532 @@
+/*
+ * Read-Copy Update mechanism for mutual exclusion (tree-based version)
+ * Internal non-public definitions that provide either classic
+ * or preemptable semantics.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Copyright Red Hat, 2009
+ * Copyright IBM Corporation, 2009
+ *
+ * Author: Ingo Molnar <mingo@elte.hu>
+ *         Paul E. McKenney <paulmck@linux.vnet.ibm.com>
+ */
+
+
+#ifdef CONFIG_TREE_PREEMPT_RCU
+
+struct rcu_state rcu_preempt_state = RCU_STATE_INITIALIZER(rcu_preempt_state);
+DEFINE_PER_CPU(struct rcu_data, rcu_preempt_data);
+
+/*
+ * Tell them what RCU they are running.
+ */
+static inline void rcu_bootup_announce(void)
+{
+        printk(KERN_INFO
+               "Experimental preemptable hierarchical RCU implementation.\n");
+}
+
+/*
+ * Return the number of RCU-preempt batches processed thus far
+ * for debug and statistics.
+ */
+long rcu_batches_completed_preempt(void)
+{
+        return rcu_preempt_state.completed;
+}
+EXPORT_SYMBOL_GPL(rcu_batches_completed_preempt);
+
+/*
+ * Return the number of RCU batches processed thus far for debug & stats.
+ */
+long rcu_batches_completed(void)
+{
+        return rcu_batches_completed_preempt();
+}
+EXPORT_SYMBOL_GPL(rcu_batches_completed);
+
+/*
+ * Record a preemptable-RCU quiescent state for the specified CPU.  Note
+ * that this just means that the task currently running on the CPU is
+ * not in a quiescent state.  There might be any number of tasks blocked
+ * while in an RCU read-side critical section.
+ */
+static void rcu_preempt_qs_record(int cpu)
+{
+        struct rcu_data *rdp = &per_cpu(rcu_preempt_data, cpu);
+        rdp->passed_quiesc = 1;
+        rdp->passed_quiesc_completed = rdp->completed;
+}
+
+/*
+ * We have entered the scheduler or are between softirqs in ksoftirqd.
+ * If we are in an RCU read-side critical section, we need to reflect
+ * that in the state of the rcu_node structure corresponding to this CPU.
+ * Caller must disable hardirqs.
+ */
+static void rcu_preempt_qs(int cpu)
+{
+        struct task_struct *t = current;
+        int phase;
+        struct rcu_data *rdp;
+        struct rcu_node *rnp;
+
+        if (t->rcu_read_lock_nesting &&
+            (t->rcu_read_unlock_special & RCU_READ_UNLOCK_BLOCKED) == 0) {
+
+                /* Possibly blocking in an RCU read-side critical section. */
+                rdp = rcu_preempt_state.rda[cpu];
+                rnp = rdp->mynode;
+                spin_lock(&rnp->lock);
+                t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BLOCKED;
+                t->rcu_blocked_node = rnp;
+
+                /*
+                 * If this CPU has already checked in, then this task
+                 * will hold up the next grace period rather than the
+                 * current grace period.  Queue the task accordingly.
+                 * If the task is queued for the current grace period
+                 * (i.e., this CPU has not yet passed through a quiescent
+                 * state for the current grace period), then as long
+                 * as that task remains queued, the current grace period
+                 * cannot end.
+                 */
+                phase = !(rnp->qsmask & rdp->grpmask) ^ (rnp->gpnum & 0x1);
+                list_add(&t->rcu_node_entry, &rnp->blocked_tasks[phase]);
+                smp_mb();  /* Ensure later ctxt swtch seen after above. */
+                spin_unlock(&rnp->lock);
+        }
+
+        /*
+         * Either we were not in an RCU read-side critical section to
+         * begin with, or we have now recorded that critical section
+         * globally.  Either way, we can now note a quiescent state
+         * for this CPU.  Again, if we were in an RCU read-side critical
+         * section, and if that critical section was blocking the current
+         * grace period, then the fact that the task has been enqueued
+         * means that we continue to block the current grace period.
+         */
+        rcu_preempt_qs_record(cpu);
+        t->rcu_read_unlock_special &= ~(RCU_READ_UNLOCK_NEED_QS |
+                                        RCU_READ_UNLOCK_GOT_QS);
+}
+
+/*
+ * Tree-preemptable RCU implementation for rcu_read_lock().
+ * Just increment ->rcu_read_lock_nesting, shared state will be updated
+ * if we block.
+ */
+void __rcu_read_lock(void)
+{
+        ACCESS_ONCE(current->rcu_read_lock_nesting)++;
+        barrier();  /* needed if we ever invoke rcu_read_lock in rcutree.c */
+}
+EXPORT_SYMBOL_GPL(__rcu_read_lock);
+
+static void rcu_read_unlock_special(struct task_struct *t)
+{
+        int empty;
+        unsigned long flags;
+        unsigned long mask;
+        struct rcu_node *rnp;
+        int special;
+
+        /* NMI handlers cannot block and cannot safely manipulate state. */
+        if (in_nmi())
+                return;
+
+        local_irq_save(flags);
+
+        /*
+         * If RCU core is waiting for this CPU to exit critical section,
+         * let it know that we have done so.
+         */
+        special = t->rcu_read_unlock_special;
+        if (special & RCU_READ_UNLOCK_NEED_QS) {
+                t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_NEED_QS;
+                t->rcu_read_unlock_special |= RCU_READ_UNLOCK_GOT_QS;
+        }
+
+        /* Hardware IRQ handlers cannot block. */
+        if (in_irq()) {
+                local_irq_restore(flags);
+                return;
+        }
+
+        /* Clean up if blocked during RCU read-side critical section. */
+        if (special & RCU_READ_UNLOCK_BLOCKED) {
+                t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_BLOCKED;
+
+                /*
+                 * Remove this task from the list it blocked on.  The
+                 * task can migrate while we acquire the lock, but at
+                 * most one time.  So at most two passes through loop.
+                 */
+                for (;;) {
+                        rnp = t->rcu_blocked_node;
+                        spin_lock(&rnp->lock);
+                        if (rnp == t->rcu_blocked_node)
+                                break;
+                        spin_unlock(&rnp->lock);
+                }
+                empty = list_empty(&rnp->blocked_tasks[rnp->gpnum & 0x1]);
+                list_del_init(&t->rcu_node_entry);
+                t->rcu_blocked_node = NULL;
+
+                /*
+                 * If this was the last task on the current list, and if
+                 * we aren't waiting on any CPUs, report the quiescent state.
+                 * Note that both cpu_quiet_msk_finish() and cpu_quiet_msk()
+                 * drop rnp->lock and restore irq.
+                 */
+                if (!empty && rnp->qsmask == 0 &&
+                    list_empty(&rnp->blocked_tasks[rnp->gpnum & 0x1])) {
+                        t->rcu_read_unlock_special &=
+                                ~(RCU_READ_UNLOCK_NEED_QS |
+                                  RCU_READ_UNLOCK_GOT_QS);
+                        if (rnp->parent == NULL) {
+                                /* Only one rcu_node in the tree. */
+                                cpu_quiet_msk_finish(&rcu_preempt_state, flags);
+                                return;
+                        }
+                        /* Report up the rest of the hierarchy. */
+                        mask = rnp->grpmask;
+                        spin_unlock_irqrestore(&rnp->lock, flags);
+                        rnp = rnp->parent;
+                        spin_lock_irqsave(&rnp->lock, flags);
+                        cpu_quiet_msk(mask, &rcu_preempt_state, rnp, flags);
+                        return;
+                }
+                spin_unlock(&rnp->lock);
+        }
+        local_irq_restore(flags);
+}
+
+/*
+ * Tree-preemptable RCU implementation for rcu_read_unlock().
+ * Decrement ->rcu_read_lock_nesting.  If the result is zero (outermost
+ * rcu_read_unlock()) and ->rcu_read_unlock_special is non-zero, then
+ * invoke rcu_read_unlock_special() to clean up after a context switch
+ * in an RCU read-side critical section and other special cases.
+ */
+void __rcu_read_unlock(void)
+{
+        struct task_struct *t = current;
+
+        barrier();  /* needed if we ever invoke rcu_read_unlock in rcutree.c */
+        if (--ACCESS_ONCE(t->rcu_read_lock_nesting) == 0 &&
+            unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
+                rcu_read_unlock_special(t);
+}
+EXPORT_SYMBOL_GPL(__rcu_read_unlock);
+
+#ifdef CONFIG_RCU_CPU_STALL_DETECTOR
+
+/*
+ * Scan the current list of tasks blocked within RCU read-side critical
+ * sections, printing out the tid of each.
+ */
+static void rcu_print_task_stall(struct rcu_node *rnp)
+{
+        unsigned long flags;
+        struct list_head *lp;
+        int phase = rnp->gpnum & 0x1;
+        struct task_struct *t;
+
+        if (!list_empty(&rnp->blocked_tasks[phase])) {
+                spin_lock_irqsave(&rnp->lock, flags);
+                phase = rnp->gpnum & 0x1; /* re-read under lock. */
+                lp = &rnp->blocked_tasks[phase];
+                list_for_each_entry(t, lp, rcu_node_entry)
+                        printk(" P%d", t->pid);
+                spin_unlock_irqrestore(&rnp->lock, flags);
+        }
+}
+
+#endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
+
+/*
+ * Check for preempted RCU readers for the specified rcu_node structure.
+ * If the caller needs a reliable answer, it must hold the rcu_node's
+ * >lock.
+ */
+static int rcu_preempted_readers(struct rcu_node *rnp)
+{
+        return !list_empty(&rnp->blocked_tasks[rnp->gpnum & 0x1]);
+}
+
+#ifdef CONFIG_HOTPLUG_CPU
+
+/*
+ * Handle tasklist migration for case in which all CPUs covered by the
+ * specified rcu_node have gone offline.  Move them up to the root
+ * rcu_node.  The reason for not just moving them to the immediate
+ * parent is to remove the need for rcu_read_unlock_special() to
+ * make more than two attempts to acquire the target rcu_node's lock.
+ *
+ * The caller must hold rnp->lock with irqs disabled.
+ */
+static void rcu_preempt_offline_tasks(struct rcu_state *rsp,
+                                      struct rcu_node *rnp)
+{
+        int i;
+        struct list_head *lp;
+        struct list_head *lp_root;
+        struct rcu_node *rnp_root = rcu_get_root(rsp);
+        struct task_struct *tp;
+
+        if (rnp == rnp_root) {
+                WARN_ONCE(1, "Last CPU thought to be offlined?");
+                return;  /* Shouldn't happen: at least one CPU online. */
+        }
+
+        /*
+         * Move tasks up to root rcu_node.  Rely on the fact that the
+         * root rcu_node can be at most one ahead of the rest of the
+         * rcu_nodes in terms of gp_num value.  This fact allows us to
+         * move the blocked_tasks[] array directly, element by element.
+         */
+        for (i = 0; i < 2; i++) {
+                lp = &rnp->blocked_tasks[i];
+                lp_root = &rnp_root->blocked_tasks[i];
+                while (!list_empty(lp)) {
+                        tp = list_entry(lp->next, typeof(*tp), rcu_node_entry);
+                        spin_lock(&rnp_root->lock); /* irqs already disabled */
+                        list_del(&tp->rcu_node_entry);
+                        tp->rcu_blocked_node = rnp_root;
+                        list_add(&tp->rcu_node_entry, lp_root);
+                        spin_unlock(&rnp_root->lock); /* irqs remain disabled */
+                }
+        }
+}
+
+/*
+ * Do CPU-offline processing for preemptable RCU.
+ */
+static void rcu_preempt_offline_cpu(int cpu)
+{
+        __rcu_offline_cpu(cpu, &rcu_preempt_state);
+}
+
+#endif /* #ifdef CONFIG_HOTPLUG_CPU */
+
+/*
+ * Check for a quiescent state from the current CPU.  When a task blocks,
+ * the task is recorded in the corresponding CPU's rcu_node structure,
+ * which is checked elsewhere.
+ *
+ * Caller must disable hard irqs.
+ */
+static void rcu_preempt_check_callbacks(int cpu)
+{
+        struct task_struct *t = current;
+
+        if (t->rcu_read_lock_nesting == 0) {
+                t->rcu_read_unlock_special &=
+                        ~(RCU_READ_UNLOCK_NEED_QS | RCU_READ_UNLOCK_GOT_QS);
+                rcu_preempt_qs_record(cpu);
+                return;
+        }
+        if (per_cpu(rcu_preempt_data, cpu).qs_pending) {
+                if (t->rcu_read_unlock_special & RCU_READ_UNLOCK_GOT_QS) {
+                        rcu_preempt_qs_record(cpu);
+                        t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_GOT_QS;
+                } else if (!(t->rcu_read_unlock_special &
+                             RCU_READ_UNLOCK_NEED_QS)) {
+                        t->rcu_read_unlock_special |= RCU_READ_UNLOCK_NEED_QS;
+                }
+        }
+}
+
+/*
+ * Process callbacks for preemptable RCU.
+ */
+static void rcu_preempt_process_callbacks(void)
+{
+        __rcu_process_callbacks(&rcu_preempt_state,
+                                &__get_cpu_var(rcu_preempt_data));
+}
+
+/*
+ * Queue a preemptable-RCU callback for invocation after a grace period.
+ */
+void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
+{
+        __call_rcu(head, func, &rcu_preempt_state);
+}
+EXPORT_SYMBOL_GPL(call_rcu);
+
+/*
+ * Check to see if there is any immediate preemptable-RCU-related work
+ * to be done.
+ */
+static int rcu_preempt_pending(int cpu)
+{
+        return __rcu_pending(&rcu_preempt_state,
+                             &per_cpu(rcu_preempt_data, cpu));
+}
+
+/*
+ * Does preemptable RCU need the CPU to stay out of dynticks mode?
+ */
+static int rcu_preempt_needs_cpu(int cpu)
+{
+        return !!per_cpu(rcu_preempt_data, cpu).nxtlist;
+}
+
+/*
+ * Initialize preemptable RCU's per-CPU data.
+ */
+static void __cpuinit rcu_preempt_init_percpu_data(int cpu)
+{
+        rcu_init_percpu_data(cpu, &rcu_preempt_state, 1);
+}
+
+/*
+ * Check for a task exiting while in a preemptable-RCU read-side
+ * critical section, clean up if so.  No need to issue warnings,
+ * as debug_check_no_locks_held() already does this if lockdep
+ * is enabled.
+ */
+void exit_rcu(void)
+{
+        struct task_struct *t = current;
+
+        if (t->rcu_read_lock_nesting == 0)
+                return;
+        t->rcu_read_lock_nesting = 1;
+        rcu_read_unlock();
+}
+
+#else /* #ifdef CONFIG_TREE_PREEMPT_RCU */
+
+/*
+ * Tell them what RCU they are running.
+ */
+static inline void rcu_bootup_announce(void)
+{
+        printk(KERN_INFO "Hierarchical RCU implementation.\n");
+}
+
+/*
+ * Return the number of RCU batches processed thus far for debug & stats.
+ */
+long rcu_batches_completed(void)
+{
+        return rcu_batches_completed_sched();
+}
+EXPORT_SYMBOL_GPL(rcu_batches_completed);
+
+/*
+ * Because preemptable RCU does not exist, we never have to check for
+ * CPUs being in quiescent states.
+ */
+static void rcu_preempt_qs(int cpu)
+{
+}
+
+#ifdef CONFIG_RCU_CPU_STALL_DETECTOR
+
+/*
+ * Because preemptable RCU does not exist, we never have to check for
+ * tasks blocked within RCU read-side critical sections.
+ */
+static void rcu_print_task_stall(struct rcu_node *rnp)
+{
+}
+
+#endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
+
+/*
+ * Because preemptable RCU does not exist, there are never any preempted
+ * RCU readers.
+ */
+static int rcu_preempted_readers(struct rcu_node *rnp)
+{
+        return 0;
+}
+
+#ifdef CONFIG_HOTPLUG_CPU
+
+/*
+ * Because preemptable RCU does not exist, it never needs to migrate
+ * tasks that were blocked within RCU read-side critical sections.
+ */
+static void rcu_preempt_offline_tasks(struct rcu_state *rsp,
+                                      struct rcu_node *rnp)
+{
+}
+
+/*
+ * Because preemptable RCU does not exist, it never needs CPU-offline
+ * processing.
+ */
+static void rcu_preempt_offline_cpu(int cpu)
+{
+}
+
+#endif /* #ifdef CONFIG_HOTPLUG_CPU */
+
+/*
+ * Because preemptable RCU does not exist, it never has any callbacks
+ * to check.
+ */
+void rcu_preempt_check_callbacks(int cpu)
+{
+}
+
+/*
+ * Because preemptable RCU does not exist, it never has any callbacks
+ * to process.
+ */
+void rcu_preempt_process_callbacks(void)
+{
+}
+
+/*
+ * In classic RCU, call_rcu() is just call_rcu_sched().
+ */
+void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
+{
+        call_rcu_sched(head, func);
+}
+EXPORT_SYMBOL_GPL(call_rcu);
+
+/*
+ * Because preemptable RCU does not exist, it never has any work to do.
+ */
+static int rcu_preempt_pending(int cpu)
+{
+        return 0;
+}
+
+/*
+ * Because preemptable RCU does not exist, it never needs any CPU.
+ */
+static int rcu_preempt_needs_cpu(int cpu)
+{
+        return 0;
+}
+
+/*
+ * Because preemptable RCU does not exist, there is no per-CPU
+ * data to initialize.
+ */
+static void __cpuinit rcu_preempt_init_percpu_data(int cpu)
+{
+}
+
+#endif /* #else #ifdef CONFIG_TREE_PREEMPT_RCU */
diff --git a/kernel/rcutree_trace.c b/kernel/rcutree_trace.c
index fe1dcdbf1ca3..0ea1bff69727 100644
--- a/kernel/rcutree_trace.c
+++ b/kernel/rcutree_trace.c
@@ -43,6 +43,7 @@
 #include <linux/debugfs.h>
 #include <linux/seq_file.h>
 
+#define RCU_TREE_NONCORE
 #include "rcutree.h"
 
 static void print_one_rcu_data(struct seq_file *m, struct rcu_data *rdp)
@@ -76,8 +77,12 @@ static void print_one_rcu_data(struct seq_file *m, struct rcu_data *rdp)
 
 static int show_rcudata(struct seq_file *m, void *unused)
 {
-        seq_puts(m, "rcu:\n");
-        PRINT_RCU_DATA(rcu_data, print_one_rcu_data, m);
+#ifdef CONFIG_TREE_PREEMPT_RCU
+        seq_puts(m, "rcu_preempt:\n");
+        PRINT_RCU_DATA(rcu_preempt_data, print_one_rcu_data, m);
+#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
+        seq_puts(m, "rcu_sched:\n");
+        PRINT_RCU_DATA(rcu_sched_data, print_one_rcu_data, m);
         seq_puts(m, "rcu_bh:\n");
         PRINT_RCU_DATA(rcu_bh_data, print_one_rcu_data, m);
         return 0;
@@ -102,7 +107,7 @@ static void print_one_rcu_data_csv(struct seq_file *m, struct rcu_data *rdp)
                 return;
         seq_printf(m, "%d,%s,%ld,%ld,%d,%ld,%d",
                    rdp->cpu,
-                   cpu_is_offline(rdp->cpu) ? "\"Y\"" : "\"N\"",
+                   cpu_is_offline(rdp->cpu) ? "\"N\"" : "\"Y\"",
                    rdp->completed, rdp->gpnum,
                    rdp->passed_quiesc, rdp->passed_quiesc_completed,
                    rdp->qs_pending);
@@ -124,8 +129,12 @@ static int show_rcudata_csv(struct seq_file *m, void *unused)
         seq_puts(m, "\"dt\",\"dt nesting\",\"dn\",\"df\",");
 #endif /* #ifdef CONFIG_NO_HZ */
         seq_puts(m, "\"of\",\"ri\",\"ql\",\"b\"\n");
-        seq_puts(m, "\"rcu:\"\n");
-        PRINT_RCU_DATA(rcu_data, print_one_rcu_data_csv, m);
+#ifdef CONFIG_TREE_PREEMPT_RCU
+        seq_puts(m, "\"rcu_preempt:\"\n");
+        PRINT_RCU_DATA(rcu_preempt_data, print_one_rcu_data_csv, m);
+#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
+        seq_puts(m, "\"rcu_sched:\"\n");
+        PRINT_RCU_DATA(rcu_sched_data, print_one_rcu_data_csv, m);
         seq_puts(m, "\"rcu_bh:\"\n");
         PRINT_RCU_DATA(rcu_bh_data, print_one_rcu_data_csv, m);
         return 0;
@@ -171,8 +180,12 @@ static void print_one_rcu_state(struct seq_file *m, struct rcu_state *rsp)
 
 static int show_rcuhier(struct seq_file *m, void *unused)
 {
-        seq_puts(m, "rcu:\n");
-        print_one_rcu_state(m, &rcu_state);
+#ifdef CONFIG_TREE_PREEMPT_RCU
+        seq_puts(m, "rcu_preempt:\n");
+        print_one_rcu_state(m, &rcu_preempt_state);
+#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
+        seq_puts(m, "rcu_sched:\n");
+        print_one_rcu_state(m, &rcu_sched_state);
         seq_puts(m, "rcu_bh:\n");
         print_one_rcu_state(m, &rcu_bh_state);
         return 0;
@@ -193,8 +206,12 @@ static struct file_operations rcuhier_fops = {
 
 static int show_rcugp(struct seq_file *m, void *unused)
 {
-        seq_printf(m, "rcu: completed=%ld  gpnum=%ld\n",
-                   rcu_state.completed, rcu_state.gpnum);
+#ifdef CONFIG_TREE_PREEMPT_RCU
+        seq_printf(m, "rcu_preempt: completed=%ld  gpnum=%ld\n",
+                   rcu_preempt_state.completed, rcu_preempt_state.gpnum);
+#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
+        seq_printf(m, "rcu_sched: completed=%ld  gpnum=%ld\n",
+                   rcu_sched_state.completed, rcu_sched_state.gpnum);
         seq_printf(m, "rcu_bh: completed=%ld  gpnum=%ld\n",
                    rcu_bh_state.completed, rcu_bh_state.gpnum);
         return 0;
@@ -243,8 +260,12 @@ static void print_rcu_pendings(struct seq_file *m, struct rcu_state *rsp)
 
 static int show_rcu_pending(struct seq_file *m, void *unused)
 {
-        seq_puts(m, "rcu:\n");
-        print_rcu_pendings(m, &rcu_state);
+#ifdef CONFIG_TREE_PREEMPT_RCU
+        seq_puts(m, "rcu_preempt:\n");
+        print_rcu_pendings(m, &rcu_preempt_state);
+#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
+        seq_puts(m, "rcu_sched:\n");
+        print_rcu_pendings(m, &rcu_sched_state);
         seq_puts(m, "rcu_bh:\n");
         print_rcu_pendings(m, &rcu_bh_state);
         return 0;
@@ -264,62 +285,47 @@ static struct file_operations rcu_pending_fops = {
 };
 
 static struct dentry *rcudir;
-static struct dentry *datadir;
-static struct dentry *datadir_csv;
-static struct dentry *gpdir;
-static struct dentry *hierdir;
-static struct dentry *rcu_pendingdir;
 
 static int __init rcuclassic_trace_init(void)
 {
+        struct dentry *retval;
+
         rcudir = debugfs_create_dir("rcu", NULL);
         if (!rcudir)
-                goto out;
+                goto free_out;
 
-        datadir = debugfs_create_file("rcudata", 0444, rcudir,
+        retval = debugfs_create_file("rcudata", 0444, rcudir,
                                                 NULL, &rcudata_fops);
-        if (!datadir)
+        if (!retval)
                 goto free_out;
 
-        datadir_csv = debugfs_create_file("rcudata.csv", 0444, rcudir,
+        retval = debugfs_create_file("rcudata.csv", 0444, rcudir,
                                                 NULL, &rcudata_csv_fops);
-        if (!datadir_csv)
+        if (!retval)
                 goto free_out;
 
-        gpdir = debugfs_create_file("rcugp", 0444, rcudir, NULL, &rcugp_fops);
-        if (!gpdir)
+        retval = debugfs_create_file("rcugp", 0444, rcudir, NULL, &rcugp_fops);
+        if (!retval)
                 goto free_out;
 
-        hierdir = debugfs_create_file("rcuhier", 0444, rcudir,
+        retval = debugfs_create_file("rcuhier", 0444, rcudir,
                                                 NULL, &rcuhier_fops);
-        if (!hierdir)
+        if (!retval)
                 goto free_out;
 
-        rcu_pendingdir = debugfs_create_file("rcu_pending", 0444, rcudir,
+        retval = debugfs_create_file("rcu_pending", 0444, rcudir,
                                                 NULL, &rcu_pending_fops);
-        if (!rcu_pendingdir)
+        if (!retval)
                 goto free_out;
         return 0;
 free_out:
-        if (datadir)
-                debugfs_remove(datadir);
-        if (datadir_csv)
-                debugfs_remove(datadir_csv);
-        if (gpdir)
-                debugfs_remove(gpdir);
-        debugfs_remove(rcudir);
-out:
+        debugfs_remove_recursive(rcudir);
         return 1;
 }
 
 static void __exit rcuclassic_trace_cleanup(void)
 {
-        debugfs_remove(datadir);
-        debugfs_remove(datadir_csv);
-        debugfs_remove(gpdir);
-        debugfs_remove(hierdir);
-        debugfs_remove(rcu_pendingdir);
-        debugfs_remove(rcudir);
+        debugfs_remove_recursive(rcudir);
 }
 
 
diff --git a/kernel/sched.c b/kernel/sched.c
index 1b59e265273b..d9db3fb17573 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -64,7 +64,6 @@
 #include <linux/tsacct_kern.h>
 #include <linux/kprobes.h>
 #include <linux/delayacct.h>
-#include <linux/reciprocal_div.h>
 #include <linux/unistd.h>
 #include <linux/pagemap.h>
 #include <linux/hrtimer.h>
@@ -120,30 +119,8 @@
  */
 #define RUNTIME_INF     ((u64)~0ULL)
 
-#ifdef CONFIG_SMP
-
 static void double_rq_lock(struct rq *rq1, struct rq *rq2);
 
-/*
- * Divide a load by a sched group cpu_power : (load / sg->__cpu_power)
- * Since cpu_power is a 'constant', we can use a reciprocal divide.
- */
-static inline u32 sg_div_cpu_power(const struct sched_group *sg, u32 load)
-{
-        return reciprocal_divide(load, sg->reciprocal_cpu_power);
-}
-
-/*
- * Each time a sched group cpu_power is changed,
- * we must compute its reciprocal value
- */
-static inline void sg_inc_cpu_power(struct sched_group *sg, u32 val)
-{
-        sg->__cpu_power += val;
-        sg->reciprocal_cpu_power = reciprocal_value(sg->__cpu_power);
-}
-#endif
-
 static inline int rt_policy(int policy)
 {
         if (unlikely(policy == SCHED_FIFO || policy == SCHED_RR))
@@ -309,8 +286,8 @@ void set_tg_uid(struct user_struct *user)
 
 /*
  * Root task group.
- *      Every UID task group (including init_task_group aka UID-0) will
- *      be a child to this group.
+ *      Every UID task group (including init_task_group aka UID-0) will
+ *      be a child to this group.
  */
 struct task_group root_task_group;
 
@@ -318,12 +295,12 @@ struct task_group root_task_group;
 /* Default task group's sched entity on each cpu */
 static DEFINE_PER_CPU(struct sched_entity, init_sched_entity);
 /* Default task group's cfs_rq on each cpu */
-static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp;
+static DEFINE_PER_CPU_SHARED_ALIGNED(struct cfs_rq, init_tg_cfs_rq);
 #endif /* CONFIG_FAIR_GROUP_SCHED */
 
 #ifdef CONFIG_RT_GROUP_SCHED
 static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity);
-static DEFINE_PER_CPU(struct rt_rq, init_rt_rq) ____cacheline_aligned_in_smp;
+static DEFINE_PER_CPU_SHARED_ALIGNED(struct rt_rq, init_rt_rq);
 #endif /* CONFIG_RT_GROUP_SCHED */
 #else /* !CONFIG_USER_SCHED */
 #define root_task_group init_task_group
@@ -616,6 +593,7 @@ struct rq {
 
         unsigned char idle_at_tick;
         /* For active balancing */
+        int post_schedule;
         int active_balance;
         int push_cpu;
         /* cpu of this runqueue: */
@@ -626,6 +604,9 @@ struct rq {
 
         struct task_struct *migration_thread;
         struct list_head migration_queue;
+
+        u64 rt_avg;
+        u64 age_stamp;
 #endif
 
         /* calc_load related fields */
@@ -693,6 +674,7 @@ static inline int cpu_of(struct rq *rq)
 #define this_rq()               (&__get_cpu_var(runqueues))
 #define task_rq(p)              cpu_rq(task_cpu(p))
 #define cpu_curr(cpu)           (cpu_rq(cpu)->curr)
+#define raw_rq()                (&__raw_get_cpu_var(runqueues))
 
 inline void update_rq_clock(struct rq *rq)
 {
@@ -861,6 +843,14 @@ unsigned int sysctl_sched_shares_ratelimit = 250000;
 unsigned int sysctl_sched_shares_thresh = 4;
 
 /*
+ * period over which we average the RT time consumption, measured
+ * in ms.
+ *
+ * default: 1s
+ */
+const_debug unsigned int sysctl_sched_time_avg = MSEC_PER_SEC;
+
+/*
  * period over which we measure -rt task cpu usage in us.
  * default: 1s
  */
@@ -1278,12 +1268,37 @@ void wake_up_idle_cpu(int cpu)
 }
 #endif /* CONFIG_NO_HZ */
 
+static u64 sched_avg_period(void)
+{
+        return (u64)sysctl_sched_time_avg * NSEC_PER_MSEC / 2;
+}
+
+static void sched_avg_update(struct rq *rq)
+{
+        s64 period = sched_avg_period();
+
+        while ((s64)(rq->clock - rq->age_stamp) > period) {
+                rq->age_stamp += period;
+                rq->rt_avg /= 2;
+        }
+}
+
+static void sched_rt_avg_update(struct rq *rq, u64 rt_delta)
+{
+        rq->rt_avg += rt_delta;
+        sched_avg_update(rq);
+}
+
 #else /* !CONFIG_SMP */
 static void resched_task(struct task_struct *p)
 {
         assert_spin_locked(&task_rq(p)->lock);
         set_tsk_need_resched(p);
 }
+
+static void sched_rt_avg_update(struct rq *rq, u64 rt_delta)
+{
+}
 #endif /* CONFIG_SMP */
 
 #if BITS_PER_LONG == 32
@@ -1513,28 +1528,35 @@ static unsigned long cpu_avg_load_per_task(int cpu)
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
 
+struct update_shares_data {
+        unsigned long rq_weight[NR_CPUS];
+};
+
+static DEFINE_PER_CPU(struct update_shares_data, update_shares_data);
+
 static void __set_se_shares(struct sched_entity *se, unsigned long shares);
 
 /*
  * Calculate and set the cpu's group shares.
  */
-static void
-update_group_shares_cpu(struct task_group *tg, int cpu,
-                        unsigned long sd_shares, unsigned long sd_rq_weight)
+static void update_group_shares_cpu(struct task_group *tg, int cpu,
+                                    unsigned long sd_shares,
+                                    unsigned long sd_rq_weight,
+                                    struct update_shares_data *usd)
 {
-        unsigned long shares;
-        unsigned long rq_weight;
-
-        if (!tg->se[cpu])
-                return;
+        unsigned long shares, rq_weight;
+        int boost = 0;
 
-        rq_weight = tg->cfs_rq[cpu]->rq_weight;
+        rq_weight = usd->rq_weight[cpu];
+        if (!rq_weight) {
+                boost = 1;
+                rq_weight = NICE_0_LOAD;
+        }
 
         /*
-         *           \Sum shares * rq_weight
-         * shares =  -----------------------
-         *               \Sum rq_weight
-         *
+         *             \Sum_j shares_j * rq_weight_i
+         * shares_i =  -----------------------------
+         *                  \Sum_j rq_weight_j
          */
         shares = (sd_shares * rq_weight) / sd_rq_weight;
         shares = clamp_t(unsigned long, shares, MIN_SHARES, MAX_SHARES);
@@ -1545,8 +1567,8 @@ update_group_shares_cpu(struct task_group *tg, int cpu,
                 unsigned long flags;
 
                 spin_lock_irqsave(&rq->lock, flags);
-                tg->cfs_rq[cpu]->shares = shares;
-
+                tg->cfs_rq[cpu]->rq_weight = boost ? 0 : rq_weight;
+                tg->cfs_rq[cpu]->shares = boost ? 0 : shares;
                 __set_se_shares(tg->se[cpu], shares);
                 spin_unlock_irqrestore(&rq->lock, flags);
         }
@@ -1559,22 +1581,30 @@ update_group_shares_cpu(struct task_group *tg, int cpu,
  */
 static int tg_shares_up(struct task_group *tg, void *data)
 {
-        unsigned long weight, rq_weight = 0;
-        unsigned long shares = 0;
+        unsigned long weight, rq_weight = 0, shares = 0;
+        struct update_shares_data *usd;
         struct sched_domain *sd = data;
+        unsigned long flags;
         int i;
 
+        if (!tg->se[0])
+                return 0;
+
+        local_irq_save(flags);
+        usd = &__get_cpu_var(update_shares_data);
+
         for_each_cpu(i, sched_domain_span(sd)) {
+                weight = tg->cfs_rq[i]->load.weight;
+                usd->rq_weight[i] = weight;
+
                 /*
                  * If there are currently no tasks on the cpu pretend there
                  * is one of average load so that when a new task gets to
                  * run here it will not get delayed by group starvation.
                  */
-                weight = tg->cfs_rq[i]->load.weight;
                 if (!weight)
                         weight = NICE_0_LOAD;
 
-                tg->cfs_rq[i]->rq_weight = weight;
                 rq_weight += weight;
                 shares += tg->cfs_rq[i]->shares;
         }
@@ -1586,7 +1616,9 @@ static int tg_shares_up(struct task_group *tg, void *data)
                 shares = tg->shares;
 
         for_each_cpu(i, sched_domain_span(sd))
-                update_group_shares_cpu(tg, i, shares, rq_weight);
+                update_group_shares_cpu(tg, i, shares, rq_weight, usd);
+
+        local_irq_restore(flags);
 
         return 0;
 }
@@ -1616,8 +1648,14 @@ static int tg_load_down(struct task_group *tg, void *data)
 
 static void update_shares(struct sched_domain *sd)
 {
-        u64 now = cpu_clock(raw_smp_processor_id());
-        s64 elapsed = now - sd->last_update;
+        s64 elapsed;
+        u64 now;
+
+        if (root_task_group_empty())
+                return;
+
+        now = cpu_clock(raw_smp_processor_id());
+        elapsed = now - sd->last_update;
 
         if (elapsed >= (s64)(u64)sysctl_sched_shares_ratelimit) {
                 sd->last_update = now;
@@ -1627,6 +1665,9 @@ static void update_shares(struct sched_domain *sd)
 
 static void update_shares_locked(struct rq *rq, struct sched_domain *sd)
 {
+        if (root_task_group_empty())
+                return;
+
         spin_unlock(&rq->lock);
         update_shares(sd);
         spin_lock(&rq->lock);
@@ -1634,6 +1675,9 @@ static void update_shares_locked(struct rq *rq, struct sched_domain *sd)
 
 static void update_h_load(long cpu)
 {
+        if (root_task_group_empty())
+                return;
+
         walk_tg_tree(tg_load_down, tg_nop, (void *)cpu);
 }
 
@@ -2268,8 +2312,7 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
                 }
 
                 /* Adjust by relative CPU power of the group */
-                avg_load = sg_div_cpu_power(group,
-                                avg_load * SCHED_LOAD_SCALE);
+                avg_load = (avg_load * SCHED_LOAD_SCALE) / group->cpu_power;
 
                 if (local_group) {
                         this_load = avg_load;
@@ -2637,9 +2680,32 @@ void sched_fork(struct task_struct *p, int clone_flags)
         set_task_cpu(p, cpu);
 
         /*
-         * Make sure we do not leak PI boosting priority to the child:
+         * Make sure we do not leak PI boosting priority to the child.
          */
         p->prio = current->normal_prio;
+
+        /*
+         * Revert to default priority/policy on fork if requested.
+         */
+        if (unlikely(p->sched_reset_on_fork)) {
+                if (p->policy == SCHED_FIFO || p->policy == SCHED_RR)
+                        p->policy = SCHED_NORMAL;
+
+                if (p->normal_prio < DEFAULT_PRIO)
+                        p->prio = DEFAULT_PRIO;
+
+                if (PRIO_TO_NICE(p->static_prio) < 0) {
+                        p->static_prio = NICE_TO_PRIO(0);
+                        set_load_weight(p);
+                }
+
+                /*
+                 * We don't need the reset flag anymore after the fork. It has
+                 * fulfilled its duty:
+                 */
+                p->sched_reset_on_fork = 0;
+        }
+
         if (!rt_prio(p->prio))
                 p->sched_class = &fair_sched_class;
 
@@ -2796,12 +2862,6 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev)
 {
         struct mm_struct *mm = rq->prev_mm;
         long prev_state;
-#ifdef CONFIG_SMP
-        int post_schedule = 0;
-
-        if (current->sched_class->needs_post_schedule)
-                post_schedule = current->sched_class->needs_post_schedule(rq);
-#endif
 
         rq->prev_mm = NULL;
 
@@ -2820,10 +2880,6 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev)
         finish_arch_switch(prev);
         perf_counter_task_sched_in(current, cpu_of(rq));
         finish_lock_switch(rq, prev);
-#ifdef CONFIG_SMP
-        if (post_schedule)
-                current->sched_class->post_schedule(rq);
-#endif
 
         fire_sched_in_preempt_notifiers(current);
         if (mm)
@@ -2838,6 +2894,42 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev)
         }
 }
 
+#ifdef CONFIG_SMP
+
+/* assumes rq->lock is held */
+static inline void pre_schedule(struct rq *rq, struct task_struct *prev)
+{
+        if (prev->sched_class->pre_schedule)
+                prev->sched_class->pre_schedule(rq, prev);
+}
+
+/* rq->lock is NOT held, but preemption is disabled */
+static inline void post_schedule(struct rq *rq)
+{
+        if (rq->post_schedule) {
+                unsigned long flags;
+
+                spin_lock_irqsave(&rq->lock, flags);
+                if (rq->curr->sched_class->post_schedule)
+                        rq->curr->sched_class->post_schedule(rq);
+                spin_unlock_irqrestore(&rq->lock, flags);
+
+                rq->post_schedule = 0;
+        }
+}
+
+#else
+
+static inline void pre_schedule(struct rq *rq, struct task_struct *p)
+{
+}
+
+static inline void post_schedule(struct rq *rq)
+{
+}
+
+#endif
+
 /**
  * schedule_tail - first thing a freshly forked thread must call.
  * @prev: the thread we just switched away from.
@@ -2848,6 +2940,13 @@ asmlinkage void schedule_tail(struct task_struct *prev)
         struct rq *rq = this_rq();
 
         finish_task_switch(rq, prev);
+
+        /*
+         * FIXME: do we need to worry about rq being invalidated by the
+         * task_switch?
+         */
+        post_schedule(rq);
+
 #ifdef __ARCH_WANT_UNLOCKED_CTXSW
         /* In this case, finish_task_switch does not reenable preemption */
         preempt_enable();
@@ -3379,9 +3478,10 @@ static int move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
 {
         const struct sched_class *class;
 
-        for (class = sched_class_highest; class; class = class->next)
+        for_each_class(class) {
                 if (class->move_one_task(this_rq, this_cpu, busiest, sd, idle))
                         return 1;
+        }
 
         return 0;
 }
@@ -3544,7 +3644,7 @@ static inline void update_sd_power_savings_stats(struct sched_group *group,
          * capacity but still has some space to pick up some load
          * from other group and save more power
          */
-        if (sgs->sum_nr_running > sgs->group_capacity - 1)
+        if (sgs->sum_nr_running + 1 > sgs->group_capacity)
                 return;
 
         if (sgs->sum_nr_running > sds->leader_nr_running ||
@@ -3611,6 +3711,77 @@ static inline int check_power_save_busiest_group(struct sd_lb_stats *sds,
 }
 #endif /* CONFIG_SCHED_MC || CONFIG_SCHED_SMT */
 
+unsigned long __weak arch_scale_smt_power(struct sched_domain *sd, int cpu)
+{
+        unsigned long weight = cpumask_weight(sched_domain_span(sd));
+        unsigned long smt_gain = sd->smt_gain;
+
+        smt_gain /= weight;
+
+        return smt_gain;
+}
+
+unsigned long scale_rt_power(int cpu)
+{
+        struct rq *rq = cpu_rq(cpu);
+        u64 total, available;
+
+        sched_avg_update(rq);
+
+        total = sched_avg_period() + (rq->clock - rq->age_stamp);
+        available = total - rq->rt_avg;
+
+        if (unlikely((s64)total < SCHED_LOAD_SCALE))
+                total = SCHED_LOAD_SCALE;
+
+        total >>= SCHED_LOAD_SHIFT;
+
+        return div_u64(available, total);
+}
+
+static void update_cpu_power(struct sched_domain *sd, int cpu)
+{
+        unsigned long weight = cpumask_weight(sched_domain_span(sd));
+        unsigned long power = SCHED_LOAD_SCALE;
+        struct sched_group *sdg = sd->groups;
+
+        /* here we could scale based on cpufreq */
+
+        if ((sd->flags & SD_SHARE_CPUPOWER) && weight > 1) {
+                power *= arch_scale_smt_power(sd, cpu);
+                power >>= SCHED_LOAD_SHIFT;
+        }
+
+        power *= scale_rt_power(cpu);
+        power >>= SCHED_LOAD_SHIFT;
+
+        if (!power)
+                power = 1;
+
+        sdg->cpu_power = power;
+}
+
+static void update_group_power(struct sched_domain *sd, int cpu)
+{
+        struct sched_domain *child = sd->child;
+        struct sched_group *group, *sdg = sd->groups;
+        unsigned long power;
+
+        if (!child) {
+                update_cpu_power(sd, cpu);
+                return;
+        }
+
+        power = 0;
+
+        group = child->groups;
+        do {
+                power += group->cpu_power;
+                group = group->next;
+        } while (group != child->groups);
+
+        sdg->cpu_power = power;
+}
 
 /**
  * update_sg_lb_stats - Update sched_group's statistics for load balancing.
@@ -3624,7 +3795,8 @@ static inline int check_power_save_busiest_group(struct sd_lb_stats *sds,
  * @balance: Should we balance.
  * @sgs: variable to hold the statistics for this group.
  */
-static inline void update_sg_lb_stats(struct sched_group *group, int this_cpu,
+static inline void update_sg_lb_stats(struct sched_domain *sd,
+                        struct sched_group *group, int this_cpu,
                         enum cpu_idle_type idle, int load_idx, int *sd_idle,
                         int local_group, const struct cpumask *cpus,
                         int *balance, struct sg_lb_stats *sgs)
@@ -3635,8 +3807,11 @@ static inline void update_sg_lb_stats(struct sched_group *group, int this_cpu,
         unsigned long sum_avg_load_per_task;
         unsigned long avg_load_per_task;
 
-        if (local_group)
+        if (local_group) {
                 balance_cpu = group_first_cpu(group);
+                if (balance_cpu == this_cpu)
+                        update_group_power(sd, this_cpu);
+        }
 
         /* Tally up the load of all CPUs in the group */
         sum_avg_load_per_task = avg_load_per_task = 0;
@@ -3685,8 +3860,7 @@ static inline void update_sg_lb_stats(struct sched_group *group, int this_cpu,
         }
 
         /* Adjust by relative CPU power of the group */
-        sgs->avg_load = sg_div_cpu_power(group,
-                        sgs->group_load * SCHED_LOAD_SCALE);
+        sgs->avg_load = (sgs->group_load * SCHED_LOAD_SCALE) / group->cpu_power;
 
 
         /*
@@ -3698,14 +3872,14 @@ static inline void update_sg_lb_stats(struct sched_group *group, int this_cpu,
          *      normalized nr_running number somewhere that negates
          *      the hierarchy?
          */
-        avg_load_per_task = sg_div_cpu_power(group,
-                        sum_avg_load_per_task * SCHED_LOAD_SCALE);
+        avg_load_per_task = (sum_avg_load_per_task * SCHED_LOAD_SCALE) /
+                group->cpu_power;
 
         if ((max_cpu_load - min_cpu_load) > 2*avg_load_per_task)
                 sgs->group_imb = 1;
 
-        sgs->group_capacity = group->__cpu_power / SCHED_LOAD_SCALE;
-
+        sgs->group_capacity =
+                DIV_ROUND_CLOSEST(group->cpu_power, SCHED_LOAD_SCALE);
 }
 
 /**
@@ -3723,9 +3897,13 @@ static inline void update_sd_lb_stats(struct sched_domain *sd, int this_cpu,
                         const struct cpumask *cpus, int *balance,
                         struct sd_lb_stats *sds)
 {
+        struct sched_domain *child = sd->child;
         struct sched_group *group = sd->groups;
         struct sg_lb_stats sgs;
-        int load_idx;
+        int load_idx, prefer_sibling = 0;
+
+        if (child && child->flags & SD_PREFER_SIBLING)
+                prefer_sibling = 1;
 
         init_sd_power_savings_stats(sd, sds, idle);
         load_idx = get_sd_load_idx(sd, idle);
@@ -3736,14 +3914,22 @@ static inline void update_sd_lb_stats(struct sched_domain *sd, int this_cpu,
                 local_group = cpumask_test_cpu(this_cpu,
                                                sched_group_cpus(group));
                 memset(&sgs, 0, sizeof(sgs));
-                update_sg_lb_stats(group, this_cpu, idle, load_idx, sd_idle,
+                update_sg_lb_stats(sd, group, this_cpu, idle, load_idx, sd_idle,
                                 local_group, cpus, balance, &sgs);
 
                 if (local_group && balance && !(*balance))
                         return;
 
                 sds->total_load += sgs.group_load;
-                sds->total_pwr += group->__cpu_power;
+                sds->total_pwr += group->cpu_power;
+
+                /*
+                 * In case the child domain prefers tasks go to siblings
+                 * first, lower the group capacity to one so that we'll try
+                 * and move all the excess tasks away.
+                 */
+                if (prefer_sibling)
+                        sgs.group_capacity = min(sgs.group_capacity, 1UL);
 
                 if (local_group) {
                         sds->this_load = sgs.avg_load;
@@ -3763,7 +3949,6 @@ static inline void update_sd_lb_stats(struct sched_domain *sd, int this_cpu,
                 update_sd_power_savings_stats(group, sds, local_group, &sgs);
                 group = group->next;
         } while (group != sd->groups);
-
 }
 
 /**
@@ -3801,28 +3986,28 @@ static inline void fix_small_imbalance(struct sd_lb_stats *sds,
          * moving them.
          */
 
-        pwr_now += sds->busiest->__cpu_power *
+        pwr_now += sds->busiest->cpu_power *
                         min(sds->busiest_load_per_task, sds->max_load);
-        pwr_now += sds->this->__cpu_power *
+        pwr_now += sds->this->cpu_power *
                         min(sds->this_load_per_task, sds->this_load);
         pwr_now /= SCHED_LOAD_SCALE;
 
         /* Amount of load we'd subtract */
-        tmp = sg_div_cpu_power(sds->busiest,
-                        sds->busiest_load_per_task * SCHED_LOAD_SCALE);
+        tmp = (sds->busiest_load_per_task * SCHED_LOAD_SCALE) /
+                sds->busiest->cpu_power;
         if (sds->max_load > tmp)
-                pwr_move += sds->busiest->__cpu_power *
+                pwr_move += sds->busiest->cpu_power *
                         min(sds->busiest_load_per_task, sds->max_load - tmp);
 
         /* Amount of load we'd add */
-        if (sds->max_load * sds->busiest->__cpu_power <
+        if (sds->max_load * sds->busiest->cpu_power <
                 sds->busiest_load_per_task * SCHED_LOAD_SCALE)
-                tmp = sg_div_cpu_power(sds->this,
-                        sds->max_load * sds->busiest->__cpu_power);
+                tmp = (sds->max_load * sds->busiest->cpu_power) /
+                        sds->this->cpu_power;
         else
-                tmp = sg_div_cpu_power(sds->this,
-                        sds->busiest_load_per_task * SCHED_LOAD_SCALE);
-        pwr_move += sds->this->__cpu_power *
+                tmp = (sds->busiest_load_per_task * SCHED_LOAD_SCALE) /
+                        sds->this->cpu_power;
+        pwr_move += sds->this->cpu_power *
                         min(sds->this_load_per_task, sds->this_load + tmp);
         pwr_move /= SCHED_LOAD_SCALE;
 
@@ -3857,8 +4042,8 @@ static inline void calculate_imbalance(struct sd_lb_stats *sds, int this_cpu,
                         sds->max_load - sds->busiest_load_per_task);
 
         /* How much load to actually move to equalise the imbalance */
-        *imbalance = min(max_pull * sds->busiest->__cpu_power,
-                (sds->avg_load - sds->this_load) * sds->this->__cpu_power)
+        *imbalance = min(max_pull * sds->busiest->cpu_power,
+                (sds->avg_load - sds->this_load) * sds->this->cpu_power)
                         / SCHED_LOAD_SCALE;
 
         /*
@@ -3976,6 +4161,26 @@ ret:
         return NULL;
 }
 
+static struct sched_group *group_of(int cpu)
+{
+        struct sched_domain *sd = rcu_dereference(cpu_rq(cpu)->sd);
+
+        if (!sd)
+                return NULL;
+
+        return sd->groups;
+}
+
+static unsigned long power_of(int cpu)
+{
+        struct sched_group *group = group_of(cpu);
+
+        if (!group)
+                return SCHED_LOAD_SCALE;
+
+        return group->cpu_power;
+}
+
 /*
  * find_busiest_queue - find the busiest runqueue among the cpus in group.
  */
@@ -3988,15 +4193,18 @@ find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
         int i;
 
         for_each_cpu(i, sched_group_cpus(group)) {
+                unsigned long power = power_of(i);
+                unsigned long capacity = DIV_ROUND_CLOSEST(power, SCHED_LOAD_SCALE);
                 unsigned long wl;
 
                 if (!cpumask_test_cpu(i, cpus))
                         continue;
 
                 rq = cpu_rq(i);
-                wl = weighted_cpuload(i);
+                wl = weighted_cpuload(i) * SCHED_LOAD_SCALE;
+                wl /= power;
 
-                if (rq->nr_running == 1 && wl > imbalance)
+                if (capacity && rq->nr_running == 1 && wl > imbalance)
                         continue;
 
                 if (wl > max_load) {
@@ -5325,7 +5533,7 @@ need_resched:
         preempt_disable();
         cpu = smp_processor_id();
         rq = cpu_rq(cpu);
-        rcu_qsctr_inc(cpu);
+        rcu_sched_qs(cpu);
         prev = rq->curr;
         switch_count = &prev->nivcsw;
 
@@ -5349,10 +5557,7 @@ need_resched_nonpreemptible:
                 switch_count = &prev->nvcsw;
         }
 
-#ifdef CONFIG_SMP
-        if (prev->sched_class->pre_schedule)
-                prev->sched_class->pre_schedule(rq, prev);
-#endif
+        pre_schedule(rq, prev);
 
         if (unlikely(!rq->nr_running))
                 idle_balance(cpu, rq);
@@ -5378,6 +5583,8 @@ need_resched_nonpreemptible:
         } else
                 spin_unlock_irq(&rq->lock);
 
+        post_schedule(rq);
+
         if (unlikely(reacquire_kernel_lock(current) < 0))
                 goto need_resched_nonpreemptible;
 
@@ -6123,17 +6330,25 @@ static int __sched_setscheduler(struct task_struct *p, int policy,
         unsigned long flags;
         const struct sched_class *prev_class = p->sched_class;
         struct rq *rq;
+        int reset_on_fork;
 
         /* may grab non-irq protected spin_locks */
         BUG_ON(in_interrupt());
 recheck:
         /* double check policy once rq lock held */
-        if (policy < 0)
+        if (policy < 0) {
+                reset_on_fork = p->sched_reset_on_fork;
                 policy = oldpolicy = p->policy;
-        else if (policy != SCHED_FIFO && policy != SCHED_RR &&
-                        policy != SCHED_NORMAL && policy != SCHED_BATCH &&
-                        policy != SCHED_IDLE)
-                return -EINVAL;
+        } else {
+                reset_on_fork = !!(policy & SCHED_RESET_ON_FORK);
+                policy &= ~SCHED_RESET_ON_FORK;
+
+                if (policy != SCHED_FIFO && policy != SCHED_RR &&
+                                policy != SCHED_NORMAL && policy != SCHED_BATCH &&
+                                policy != SCHED_IDLE)
+                        return -EINVAL;
+        }
+
         /*
          * Valid priorities for SCHED_FIFO and SCHED_RR are
          * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
@@ -6177,6 +6392,10 @@ recheck:
                 /* can't change other user's priorities */
                 if (!check_same_owner(p))
                         return -EPERM;
+
+                /* Normal users shall not reset the sched_reset_on_fork flag */
+                if (p->sched_reset_on_fork && !reset_on_fork)
+                        return -EPERM;
         }
 
         if (user) {
@@ -6220,6 +6439,8 @@ recheck:
         if (running)
                 p->sched_class->put_prev_task(rq, p);
 
+        p->sched_reset_on_fork = reset_on_fork;
+
         oldprio = p->prio;
         __setscheduler(rq, p, policy, param->sched_priority);
 
@@ -6336,14 +6557,15 @@ SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
         if (p) {
                 retval = security_task_getscheduler(p);
                 if (!retval)
-                        retval = p->policy;
+                        retval = p->policy
+                                | (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0);
         }
         read_unlock(&tasklist_lock);
         return retval;
 }
 
 /**
- * sys_sched_getscheduler - get the RT priority of a thread
+ * sys_sched_getparam - get the RT priority of a thread
  * @pid: the pid in question.
  * @param: structure containing the RT priority.
  */
@@ -6571,19 +6793,9 @@ static inline int should_resched(void)
 
 static void __cond_resched(void)
 {
-#ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
-        __might_sleep(__FILE__, __LINE__);
-#endif
-        /*
-         * The BKS might be reacquired before we have dropped
-         * PREEMPT_ACTIVE, which could trigger a second
-         * cond_resched() call.
-         */
-        do {
-                add_preempt_count(PREEMPT_ACTIVE);
-                schedule();
-                sub_preempt_count(PREEMPT_ACTIVE);
-        } while (need_resched());
+        add_preempt_count(PREEMPT_ACTIVE);
+        schedule();
+        sub_preempt_count(PREEMPT_ACTIVE);
 }
 
 int __sched _cond_resched(void)
@@ -6597,18 +6809,20 @@ int __sched _cond_resched(void)
 EXPORT_SYMBOL(_cond_resched);
 
 /*
- * cond_resched_lock() - if a reschedule is pending, drop the given lock,
+ * __cond_resched_lock() - if a reschedule is pending, drop the given lock,
  * call schedule, and on return reacquire the lock.
  *
  * This works OK both with and without CONFIG_PREEMPT. We do strange low-level
  * operations here to prevent schedule() from being called twice (once via
  * spin_unlock(), once by hand).
  */
-int cond_resched_lock(spinlock_t *lock)
+int __cond_resched_lock(spinlock_t *lock)
 {
         int resched = should_resched();
         int ret = 0;
 
+        lockdep_assert_held(lock);
+
         if (spin_needbreak(lock) || resched) {
                 spin_unlock(lock);
                 if (resched)
@@ -6620,9 +6834,9 @@ int cond_resched_lock(spinlock_t *lock)
         }
         return ret;
 }
-EXPORT_SYMBOL(cond_resched_lock);
+EXPORT_SYMBOL(__cond_resched_lock);
 
-int __sched cond_resched_softirq(void)
+int __sched __cond_resched_softirq(void)
 {
         BUG_ON(!in_softirq());
 
@@ -6634,7 +6848,7 @@ int __sched cond_resched_softirq(void)
         }
         return 0;
 }
-EXPORT_SYMBOL(cond_resched_softirq);
+EXPORT_SYMBOL(__cond_resched_softirq);
 
 /**
  * yield - yield the current processor to other threads.
@@ -6658,11 +6872,13 @@ EXPORT_SYMBOL(yield);
  */
 void __sched io_schedule(void)
 {
-        struct rq *rq = &__raw_get_cpu_var(runqueues);
+        struct rq *rq = raw_rq();
 
         delayacct_blkio_start();
         atomic_inc(&rq->nr_iowait);
+        current->in_iowait = 1;
         schedule();
+        current->in_iowait = 0;
         atomic_dec(&rq->nr_iowait);
         delayacct_blkio_end();
 }
@@ -6670,12 +6886,14 @@ EXPORT_SYMBOL(io_schedule);
 
 long __sched io_schedule_timeout(long timeout)
 {
-        struct rq *rq = &__raw_get_cpu_var(runqueues);
+        struct rq *rq = raw_rq();
         long ret;
 
         delayacct_blkio_start();
         atomic_inc(&rq->nr_iowait);
+        current->in_iowait = 1;
         ret = schedule_timeout(timeout);
+        current->in_iowait = 0;
         atomic_dec(&rq->nr_iowait);
         delayacct_blkio_end();
         return ret;
@@ -6992,8 +7210,12 @@ int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
 
         if (migrate_task(p, cpumask_any_and(cpu_online_mask, new_mask), &req)) {
                 /* Need help from migration thread: drop lock and wait. */
+                struct task_struct *mt = rq->migration_thread;
+
+                get_task_struct(mt);
                 task_rq_unlock(rq, &flags);
                 wake_up_process(rq->migration_thread);
+                put_task_struct(mt);
                 wait_for_completion(&req.done);
                 tlb_migrate_finish(p->mm);
                 return 0;
@@ -7051,6 +7273,11 @@ fail:
         return ret;
 }
 
+#define RCU_MIGRATION_IDLE      0
+#define RCU_MIGRATION_NEED_QS   1
+#define RCU_MIGRATION_GOT_QS    2
+#define RCU_MIGRATION_MUST_SYNC 3
+
 /*
  * migration_thread - this is a highprio system thread that performs
  * thread migration by bumping thread off CPU then 'pushing' onto
@@ -7058,6 +7285,7 @@ fail:
  */
 static int migration_thread(void *data)
 {
+        int badcpu;
         int cpu = (long)data;
         struct rq *rq;
 
@@ -7092,8 +7320,17 @@ static int migration_thread(void *data)
                 req = list_entry(head->next, struct migration_req, list);
                 list_del_init(head->next);
 
-                spin_unlock(&rq->lock);
-                __migrate_task(req->task, cpu, req->dest_cpu);
+                if (req->task != NULL) {
+                        spin_unlock(&rq->lock);
+                        __migrate_task(req->task, cpu, req->dest_cpu);
+                } else if (likely(cpu == (badcpu = smp_processor_id()))) {
+                        req->dest_cpu = RCU_MIGRATION_GOT_QS;
+                        spin_unlock(&rq->lock);
+                } else {
+                        req->dest_cpu = RCU_MIGRATION_MUST_SYNC;
+                        spin_unlock(&rq->lock);
+                        WARN_ONCE(1, "migration_thread() on CPU %d, expected %d\n", badcpu, cpu);
+                }
                 local_irq_enable();
 
                 complete(&req->done);
@@ -7625,7 +7862,7 @@ static int __init migration_init(void)
         migration_call(&migration_notifier, CPU_ONLINE, cpu);
         register_cpu_notifier(&migration_notifier);
 
-        return err;
+        return 0;
 }
 early_initcall(migration_init);
 #endif
@@ -7672,7 +7909,7 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
                         break;
                 }
 
-                if (!group->__cpu_power) {
+                if (!group->cpu_power) {
                         printk(KERN_CONT "\n");
                         printk(KERN_ERR "ERROR: domain->cpu_power not "
                                         "set\n");
@@ -7696,9 +7933,9 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
                 cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group));
 
                 printk(KERN_CONT " %s", str);
-                if (group->__cpu_power != SCHED_LOAD_SCALE) {
-                        printk(KERN_CONT " (__cpu_power = %d)",
-                                group->__cpu_power);
+                if (group->cpu_power != SCHED_LOAD_SCALE) {
+                        printk(KERN_CONT " (cpu_power = %d)",
+                                group->cpu_power);
                 }
 
                 group = group->next;
@@ -7841,7 +8078,7 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd)
         rq->rd = rd;
 
         cpumask_set_cpu(rq->cpu, rd->span);
-        if (cpumask_test_cpu(rq->cpu, cpu_online_mask))
+        if (cpumask_test_cpu(rq->cpu, cpu_active_mask))
                 set_rq_online(rq);
 
         spin_unlock_irqrestore(&rq->lock, flags);
@@ -7983,7 +8220,7 @@ init_sched_build_groups(const struct cpumask *span,
                         continue;
 
                 cpumask_clear(sched_group_cpus(sg));
-                sg->__cpu_power = 0;
+                sg->cpu_power = 0;
 
                 for_each_cpu(j, span) {
                         if (group_fn(j, cpu_map, NULL, tmpmask) != group)
@@ -8091,6 +8328,39 @@ struct static_sched_domain {
         DECLARE_BITMAP(span, CONFIG_NR_CPUS);
 };
 
+struct s_data {
+#ifdef CONFIG_NUMA
+        int                     sd_allnodes;
+        cpumask_var_t           domainspan;
+        cpumask_var_t           covered;
+        cpumask_var_t           notcovered;
+#endif
+        cpumask_var_t           nodemask;
+        cpumask_var_t           this_sibling_map;
+        cpumask_var_t           this_core_map;
+        cpumask_var_t           send_covered;
+        cpumask_var_t           tmpmask;
+        struct sched_group      **sched_group_nodes;
+        struct root_domain      *rd;
+};
+
+enum s_alloc {
+        sa_sched_groups = 0,
+        sa_rootdomain,
+        sa_tmpmask,
+        sa_send_covered,
+        sa_this_core_map,
+        sa_this_sibling_map,
+        sa_nodemask,
+        sa_sched_group_nodes,
+#ifdef CONFIG_NUMA
+        sa_notcovered,
+        sa_covered,
+        sa_domainspan,
+#endif
+        sa_none,
+};
+
 /*
  * SMT sched-domains:
  */
@@ -8208,11 +8478,76 @@ static void init_numa_sched_groups_power(struct sched_group *group_head)
                                 continue;
                         }
 
-                        sg_inc_cpu_power(sg, sd->groups->__cpu_power);
+                        sg->cpu_power += sd->groups->cpu_power;
                 }
                 sg = sg->next;
         } while (sg != group_head);
 }
+
+static int build_numa_sched_groups(struct s_data *d,
+                                   const struct cpumask *cpu_map, int num)
+{
+        struct sched_domain *sd;
+        struct sched_group *sg, *prev;
+        int n, j;
+
+        cpumask_clear(d->covered);
+        cpumask_and(d->nodemask, cpumask_of_node(num), cpu_map);
+        if (cpumask_empty(d->nodemask)) {
+                d->sched_group_nodes[num] = NULL;
+                goto out;
+        }
+
+        sched_domain_node_span(num, d->domainspan);
+        cpumask_and(d->domainspan, d->domainspan, cpu_map);
+
+        sg = kmalloc_node(sizeof(struct sched_group) + cpumask_size(),
+                          GFP_KERNEL, num);
+        if (!sg) {
+                printk(KERN_WARNING "Can not alloc domain group for node %d\n",
+                       num);
+                return -ENOMEM;
+        }
+        d->sched_group_nodes[num] = sg;
+
+        for_each_cpu(j, d->nodemask) {
+                sd = &per_cpu(node_domains, j).sd;
+                sd->groups = sg;
+        }
+
+        sg->cpu_power = 0;
+        cpumask_copy(sched_group_cpus(sg), d->nodemask);
+        sg->next = sg;
+        cpumask_or(d->covered, d->covered, d->nodemask);
+
+        prev = sg;
+        for (j = 0; j < nr_node_ids; j++) {
+                n = (num + j) % nr_node_ids;
+                cpumask_complement(d->notcovered, d->covered);
+                cpumask_and(d->tmpmask, d->notcovered, cpu_map);
+                cpumask_and(d->tmpmask, d->tmpmask, d->domainspan);
+                if (cpumask_empty(d->tmpmask))
+                        break;
+                cpumask_and(d->tmpmask, d->tmpmask, cpumask_of_node(n));
+                if (cpumask_empty(d->tmpmask))
+                        continue;
+                sg = kmalloc_node(sizeof(struct sched_group) + cpumask_size(),
+                                  GFP_KERNEL, num);
+                if (!sg) {
+                        printk(KERN_WARNING
+                               "Can not alloc domain group for node %d\n", j);
+                        return -ENOMEM;
+                }
+                sg->cpu_power = 0;
+                cpumask_copy(sched_group_cpus(sg), d->tmpmask);
+                sg->next = prev->next;
+                cpumask_or(d->covered, d->covered, d->tmpmask);
+                prev->next = sg;
+                prev = sg;
+        }
+out:
+        return 0;
+}
 #endif /* CONFIG_NUMA */
 
 #ifdef CONFIG_NUMA
@@ -8266,15 +8601,13 @@ static void free_sched_groups(const struct cpumask *cpu_map,
  * there are asymmetries in the topology. If there are asymmetries, group
  * having more cpu_power will pickup more load compared to the group having
  * less cpu_power.
- *
- * cpu_power will be a multiple of SCHED_LOAD_SCALE. This multiple represents
- * the maximum number of tasks a group can handle in the presence of other idle
- * or lightly loaded groups in the same sched domain.
  */
 static void init_sched_groups_power(int cpu, struct sched_domain *sd)
 {
         struct sched_domain *child;
         struct sched_group *group;
+        long power;
+        int weight;
 
         WARN_ON(!sd || !sd->groups);
 
@@ -8283,28 +8616,32 @@ static void init_sched_groups_power(int cpu, struct sched_domain *sd)
 
         child = sd->child;
 
-        sd->groups->__cpu_power = 0;
+        sd->groups->cpu_power = 0;
 
-        /*
-         * For perf policy, if the groups in child domain share resources
-         * (for example cores sharing some portions of the cache hierarchy
-         * or SMT), then set this domain groups cpu_power such that each group
-         * can handle only one task, when there are other idle groups in the
-         * same sched domain.
-         */
-        if (!child || (!(sd->flags & SD_POWERSAVINGS_BALANCE) &&
-                       (child->flags &
-                        (SD_SHARE_CPUPOWER | SD_SHARE_PKG_RESOURCES)))) {
-                sg_inc_cpu_power(sd->groups, SCHED_LOAD_SCALE);
+        if (!child) {
+                power = SCHED_LOAD_SCALE;
+                weight = cpumask_weight(sched_domain_span(sd));
+                /*
+                 * SMT siblings share the power of a single core.
+                 * Usually multiple threads get a better yield out of
+                 * that one core than a single thread would have,
+                 * reflect that in sd->smt_gain.
+                 */
+                if ((sd->flags & SD_SHARE_CPUPOWER) && weight > 1) {
+                        power *= sd->smt_gain;
+                        power /= weight;
+                        power >>= SCHED_LOAD_SHIFT;
+                }
+                sd->groups->cpu_power += power;
                 return;
         }
 
         /*
-         * add cpu_power of each child group to this groups cpu_power
+         * Add cpu_power of each child group to this groups cpu_power.
          */
         group = child->groups;
         do {
-                sg_inc_cpu_power(sd->groups, group->__cpu_power);
+                sd->groups->cpu_power += group->cpu_power;
                 group = group->next;
         } while (group != child->groups);
 }
@@ -8378,280 +8715,285 @@ static void set_domain_attribute(struct sched_domain *sd,
         }
 }
 
-/*
- * Build sched domains for a given set of cpus and attach the sched domains
- * to the individual cpus
- */
-static int __build_sched_domains(const struct cpumask *cpu_map,
-                                 struct sched_domain_attr *attr)
-{
-        int i, err = -ENOMEM;
-        struct root_domain *rd;
-        cpumask_var_t nodemask, this_sibling_map, this_core_map, send_covered,
-                tmpmask;
+static void __free_domain_allocs(struct s_data *d, enum s_alloc what,
+                                 const struct cpumask *cpu_map)
+{
+        switch (what) {
+        case sa_sched_groups:
+                free_sched_groups(cpu_map, d->tmpmask); /* fall through */
+                d->sched_group_nodes = NULL;
+        case sa_rootdomain:
+                free_rootdomain(d->rd); /* fall through */
+        case sa_tmpmask:
+                free_cpumask_var(d->tmpmask); /* fall through */
+        case sa_send_covered:
+                free_cpumask_var(d->send_covered); /* fall through */
+        case sa_this_core_map:
+                free_cpumask_var(d->this_core_map); /* fall through */
+        case sa_this_sibling_map:
+                free_cpumask_var(d->this_sibling_map); /* fall through */
+        case sa_nodemask:
+                free_cpumask_var(d->nodemask); /* fall through */
+        case sa_sched_group_nodes:
 #ifdef CONFIG_NUMA
-        cpumask_var_t domainspan, covered, notcovered;
-        struct sched_group **sched_group_nodes = NULL;
-        int sd_allnodes = 0;
-
-        if (!alloc_cpumask_var(&domainspan, GFP_KERNEL))
-                goto out;
-        if (!alloc_cpumask_var(&covered, GFP_KERNEL))
-                goto free_domainspan;
-        if (!alloc_cpumask_var(&notcovered, GFP_KERNEL))
-                goto free_covered;
-#endif
-
-        if (!alloc_cpumask_var(&nodemask, GFP_KERNEL))
-                goto free_notcovered;
-        if (!alloc_cpumask_var(&this_sibling_map, GFP_KERNEL))
-                goto free_nodemask;
-        if (!alloc_cpumask_var(&this_core_map, GFP_KERNEL))
-                goto free_this_sibling_map;
-        if (!alloc_cpumask_var(&send_covered, GFP_KERNEL))
-                goto free_this_core_map;
-        if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
-                goto free_send_covered;
+                kfree(d->sched_group_nodes); /* fall through */
+        case sa_notcovered:
+                free_cpumask_var(d->notcovered); /* fall through */
+        case sa_covered:
+                free_cpumask_var(d->covered); /* fall through */
+        case sa_domainspan:
+                free_cpumask_var(d->domainspan); /* fall through */
+#endif
+        case sa_none:
+                break;
+        }
+}
 
+static enum s_alloc __visit_domain_allocation_hell(struct s_data *d,
+                                                   const struct cpumask *cpu_map)
+{
 #ifdef CONFIG_NUMA
-        /*
-         * Allocate the per-node list of sched groups
-         */
-        sched_group_nodes = kcalloc(nr_node_ids, sizeof(struct sched_group *),
-                                    GFP_KERNEL);
-        if (!sched_group_nodes) {
+        if (!alloc_cpumask_var(&d->domainspan, GFP_KERNEL))
+                return sa_none;
+        if (!alloc_cpumask_var(&d->covered, GFP_KERNEL))
+                return sa_domainspan;
+        if (!alloc_cpumask_var(&d->notcovered, GFP_KERNEL))
+                return sa_covered;
+        /* Allocate the per-node list of sched groups */
+        d->sched_group_nodes = kcalloc(nr_node_ids,
+                                      sizeof(struct sched_group *), GFP_KERNEL);
+        if (!d->sched_group_nodes) {
                 printk(KERN_WARNING "Can not alloc sched group node list\n");
-                goto free_tmpmask;
-        }
-#endif
-
-        rd = alloc_rootdomain();
-        if (!rd) {
+                return sa_notcovered;
+        }
+        sched_group_nodes_bycpu[cpumask_first(cpu_map)] = d->sched_group_nodes;
+#endif
+        if (!alloc_cpumask_var(&d->nodemask, GFP_KERNEL))
+                return sa_sched_group_nodes;
+        if (!alloc_cpumask_var(&d->this_sibling_map, GFP_KERNEL))
+                return sa_nodemask;
+        if (!alloc_cpumask_var(&d->this_core_map, GFP_KERNEL))
+                return sa_this_sibling_map;
+        if (!alloc_cpumask_var(&d->send_covered, GFP_KERNEL))
+                return sa_this_core_map;
+        if (!alloc_cpumask_var(&d->tmpmask, GFP_KERNEL))
+                return sa_send_covered;
+        d->rd = alloc_rootdomain();
+        if (!d->rd) {
                 printk(KERN_WARNING "Cannot alloc root domain\n");
-                goto free_sched_groups;
+                return sa_tmpmask;
         }
+        return sa_rootdomain;
+}
 
+static struct sched_domain *__build_numa_sched_domains(struct s_data *d,
+        const struct cpumask *cpu_map, struct sched_domain_attr *attr, int i)
+{
+        struct sched_domain *sd = NULL;
 #ifdef CONFIG_NUMA
-        sched_group_nodes_bycpu[cpumask_first(cpu_map)] = sched_group_nodes;
-#endif
-
-        /*
-         * Set up domains for cpus specified by the cpu_map.
-         */
-        for_each_cpu(i, cpu_map) {
-                struct sched_domain *sd = NULL, *p;
-
-                cpumask_and(nodemask, cpumask_of_node(cpu_to_node(i)), cpu_map);
-
-#ifdef CONFIG_NUMA
-                if (cpumask_weight(cpu_map) >
-                                SD_NODES_PER_DOMAIN*cpumask_weight(nodemask)) {
-                        sd = &per_cpu(allnodes_domains, i).sd;
-                        SD_INIT(sd, ALLNODES);
-                        set_domain_attribute(sd, attr);
-                        cpumask_copy(sched_domain_span(sd), cpu_map);
-                        cpu_to_allnodes_group(i, cpu_map, &sd->groups, tmpmask);
-                        p = sd;
-                        sd_allnodes = 1;
-                } else
-                        p = NULL;
+        struct sched_domain *parent;
 
-                sd = &per_cpu(node_domains, i).sd;
-                SD_INIT(sd, NODE);
+        d->sd_allnodes = 0;
+        if (cpumask_weight(cpu_map) >
+            SD_NODES_PER_DOMAIN * cpumask_weight(d->nodemask)) {
+                sd = &per_cpu(allnodes_domains, i).sd;
+                SD_INIT(sd, ALLNODES);
                 set_domain_attribute(sd, attr);
-                sched_domain_node_span(cpu_to_node(i), sched_domain_span(sd));
-                sd->parent = p;
-                if (p)
-                        p->child = sd;
-                cpumask_and(sched_domain_span(sd),
-                            sched_domain_span(sd), cpu_map);
+                cpumask_copy(sched_domain_span(sd), cpu_map);
+                cpu_to_allnodes_group(i, cpu_map, &sd->groups, d->tmpmask);
+                d->sd_allnodes = 1;
+        }
+        parent = sd;
+
+        sd = &per_cpu(node_domains, i).sd;
+        SD_INIT(sd, NODE);
+        set_domain_attribute(sd, attr);
+        sched_domain_node_span(cpu_to_node(i), sched_domain_span(sd));
+        sd->parent = parent;
+        if (parent)
+                parent->child = sd;
+        cpumask_and(sched_domain_span(sd), sched_domain_span(sd), cpu_map);
 #endif
+        return sd;
+}
 
-                p = sd;
-                sd = &per_cpu(phys_domains, i).sd;
-                SD_INIT(sd, CPU);
-                set_domain_attribute(sd, attr);
-                cpumask_copy(sched_domain_span(sd), nodemask);
-                sd->parent = p;
-                if (p)
-                        p->child = sd;
-                cpu_to_phys_group(i, cpu_map, &sd->groups, tmpmask);
+static struct sched_domain *__build_cpu_sched_domain(struct s_data *d,
+        const struct cpumask *cpu_map, struct sched_domain_attr *attr,
+        struct sched_domain *parent, int i)
+{
+        struct sched_domain *sd;
+        sd = &per_cpu(phys_domains, i).sd;
+        SD_INIT(sd, CPU);
+        set_domain_attribute(sd, attr);
+        cpumask_copy(sched_domain_span(sd), d->nodemask);
+        sd->parent = parent;
+        if (parent)
+                parent->child = sd;
+        cpu_to_phys_group(i, cpu_map, &sd->groups, d->tmpmask);
+        return sd;
+}
 
+static struct sched_domain *__build_mc_sched_domain(struct s_data *d,
+        const struct cpumask *cpu_map, struct sched_domain_attr *attr,
+        struct sched_domain *parent, int i)
+{
+        struct sched_domain *sd = parent;
 #ifdef CONFIG_SCHED_MC
-                p = sd;
-                sd = &per_cpu(core_domains, i).sd;
-                SD_INIT(sd, MC);
-                set_domain_attribute(sd, attr);
-                cpumask_and(sched_domain_span(sd), cpu_map,
-                                                   cpu_coregroup_mask(i));
-                sd->parent = p;
-                p->child = sd;
-                cpu_to_core_group(i, cpu_map, &sd->groups, tmpmask);
+        sd = &per_cpu(core_domains, i).sd;
+        SD_INIT(sd, MC);
+        set_domain_attribute(sd, attr);
+        cpumask_and(sched_domain_span(sd), cpu_map, cpu_coregroup_mask(i));
+        sd->parent = parent;
+        parent->child = sd;
+        cpu_to_core_group(i, cpu_map, &sd->groups, d->tmpmask);
 #endif
+        return sd;
+}
 
+static struct sched_domain *__build_smt_sched_domain(struct s_data *d,
+        const struct cpumask *cpu_map, struct sched_domain_attr *attr,
+        struct sched_domain *parent, int i)
+{
+        struct sched_domain *sd = parent;
 #ifdef CONFIG_SCHED_SMT
-                p = sd;
-                sd = &per_cpu(cpu_domains, i).sd;
-                SD_INIT(sd, SIBLING);
-                set_domain_attribute(sd, attr);
-                cpumask_and(sched_domain_span(sd),
-                            topology_thread_cpumask(i), cpu_map);
-                sd->parent = p;
-                p->child = sd;
-                cpu_to_cpu_group(i, cpu_map, &sd->groups, tmpmask);
+        sd = &per_cpu(cpu_domains, i).sd;
+        SD_INIT(sd, SIBLING);
+        set_domain_attribute(sd, attr);
+        cpumask_and(sched_domain_span(sd), cpu_map, topology_thread_cpumask(i));
+        sd->parent = parent;
+        parent->child = sd;
+        cpu_to_cpu_group(i, cpu_map, &sd->groups, d->tmpmask);
 #endif
-        }
+        return sd;
+}
 
+static void build_sched_groups(struct s_data *d, enum sched_domain_level l,
+                               const struct cpumask *cpu_map, int cpu)
+{
+        switch (l) {
 #ifdef CONFIG_SCHED_SMT
-        /* Set up CPU (sibling) groups */
-        for_each_cpu(i, cpu_map) {
-                cpumask_and(this_sibling_map,
-                            topology_thread_cpumask(i), cpu_map);
-                if (i != cpumask_first(this_sibling_map))
-                        continue;
-
-                init_sched_build_groups(this_sibling_map, cpu_map,
-                                        &cpu_to_cpu_group,
-                                        send_covered, tmpmask);
-        }
+        case SD_LV_SIBLING: /* set up CPU (sibling) groups */
+                cpumask_and(d->this_sibling_map, cpu_map,
+                            topology_thread_cpumask(cpu));
+                if (cpu == cpumask_first(d->this_sibling_map))
+                        init_sched_build_groups(d->this_sibling_map, cpu_map,
+                                                &cpu_to_cpu_group,
+                                                d->send_covered, d->tmpmask);
+                break;
 #endif
-
 #ifdef CONFIG_SCHED_MC
-        /* Set up multi-core groups */
-        for_each_cpu(i, cpu_map) {
-                cpumask_and(this_core_map, cpu_coregroup_mask(i), cpu_map);
-                if (i != cpumask_first(this_core_map))
-                        continue;
-
-                init_sched_build_groups(this_core_map, cpu_map,
-                                        &cpu_to_core_group,
-                                        send_covered, tmpmask);
-        }
+        case SD_LV_MC: /* set up multi-core groups */
+                cpumask_and(d->this_core_map, cpu_map, cpu_coregroup_mask(cpu));
+                if (cpu == cpumask_first(d->this_core_map))
+                        init_sched_build_groups(d->this_core_map, cpu_map,
+                                                &cpu_to_core_group,
+                                                d->send_covered, d->tmpmask);
+                break;
 #endif
-
-        /* Set up physical groups */
-        for (i = 0; i < nr_node_ids; i++) {
-                cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
-                if (cpumask_empty(nodemask))
-                        continue;
-
-                init_sched_build_groups(nodemask, cpu_map,
-                                        &cpu_to_phys_group,
-                                        send_covered, tmpmask);
-        }
-
+        case SD_LV_CPU: /* set up physical groups */
+                cpumask_and(d->nodemask, cpumask_of_node(cpu), cpu_map);
+                if (!cpumask_empty(d->nodemask))
+                        init_sched_build_groups(d->nodemask, cpu_map,
+                                                &cpu_to_phys_group,
+                                                d->send_covered, d->tmpmask);
+                break;
 #ifdef CONFIG_NUMA
-        /* Set up node groups */
-        if (sd_allnodes) {
-                init_sched_build_groups(cpu_map, cpu_map,
-                                        &cpu_to_allnodes_group,
-                                        send_covered, tmpmask);
+        case SD_LV_ALLNODES:
+                init_sched_build_groups(cpu_map, cpu_map, &cpu_to_allnodes_group,
+                                        d->send_covered, d->tmpmask);
+                break;
+#endif
+        default:
+                break;
         }
+}
 
-        for (i = 0; i < nr_node_ids; i++) {
-                /* Set up node groups */
-                struct sched_group *sg, *prev;
-                int j;
-
-                cpumask_clear(covered);
-                cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
-                if (cpumask_empty(nodemask)) {
-                        sched_group_nodes[i] = NULL;
-                        continue;
-                }
+/*
+ * Build sched domains for a given set of cpus and attach the sched domains
+ * to the individual cpus
+ */
+static int __build_sched_domains(const struct cpumask *cpu_map,
+                                 struct sched_domain_attr *attr)
+{
+        enum s_alloc alloc_state = sa_none;
+        struct s_data d;
+        struct sched_domain *sd;
+        int i;
+#ifdef CONFIG_NUMA
+        d.sd_allnodes = 0;
+#endif
 
-                sched_domain_node_span(i, domainspan);
-                cpumask_and(domainspan, domainspan, cpu_map);
+        alloc_state = __visit_domain_allocation_hell(&d, cpu_map);
+        if (alloc_state != sa_rootdomain)
+                goto error;
+        alloc_state = sa_sched_groups;
 
-                sg = kmalloc_node(sizeof(struct sched_group) + cpumask_size(),
-                                  GFP_KERNEL, i);
-                if (!sg) {
-                        printk(KERN_WARNING "Can not alloc domain group for "
-                                "node %d\n", i);
-                        goto error;
-                }
-                sched_group_nodes[i] = sg;
-                for_each_cpu(j, nodemask) {
-                        struct sched_domain *sd;
+        /*
+         * Set up domains for cpus specified by the cpu_map.
+         */
+        for_each_cpu(i, cpu_map) {
+                cpumask_and(d.nodemask, cpumask_of_node(cpu_to_node(i)),
+                            cpu_map);
 
-                        sd = &per_cpu(node_domains, j).sd;
-                        sd->groups = sg;
-                }
-                sg->__cpu_power = 0;
-                cpumask_copy(sched_group_cpus(sg), nodemask);
-                sg->next = sg;
-                cpumask_or(covered, covered, nodemask);
-                prev = sg;
+                sd = __build_numa_sched_domains(&d, cpu_map, attr, i);
+                sd = __build_cpu_sched_domain(&d, cpu_map, attr, sd, i);
+                sd = __build_mc_sched_domain(&d, cpu_map, attr, sd, i);
+                sd = __build_smt_sched_domain(&d, cpu_map, attr, sd, i);
+        }
 
-                for (j = 0; j < nr_node_ids; j++) {
-                        int n = (i + j) % nr_node_ids;
+        for_each_cpu(i, cpu_map) {
+                build_sched_groups(&d, SD_LV_SIBLING, cpu_map, i);
+                build_sched_groups(&d, SD_LV_MC, cpu_map, i);
+        }
 
-                        cpumask_complement(notcovered, covered);
-                        cpumask_and(tmpmask, notcovered, cpu_map);
-                        cpumask_and(tmpmask, tmpmask, domainspan);
-                        if (cpumask_empty(tmpmask))
-                                break;
+        /* Set up physical groups */
+        for (i = 0; i < nr_node_ids; i++)
+                build_sched_groups(&d, SD_LV_CPU, cpu_map, i);
 
-                        cpumask_and(tmpmask, tmpmask, cpumask_of_node(n));
-                        if (cpumask_empty(tmpmask))
-                                continue;
+#ifdef CONFIG_NUMA
+        /* Set up node groups */
+        if (d.sd_allnodes)
+                build_sched_groups(&d, SD_LV_ALLNODES, cpu_map, 0);
 
-                        sg = kmalloc_node(sizeof(struct sched_group) +
-                                          cpumask_size(),
-                                          GFP_KERNEL, i);
-                        if (!sg) {
-                                printk(KERN_WARNING
-                                "Can not alloc domain group for node %d\n", j);
-                                goto error;
-                        }
-                        sg->__cpu_power = 0;
-                        cpumask_copy(sched_group_cpus(sg), tmpmask);
-                        sg->next = prev->next;
-                        cpumask_or(covered, covered, tmpmask);
-                        prev->next = sg;
-                        prev = sg;
-                }
-        }
+        for (i = 0; i < nr_node_ids; i++)
+                if (build_numa_sched_groups(&d, cpu_map, i))
+                        goto error;
 #endif
 
         /* Calculate CPU power for physical packages and nodes */
 #ifdef CONFIG_SCHED_SMT
         for_each_cpu(i, cpu_map) {
-                struct sched_domain *sd = &per_cpu(cpu_domains, i).sd;
-
+                sd = &per_cpu(cpu_domains, i).sd;
                 init_sched_groups_power(i, sd);
         }
 #endif
 #ifdef CONFIG_SCHED_MC
         for_each_cpu(i, cpu_map) {
-                struct sched_domain *sd = &per_cpu(core_domains, i).sd;
-
+                sd = &per_cpu(core_domains, i).sd;
                 init_sched_groups_power(i, sd);
         }
 #endif
 
         for_each_cpu(i, cpu_map) {
-                struct sched_domain *sd = &per_cpu(phys_domains, i).sd;
-
+                sd = &per_cpu(phys_domains, i).sd;
                 init_sched_groups_power(i, sd);
         }
 
 #ifdef CONFIG_NUMA
         for (i = 0; i < nr_node_ids; i++)
-                init_numa_sched_groups_power(sched_group_nodes[i]);
+                init_numa_sched_groups_power(d.sched_group_nodes[i]);
 
-        if (sd_allnodes) {
+        if (d.sd_allnodes) {
                 struct sched_group *sg;
 
                 cpu_to_allnodes_group(cpumask_first(cpu_map), cpu_map, &sg,
-                                                                tmpmask);
+                                                                d.tmpmask);
                 init_numa_sched_groups_power(sg);
         }
 #endif
 
         /* Attach the domains */
         for_each_cpu(i, cpu_map) {
-                struct sched_domain *sd;
 #ifdef CONFIG_SCHED_SMT
                 sd = &per_cpu(cpu_domains, i).sd;
 #elif defined(CONFIG_SCHED_MC)
@@ -8659,44 +9001,16 @@ static int __build_sched_domains(const struct cpumask *cpu_map,
 #else
                 sd = &per_cpu(phys_domains, i).sd;
 #endif
-                cpu_attach_domain(sd, rd, i);
+                cpu_attach_domain(sd, d.rd, i);
         }
 
-        err = 0;
-
-free_tmpmask:
-        free_cpumask_var(tmpmask);
-free_send_covered:
-        free_cpumask_var(send_covered);
-free_this_core_map:
-        free_cpumask_var(this_core_map);
-free_this_sibling_map:
-        free_cpumask_var(this_sibling_map);
-free_nodemask:
-        free_cpumask_var(nodemask);
-free_notcovered:
-#ifdef CONFIG_NUMA
-        free_cpumask_var(notcovered);
-free_covered:
-        free_cpumask_var(covered);
-free_domainspan:
-        free_cpumask_var(domainspan);
-out:
-#endif
-        return err;
-
-free_sched_groups:
-#ifdef CONFIG_NUMA
-        kfree(sched_group_nodes);
-#endif
-        goto free_tmpmask;
+        d.sched_group_nodes = NULL; /* don't free this we still need it */
+        __free_domain_allocs(&d, sa_tmpmask, cpu_map);
+        return 0;
 
-#ifdef CONFIG_NUMA
 error:
-        free_sched_groups(cpu_map, tmpmask);
-        free_rootdomain(rd);
-        goto free_tmpmask;
-#endif
+        __free_domain_allocs(&d, alloc_state, cpu_map);
+        return -ENOMEM;
 }
 
 static int build_sched_domains(const struct cpumask *cpu_map)
@@ -9304,11 +9618,11 @@ void __init sched_init(void)
                  * system cpu resource, based on the weight assigned to root
                  * user's cpu share (INIT_TASK_GROUP_LOAD). This is accomplished
                  * by letting tasks of init_task_group sit in a separate cfs_rq
-                 * (init_cfs_rq) and having one entity represent this group of
+                 * (init_tg_cfs_rq) and having one entity represent this group of
                  * tasks in rq->cfs (i.e init_task_group->se[] != NULL).
                  */
                 init_tg_cfs_entry(&init_task_group,
-                                &per_cpu(init_cfs_rq, i),
+                                &per_cpu(init_tg_cfs_rq, i),
                                 &per_cpu(init_sched_entity, i), i, 1,
                                 root_task_group.se[i]);
 
@@ -9334,6 +9648,7 @@ void __init sched_init(void)
 #ifdef CONFIG_SMP
                 rq->sd = NULL;
                 rq->rd = NULL;
+                rq->post_schedule = 0;
                 rq->active_balance = 0;
                 rq->next_balance = jiffies;
                 rq->push_cpu = 0;
@@ -9398,13 +9713,20 @@ void __init sched_init(void)
 }
 
 #ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
-void __might_sleep(char *file, int line)
+static inline int preempt_count_equals(int preempt_offset)
+{
+        int nested = preempt_count() & ~PREEMPT_ACTIVE;
+
+        return (nested == PREEMPT_INATOMIC_BASE + preempt_offset);
+}
+
+void __might_sleep(char *file, int line, int preempt_offset)
 {
 #ifdef in_atomic
         static unsigned long prev_jiffy;        /* ratelimiting */
 
-        if ((!in_atomic() && !irqs_disabled()) ||
-                    system_state != SYSTEM_RUNNING || oops_in_progress)
+        if ((preempt_count_equals(preempt_offset) && !irqs_disabled()) ||
+            system_state != SYSTEM_RUNNING || oops_in_progress)
                 return;
         if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
                 return;
@@ -10581,3 +10903,113 @@ struct cgroup_subsys cpuacct_subsys = {
         .subsys_id = cpuacct_subsys_id,
 };
 #endif  /* CONFIG_CGROUP_CPUACCT */
+
+#ifndef CONFIG_SMP
+
+int rcu_expedited_torture_stats(char *page)
+{
+        return 0;
+}
+EXPORT_SYMBOL_GPL(rcu_expedited_torture_stats);
+
+void synchronize_sched_expedited(void)
+{
+}
+EXPORT_SYMBOL_GPL(synchronize_sched_expedited);
+
+#else /* #ifndef CONFIG_SMP */
+
+static DEFINE_PER_CPU(struct migration_req, rcu_migration_req);
+static DEFINE_MUTEX(rcu_sched_expedited_mutex);
+
+#define RCU_EXPEDITED_STATE_POST -2
+#define RCU_EXPEDITED_STATE_IDLE -1
+
+static int rcu_expedited_state = RCU_EXPEDITED_STATE_IDLE;
+
+int rcu_expedited_torture_stats(char *page)
+{
+        int cnt = 0;
+        int cpu;
+
+        cnt += sprintf(&page[cnt], "state: %d /", rcu_expedited_state);
+        for_each_online_cpu(cpu) {
+                 cnt += sprintf(&page[cnt], " %d:%d",
+                                cpu, per_cpu(rcu_migration_req, cpu).dest_cpu);
+        }
+        cnt += sprintf(&page[cnt], "\n");
+        return cnt;
+}
+EXPORT_SYMBOL_GPL(rcu_expedited_torture_stats);
+
+static long synchronize_sched_expedited_count;
+
+/*
+ * Wait for an rcu-sched grace period to elapse, but use "big hammer"
+ * approach to force grace period to end quickly.  This consumes
+ * significant time on all CPUs, and is thus not recommended for
+ * any sort of common-case code.
+ *
+ * Note that it is illegal to call this function while holding any
+ * lock that is acquired by a CPU-hotplug notifier.  Failing to
+ * observe this restriction will result in deadlock.
+ */
+void synchronize_sched_expedited(void)
+{
+        int cpu;
+        unsigned long flags;
+        bool need_full_sync = 0;
+        struct rq *rq;
+        struct migration_req *req;
+        long snap;
+        int trycount = 0;
+
+        smp_mb();  /* ensure prior mod happens before capturing snap. */
+        snap = ACCESS_ONCE(synchronize_sched_expedited_count) + 1;
+        get_online_cpus();
+        while (!mutex_trylock(&rcu_sched_expedited_mutex)) {
+                put_online_cpus();
+                if (trycount++ < 10)
+                        udelay(trycount * num_online_cpus());
+                else {
+                        synchronize_sched();
+                        return;
+                }
+                if (ACCESS_ONCE(synchronize_sched_expedited_count) - snap > 0) {
+                        smp_mb(); /* ensure test happens before caller kfree */
+                        return;
+                }
+                get_online_cpus();
+        }
+        rcu_expedited_state = RCU_EXPEDITED_STATE_POST;
+        for_each_online_cpu(cpu) {
+                rq = cpu_rq(cpu);
+                req = &per_cpu(rcu_migration_req, cpu);
+                init_completion(&req->done);
+                req->task = NULL;
+                req->dest_cpu = RCU_MIGRATION_NEED_QS;
+                spin_lock_irqsave(&rq->lock, flags);
+                list_add(&req->list, &rq->migration_queue);
+                spin_unlock_irqrestore(&rq->lock, flags);
+                wake_up_process(rq->migration_thread);
+        }
+        for_each_online_cpu(cpu) {
+                rcu_expedited_state = cpu;
+                req = &per_cpu(rcu_migration_req, cpu);
+                rq = cpu_rq(cpu);
+                wait_for_completion(&req->done);
+                spin_lock_irqsave(&rq->lock, flags);
+                if (unlikely(req->dest_cpu == RCU_MIGRATION_MUST_SYNC))
+                        need_full_sync = 1;
+                req->dest_cpu = RCU_MIGRATION_IDLE;
+                spin_unlock_irqrestore(&rq->lock, flags);
+        }
+        rcu_expedited_state = RCU_EXPEDITED_STATE_IDLE;
+        mutex_unlock(&rcu_sched_expedited_mutex);
+        put_online_cpus();
+        if (need_full_sync)
+                synchronize_sched();
+}
+EXPORT_SYMBOL_GPL(synchronize_sched_expedited);
+
+#endif /* #else #ifndef CONFIG_SMP */
diff --git a/kernel/sched_cpupri.c b/kernel/sched_cpupri.c
index d014efbf947a..0f052fc674d5 100644
--- a/kernel/sched_cpupri.c
+++ b/kernel/sched_cpupri.c
@@ -127,21 +127,11 @@ void cpupri_set(struct cpupri *cp, int cpu, int newpri)
 
         /*
          * If the cpu was currently mapped to a different value, we
-         * first need to unmap the old value
+         * need to map it to the new value then remove the old value.
+         * Note, we must add the new value first, otherwise we risk the
+         * cpu being cleared from pri_active, and this cpu could be
+         * missed for a push or pull.
          */
-        if (likely(oldpri != CPUPRI_INVALID)) {
-                struct cpupri_vec *vec  = &cp->pri_to_cpu[oldpri];
-
-                spin_lock_irqsave(&vec->lock, flags);
-
-                vec->count--;
-                if (!vec->count)
-                        clear_bit(oldpri, cp->pri_active);
-                cpumask_clear_cpu(cpu, vec->mask);
-
-                spin_unlock_irqrestore(&vec->lock, flags);
-        }
-
         if (likely(newpri != CPUPRI_INVALID)) {
                 struct cpupri_vec *vec = &cp->pri_to_cpu[newpri];
 
@@ -154,6 +144,18 @@ void cpupri_set(struct cpupri *cp, int cpu, int newpri)
 
                 spin_unlock_irqrestore(&vec->lock, flags);
         }
+        if (likely(oldpri != CPUPRI_INVALID)) {
+                struct cpupri_vec *vec  = &cp->pri_to_cpu[oldpri];
+
+                spin_lock_irqsave(&vec->lock, flags);
+
+                vec->count--;
+                if (!vec->count)
+                        clear_bit(oldpri, cp->pri_active);
+                cpumask_clear_cpu(cpu, vec->mask);
+
+                spin_unlock_irqrestore(&vec->lock, flags);
+        }
 
         *currpri = newpri;
 }
diff --git a/kernel/sched_debug.c b/kernel/sched_debug.c
index 70c7e0b79946..5ddbd0891267 100644
--- a/kernel/sched_debug.c
+++ b/kernel/sched_debug.c
@@ -409,6 +409,8 @@ void proc_sched_show_task(struct task_struct *p, struct seq_file *m)
         PN(se.wait_max);
         PN(se.wait_sum);
         P(se.wait_count);
+        PN(se.iowait_sum);
+        P(se.iowait_count);
         P(sched_info.bkl_count);
         P(se.nr_migrations);
         P(se.nr_migrations_cold);
@@ -479,6 +481,8 @@ void proc_sched_set_task(struct task_struct *p)
         p->se.wait_max                          = 0;
         p->se.wait_sum                          = 0;
         p->se.wait_count                        = 0;
+        p->se.iowait_sum                        = 0;
+        p->se.iowait_count                      = 0;
         p->se.sleep_max                         = 0;
         p->se.sum_sleep_runtime                 = 0;
         p->se.block_max                         = 0;
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index 652e8bdef9aa..aa7f84121016 100644
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -24,7 +24,7 @@
 
 /*
  * Targeted preemption latency for CPU-bound tasks:
- * (default: 20ms * (1 + ilog(ncpus)), units: nanoseconds)
+ * (default: 5ms * (1 + ilog(ncpus)), units: nanoseconds)
  *
  * NOTE: this latency value is not the same as the concept of
  * 'timeslice length' - timeslices in CFS are of variable length
@@ -34,13 +34,13 @@
  * (to see the precise effective timeslice length of your workload,
  *  run vmstat and monitor the context-switches (cs) field)
  */
-unsigned int sysctl_sched_latency = 20000000ULL;
+unsigned int sysctl_sched_latency = 5000000ULL;
 
 /*
  * Minimal preemption granularity for CPU-bound tasks:
- * (default: 4 msec * (1 + ilog(ncpus)), units: nanoseconds)
+ * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds)
  */
-unsigned int sysctl_sched_min_granularity = 4000000ULL;
+unsigned int sysctl_sched_min_granularity = 1000000ULL;
 
 /*
  * is kept at sysctl_sched_latency / sysctl_sched_min_granularity
@@ -48,10 +48,10 @@ unsigned int sysctl_sched_min_granularity = 4000000ULL;
 static unsigned int sched_nr_latency = 5;
 
 /*
- * After fork, child runs first. (default) If set to 0 then
+ * After fork, child runs first. If set to 0 (default) then
  * parent will (try to) run first.
  */
-const_debug unsigned int sysctl_sched_child_runs_first = 1;
+unsigned int sysctl_sched_child_runs_first __read_mostly;
 
 /*
  * sys_sched_yield() compat mode
@@ -63,13 +63,13 @@ unsigned int __read_mostly sysctl_sched_compat_yield;
 
 /*
  * SCHED_OTHER wake-up granularity.
- * (default: 5 msec * (1 + ilog(ncpus)), units: nanoseconds)
+ * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds)
  *
  * This option delays the preemption effects of decoupled workloads
  * and reduces their over-scheduling. Synchronous workloads will still
  * have immediate wakeup/sleep latencies.
  */
-unsigned int sysctl_sched_wakeup_granularity = 5000000UL;
+unsigned int sysctl_sched_wakeup_granularity = 1000000UL;
 
 const_debug unsigned int sysctl_sched_migration_cost = 500000UL;
 
@@ -79,11 +79,6 @@ static const struct sched_class fair_sched_class;
  * CFS operations on generic schedulable entities:
  */
 
-static inline struct task_struct *task_of(struct sched_entity *se)
-{
-        return container_of(se, struct task_struct, se);
-}
-
 #ifdef CONFIG_FAIR_GROUP_SCHED
 
 /* cpu runqueue to which this cfs_rq is attached */
@@ -95,6 +90,14 @@ static inline struct rq *rq_of(struct cfs_rq *cfs_rq)
 /* An entity is a task if it doesn't "own" a runqueue */
 #define entity_is_task(se)      (!se->my_q)
 
+static inline struct task_struct *task_of(struct sched_entity *se)
+{
+#ifdef CONFIG_SCHED_DEBUG
+        WARN_ON_ONCE(!entity_is_task(se));
+#endif
+        return container_of(se, struct task_struct, se);
+}
+
 /* Walk up scheduling entities hierarchy */
 #define for_each_sched_entity(se) \
                 for (; se; se = se->parent)
@@ -186,7 +189,12 @@ find_matching_se(struct sched_entity **se, struct sched_entity **pse)
         }
 }
 
-#else   /* CONFIG_FAIR_GROUP_SCHED */
+#else   /* !CONFIG_FAIR_GROUP_SCHED */
+
+static inline struct task_struct *task_of(struct sched_entity *se)
+{
+        return container_of(se, struct task_struct, se);
+}
 
 static inline struct rq *rq_of(struct cfs_rq *cfs_rq)
 {
@@ -537,6 +545,12 @@ update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se)
         schedstat_set(se->wait_count, se->wait_count + 1);
         schedstat_set(se->wait_sum, se->wait_sum +
                         rq_of(cfs_rq)->clock - se->wait_start);
+#ifdef CONFIG_SCHEDSTATS
+        if (entity_is_task(se)) {
+                trace_sched_stat_wait(task_of(se),
+                        rq_of(cfs_rq)->clock - se->wait_start);
+        }
+#endif
         schedstat_set(se->wait_start, 0);
 }
 
@@ -628,8 +642,10 @@ static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)
                 se->sleep_start = 0;
                 se->sum_sleep_runtime += delta;
 
-                if (tsk)
+                if (tsk) {
                         account_scheduler_latency(tsk, delta >> 10, 1);
+                        trace_sched_stat_sleep(tsk, delta);
+                }
         }
         if (se->block_start) {
                 u64 delta = rq_of(cfs_rq)->clock - se->block_start;
@@ -644,6 +660,12 @@ static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)
                 se->sum_sleep_runtime += delta;
 
                 if (tsk) {
+                        if (tsk->in_iowait) {
+                                se->iowait_sum += delta;
+                                se->iowait_count++;
+                                trace_sched_stat_iowait(tsk, delta);
+                        }
+
                         /*
                          * Blocking time is in units of nanosecs, so shift by
                          * 20 to get a milliseconds-range estimation of the
@@ -705,11 +727,11 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial)
 
                         vruntime -= thresh;
                 }
-
-                /* ensure we never gain time by being placed backwards. */
-                vruntime = max_vruntime(se->vruntime, vruntime);
         }
 
+        /* ensure we never gain time by being placed backwards. */
+        vruntime = max_vruntime(se->vruntime, vruntime);
+
         se->vruntime = vruntime;
 }
 
@@ -1046,17 +1068,21 @@ static void yield_task_fair(struct rq *rq)
  * search starts with cpus closest then further out as needed,
  * so we always favor a closer, idle cpu.
  * Domains may include CPUs that are not usable for migration,
- * hence we need to mask them out (cpu_active_mask)
+ * hence we need to mask them out (rq->rd->online)
  *
  * Returns the CPU we should wake onto.
  */
 #if defined(ARCH_HAS_SCHED_WAKE_IDLE)
+
+#define cpu_rd_active(cpu, rq) cpumask_test_cpu(cpu, rq->rd->online)
+
 static int wake_idle(int cpu, struct task_struct *p)
 {
         struct sched_domain *sd;
         int i;
         unsigned int chosen_wakeup_cpu;
         int this_cpu;
+        struct rq *task_rq = task_rq(p);
 
         /*
          * At POWERSAVINGS_BALANCE_WAKEUP level, if both this_cpu and prev_cpu
@@ -1089,10 +1115,10 @@ static int wake_idle(int cpu, struct task_struct *p)
         for_each_domain(cpu, sd) {
                 if ((sd->flags & SD_WAKE_IDLE)
                     || ((sd->flags & SD_WAKE_IDLE_FAR)
-                        && !task_hot(p, task_rq(p)->clock, sd))) {
+                        && !task_hot(p, task_rq->clock, sd))) {
                         for_each_cpu_and(i, sched_domain_span(sd),
                                          &p->cpus_allowed) {
-                                if (cpu_active(i) && idle_cpu(i)) {
+                                if (cpu_rd_active(i, task_rq) && idle_cpu(i)) {
                                         if (i != task_cpu(p)) {
                                                 schedstat_inc(p,
                                                        se.nr_wakeups_idle);
@@ -1235,7 +1261,17 @@ wake_affine(struct sched_domain *this_sd, struct rq *this_rq,
         tg = task_group(p);
         weight = p->se.load.weight;
 
-        balanced = 100*(tl + effective_load(tg, this_cpu, weight, weight)) <=
+        /*
+         * In low-load situations, where prev_cpu is idle and this_cpu is idle
+         * due to the sync cause above having dropped tl to 0, we'll always have
+         * an imbalance, but there's really nothing you can do about that, so
+         * that's good too.
+         *
+         * Otherwise check if either cpus are near enough in load to allow this
+         * task to be woken on this_cpu.
+         */
+        balanced = !tl ||
+                100*(tl + effective_load(tg, this_cpu, weight, weight)) <=
                 imbalance*(load + effective_load(tg, prev_cpu, 0, weight));
 
         /*
@@ -1278,8 +1314,6 @@ static int select_task_rq_fair(struct task_struct *p, int sync)
         this_rq         = cpu_rq(this_cpu);
         new_cpu         = prev_cpu;
 
-        if (prev_cpu == this_cpu)
-                goto out;
         /*
          * 'this_sd' is the first domain that both
          * this_cpu and prev_cpu are present in:
@@ -1721,6 +1755,8 @@ static void task_new_fair(struct rq *rq, struct task_struct *p)
         sched_info_queued(p);
 
         update_curr(cfs_rq);
+        if (curr)
+                se->vruntime = curr->vruntime;
         place_entity(cfs_rq, se, 1);
 
         /* 'curr' will be NULL if the child belongs to a different group */
diff --git a/kernel/sched_features.h b/kernel/sched_features.h
index 4569bfa7df9b..e2dc63a5815d 100644
--- a/kernel/sched_features.h
+++ b/kernel/sched_features.h
@@ -1,4 +1,4 @@
-SCHED_FEAT(NEW_FAIR_SLEEPERS, 1)
+SCHED_FEAT(NEW_FAIR_SLEEPERS, 0)
 SCHED_FEAT(NORMALIZED_SLEEPER, 0)
 SCHED_FEAT(ADAPTIVE_GRAN, 1)
 SCHED_FEAT(WAKEUP_PREEMPT, 1)
diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c
index 3918e01994e0..2eb4bd6a526c 100644
--- a/kernel/sched_rt.c
+++ b/kernel/sched_rt.c
@@ -3,15 +3,18 @@
  * policies)
  */
 
+#ifdef CONFIG_RT_GROUP_SCHED
+
+#define rt_entity_is_task(rt_se) (!(rt_se)->my_q)
+
 static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se)
 {
+#ifdef CONFIG_SCHED_DEBUG
+        WARN_ON_ONCE(!rt_entity_is_task(rt_se));
+#endif
         return container_of(rt_se, struct task_struct, rt);
 }
 
-#ifdef CONFIG_RT_GROUP_SCHED
-
-#define rt_entity_is_task(rt_se) (!(rt_se)->my_q)
-
 static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)
 {
         return rt_rq->rq;
@@ -26,6 +29,11 @@ static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)
 
 #define rt_entity_is_task(rt_se) (1)
 
+static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se)
+{
+        return container_of(rt_se, struct task_struct, rt);
+}
+
 static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)
 {
         return container_of(rt_rq, struct rq, rt);
@@ -128,6 +136,11 @@ static void dequeue_pushable_task(struct rq *rq, struct task_struct *p)
         plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);
 }
 
+static inline int has_pushable_tasks(struct rq *rq)
+{
+        return !plist_head_empty(&rq->rt.pushable_tasks);
+}
+
 #else
 
 static inline void enqueue_pushable_task(struct rq *rq, struct task_struct *p)
@@ -602,6 +615,8 @@ static void update_curr_rt(struct rq *rq)
         curr->se.exec_start = rq->clock;
         cpuacct_charge(curr, delta_exec);
 
+        sched_rt_avg_update(rq, delta_exec);
+
         if (!rt_bandwidth_enabled())
                 return;
 
@@ -874,8 +889,6 @@ static void enqueue_task_rt(struct rq *rq, struct task_struct *p, int wakeup)
 
         if (!task_current(rq, p) && p->rt.nr_cpus_allowed > 1)
                 enqueue_pushable_task(rq, p);
-
-        inc_cpu_load(rq, p->se.load.weight);
 }
 
 static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int sleep)
@@ -886,8 +899,6 @@ static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int sleep)
         dequeue_rt_entity(rt_se);
 
         dequeue_pushable_task(rq, p);
-
-        dec_cpu_load(rq, p->se.load.weight);
 }
 
 /*
@@ -1064,6 +1075,14 @@ static struct task_struct *pick_next_task_rt(struct rq *rq)
         if (p)
                 dequeue_pushable_task(rq, p);
 
+#ifdef CONFIG_SMP
+        /*
+         * We detect this state here so that we can avoid taking the RQ
+         * lock again later if there is no need to push
+         */
+        rq->post_schedule = has_pushable_tasks(rq);
+#endif
+
         return p;
 }
 
@@ -1162,13 +1181,6 @@ static int find_lowest_rq(struct task_struct *task)
                 return -1; /* No targets found */
 
         /*
-         * Only consider CPUs that are usable for migration.
-         * I guess we might want to change cpupri_find() to ignore those
-         * in the first place.
-         */
-        cpumask_and(lowest_mask, lowest_mask, cpu_active_mask);
-
-        /*
          * At this point we have built a mask of cpus representing the
          * lowest priority tasks in the system.  Now we want to elect
          * the best one based on our affinity and topology.
@@ -1262,11 +1274,6 @@ static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq)
         return lowest_rq;
 }
 
-static inline int has_pushable_tasks(struct rq *rq)
-{
-        return !plist_head_empty(&rq->rt.pushable_tasks);
-}
-
 static struct task_struct *pick_next_pushable_task(struct rq *rq)
 {
         struct task_struct *p;
@@ -1466,23 +1473,9 @@ static void pre_schedule_rt(struct rq *rq, struct task_struct *prev)
                 pull_rt_task(rq);
 }
 
-/*
- * assumes rq->lock is held
- */
-static int needs_post_schedule_rt(struct rq *rq)
-{
-        return has_pushable_tasks(rq);
-}
-
 static void post_schedule_rt(struct rq *rq)
 {
-        /*
-         * This is only called if needs_post_schedule_rt() indicates that
-         * we need to push tasks away
-         */
-        spin_lock_irq(&rq->lock);
         push_rt_tasks(rq);
-        spin_unlock_irq(&rq->lock);
 }
 
 /*
@@ -1758,7 +1751,6 @@ static const struct sched_class rt_sched_class = {
         .rq_online              = rq_online_rt,
         .rq_offline             = rq_offline_rt,
         .pre_schedule           = pre_schedule_rt,
-        .needs_post_schedule    = needs_post_schedule_rt,
         .post_schedule          = post_schedule_rt,
         .task_wake_up           = task_wake_up_rt,
         .switched_from          = switched_from_rt,
diff --git a/kernel/smp.c b/kernel/smp.c
index 94188b8ecc33..8e218500ab14 100644
--- a/kernel/smp.c
+++ b/kernel/smp.c
@@ -177,6 +177,11 @@ void generic_smp_call_function_interrupt(void)
         int cpu = get_cpu();
 
         /*
+         * Shouldn't receive this interrupt on a cpu that is not yet online.
+         */
+        WARN_ON_ONCE(!cpu_online(cpu));
+
+        /*
          * Ensure entry is visible on call_function_queue after we have
          * entered the IPI. See comment in smp_call_function_many.
          * If we don't have this, then we may miss an entry on the list
@@ -230,6 +235,11 @@ void generic_smp_call_function_single_interrupt(void)
         unsigned int data_flags;
         LIST_HEAD(list);
 
+        /*
+         * Shouldn't receive this interrupt on a cpu that is not yet online.
+         */
+        WARN_ON_ONCE(!cpu_online(smp_processor_id()));
+
         spin_lock(&q->lock);
         list_replace_init(&q->list, &list);
         spin_unlock(&q->lock);
@@ -285,8 +295,14 @@ int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
          */
         this_cpu = get_cpu();
 
-        /* Can deadlock when called with interrupts disabled */
-        WARN_ON_ONCE(irqs_disabled() && !oops_in_progress);
+        /*
+         * Can deadlock when called with interrupts disabled.
+         * We allow cpu's that are not yet online though, as no one else can
+         * send smp call function interrupt to this cpu and as such deadlocks
+         * can't happen.
+         */
+        WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled()
+                     && !oops_in_progress);
 
         if (cpu == this_cpu) {
                 local_irq_save(flags);
@@ -329,8 +345,14 @@ void __smp_call_function_single(int cpu, struct call_single_data *data,
 {
         csd_lock(data);
 
-        /* Can deadlock when called with interrupts disabled */
-        WARN_ON_ONCE(wait && irqs_disabled() && !oops_in_progress);
+        /*
+         * Can deadlock when called with interrupts disabled.
+         * We allow cpu's that are not yet online though, as no one else can
+         * send smp call function interrupt to this cpu and as such deadlocks
+         * can't happen.
+         */
+        WARN_ON_ONCE(cpu_online(smp_processor_id()) && wait && irqs_disabled()
+                     && !oops_in_progress);
 
         generic_exec_single(cpu, data, wait);
 }
@@ -365,8 +387,14 @@ void smp_call_function_many(const struct cpumask *mask,
         unsigned long flags;
         int cpu, next_cpu, this_cpu = smp_processor_id();
 
-        /* Can deadlock when called with interrupts disabled */
-        WARN_ON_ONCE(irqs_disabled() && !oops_in_progress);
+        /*
+         * Can deadlock when called with interrupts disabled.
+         * We allow cpu's that are not yet online though, as no one else can
+         * send smp call function interrupt to this cpu and as such deadlocks
+         * can't happen.
+         */
+        WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled()
+                     && !oops_in_progress);
 
         /* So, what's a CPU they want? Ignoring this one. */
         cpu = cpumask_first_and(mask, cpu_online_mask);
diff --git a/kernel/softirq.c b/kernel/softirq.c
index eb5e131a0485..7db25067cd2d 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -227,7 +227,7 @@ restart:
                                 preempt_count() = prev_count;
                         }
 
-                        rcu_bh_qsctr_inc(cpu);
+                        rcu_bh_qs(cpu);
                 }
                 h++;
                 pending >>= 1;
@@ -721,7 +721,7 @@ static int ksoftirqd(void * __bind_cpu)
                         preempt_enable_no_resched();
                         cond_resched();
                         preempt_disable();
-                        rcu_qsctr_inc((long)__bind_cpu);
+                        rcu_sched_qs((long)__bind_cpu);
                 }
                 preempt_enable();
                 set_current_state(TASK_INTERRUPTIBLE);
diff --git a/kernel/spinlock.c b/kernel/spinlock.c
index 7932653c4ebd..5ddab730cb2f 100644
--- a/kernel/spinlock.c
+++ b/kernel/spinlock.c
@@ -21,44 +21,29 @@
 #include <linux/debug_locks.h>
 #include <linux/module.h>
 
+#ifndef _spin_trylock
 int __lockfunc _spin_trylock(spinlock_t *lock)
 {
-        preempt_disable();
-        if (_raw_spin_trylock(lock)) {
-                spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
-                return 1;
-        }
-        
-        preempt_enable();
-        return 0;
+        return __spin_trylock(lock);
 }
 EXPORT_SYMBOL(_spin_trylock);
+#endif
 
+#ifndef _read_trylock
 int __lockfunc _read_trylock(rwlock_t *lock)
 {
-        preempt_disable();
-        if (_raw_read_trylock(lock)) {
-                rwlock_acquire_read(&lock->dep_map, 0, 1, _RET_IP_);
-                return 1;
-        }
-
-        preempt_enable();
-        return 0;
+        return __read_trylock(lock);
 }
 EXPORT_SYMBOL(_read_trylock);
+#endif
 
+#ifndef _write_trylock
 int __lockfunc _write_trylock(rwlock_t *lock)
 {
-        preempt_disable();
-        if (_raw_write_trylock(lock)) {
-                rwlock_acquire(&lock->dep_map, 0, 1, _RET_IP_);
-                return 1;
-        }
-
-        preempt_enable();
-        return 0;
+        return __write_trylock(lock);
 }
 EXPORT_SYMBOL(_write_trylock);
+#endif
 
 /*
  * If lockdep is enabled then we use the non-preemption spin-ops
@@ -67,132 +52,101 @@ EXPORT_SYMBOL(_write_trylock);
  */
 #if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)
 
+#ifndef _read_lock
 void __lockfunc _read_lock(rwlock_t *lock)
 {
-        preempt_disable();
-        rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
-        LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
+        __read_lock(lock);
 }
 EXPORT_SYMBOL(_read_lock);
+#endif
 
+#ifndef _spin_lock_irqsave
 unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
 {
-        unsigned long flags;
-
-        local_irq_save(flags);
-        preempt_disable();
-        spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
-        /*
-         * On lockdep we dont want the hand-coded irq-enable of
-         * _raw_spin_lock_flags() code, because lockdep assumes
-         * that interrupts are not re-enabled during lock-acquire:
-         */
-#ifdef CONFIG_LOCKDEP
-        LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
-#else
-        _raw_spin_lock_flags(lock, &flags);
-#endif
-        return flags;
+        return __spin_lock_irqsave(lock);
 }
 EXPORT_SYMBOL(_spin_lock_irqsave);
+#endif
 
+#ifndef _spin_lock_irq
 void __lockfunc _spin_lock_irq(spinlock_t *lock)
 {
-        local_irq_disable();
-        preempt_disable();
-        spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
-        LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
+        __spin_lock_irq(lock);
 }
 EXPORT_SYMBOL(_spin_lock_irq);
+#endif
 
+#ifndef _spin_lock_bh
 void __lockfunc _spin_lock_bh(spinlock_t *lock)
 {
-        local_bh_disable();
-        preempt_disable();
-        spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
-        LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
+        __spin_lock_bh(lock);
 }
 EXPORT_SYMBOL(_spin_lock_bh);
+#endif
 
+#ifndef _read_lock_irqsave
 unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)
 {
-        unsigned long flags;
-
-        local_irq_save(flags);
-        preempt_disable();
-        rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
-        LOCK_CONTENDED_FLAGS(lock, _raw_read_trylock, _raw_read_lock,
-                             _raw_read_lock_flags, &flags);
-        return flags;
+        return __read_lock_irqsave(lock);
 }
 EXPORT_SYMBOL(_read_lock_irqsave);
+#endif
 
+#ifndef _read_lock_irq
 void __lockfunc _read_lock_irq(rwlock_t *lock)
 {
-        local_irq_disable();
-        preempt_disable();
-        rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
-        LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
+        __read_lock_irq(lock);
 }
 EXPORT_SYMBOL(_read_lock_irq);
+#endif
 
+#ifndef _read_lock_bh
 void __lockfunc _read_lock_bh(rwlock_t *lock)
 {
-        local_bh_disable();
-        preempt_disable();
-        rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
-        LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
+        __read_lock_bh(lock);
 }
 EXPORT_SYMBOL(_read_lock_bh);
+#endif
 
+#ifndef _write_lock_irqsave
 unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)
 {
-        unsigned long flags;
-
-        local_irq_save(flags);
-        preempt_disable();
-        rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
-        LOCK_CONTENDED_FLAGS(lock, _raw_write_trylock, _raw_write_lock,
-                             _raw_write_lock_flags, &flags);
-        return flags;
+        return __write_lock_irqsave(lock);
 }
 EXPORT_SYMBOL(_write_lock_irqsave);
+#endif
 
+#ifndef _write_lock_irq
 void __lockfunc _write_lock_irq(rwlock_t *lock)
 {
-        local_irq_disable();
-        preempt_disable();
-        rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
-        LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
+        __write_lock_irq(lock);
 }
 EXPORT_SYMBOL(_write_lock_irq);
+#endif
 
+#ifndef _write_lock_bh
 void __lockfunc _write_lock_bh(rwlock_t *lock)
 {
-        local_bh_disable();
-        preempt_disable();
-        rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
-        LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
+        __write_lock_bh(lock);
 }
 EXPORT_SYMBOL(_write_lock_bh);
+#endif
 
+#ifndef _spin_lock
 void __lockfunc _spin_lock(spinlock_t *lock)
 {
-        preempt_disable();
-        spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
-        LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
+        __spin_lock(lock);
 }
-
 EXPORT_SYMBOL(_spin_lock);
+#endif
 
+#ifndef _write_lock
 void __lockfunc _write_lock(rwlock_t *lock)
 {
-        preempt_disable();
-        rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
-        LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
+        __write_lock(lock);
 }
-
 EXPORT_SYMBOL(_write_lock);
+#endif
 
 #else /* CONFIG_PREEMPT: */
 
@@ -318,125 +272,109 @@ EXPORT_SYMBOL(_spin_lock_nest_lock);
 
 #endif
 
+#ifndef _spin_unlock
 void __lockfunc _spin_unlock(spinlock_t *lock)
 {
-        spin_release(&lock->dep_map, 1, _RET_IP_);
-        _raw_spin_unlock(lock);
-        preempt_enable();
+        __spin_unlock(lock);
 }
 EXPORT_SYMBOL(_spin_unlock);
+#endif
 
+#ifndef _write_unlock
 void __lockfunc _write_unlock(rwlock_t *lock)
 {
-        rwlock_release(&lock->dep_map, 1, _RET_IP_);
-        _raw_write_unlock(lock);
-        preempt_enable();
+        __write_unlock(lock);
 }
 EXPORT_SYMBOL(_write_unlock);
+#endif
 
+#ifndef _read_unlock
 void __lockfunc _read_unlock(rwlock_t *lock)
 {
-        rwlock_release(&lock->dep_map, 1, _RET_IP_);
-        _raw_read_unlock(lock);
-        preempt_enable();
+        __read_unlock(lock);
 }
 EXPORT_SYMBOL(_read_unlock);
+#endif
 
+#ifndef _spin_unlock_irqrestore
 void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
 {
-        spin_release(&lock->dep_map, 1, _RET_IP_);
-        _raw_spin_unlock(lock);
-        local_irq_restore(flags);
-        preempt_enable();
+        __spin_unlock_irqrestore(lock, flags);
 }
 EXPORT_SYMBOL(_spin_unlock_irqrestore);
+#endif
 
+#ifndef _spin_unlock_irq
 void __lockfunc _spin_unlock_irq(spinlock_t *lock)
 {
-        spin_release(&lock->dep_map, 1, _RET_IP_);
-        _raw_spin_unlock(lock);
-        local_irq_enable();
-        preempt_enable();
+        __spin_unlock_irq(lock);
 }
 EXPORT_SYMBOL(_spin_unlock_irq);
+#endif
 
+#ifndef _spin_unlock_bh
 void __lockfunc _spin_unlock_bh(spinlock_t *lock)
 {
-        spin_release(&lock->dep_map, 1, _RET_IP_);
-        _raw_spin_unlock(lock);
-        preempt_enable_no_resched();
-        local_bh_enable_ip((unsigned long)__builtin_return_address(0));
+        __spin_unlock_bh(lock);
 }
 EXPORT_SYMBOL(_spin_unlock_bh);
+#endif
 
+#ifndef _read_unlock_irqrestore
 void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
 {
-        rwlock_release(&lock->dep_map, 1, _RET_IP_);
-        _raw_read_unlock(lock);
-        local_irq_restore(flags);
-        preempt_enable();
+        __read_unlock_irqrestore(lock, flags);
 }
 EXPORT_SYMBOL(_read_unlock_irqrestore);
+#endif
 
+#ifndef _read_unlock_irq
 void __lockfunc _read_unlock_irq(rwlock_t *lock)
 {
-        rwlock_release(&lock->dep_map, 1, _RET_IP_);
-        _raw_read_unlock(lock);
-        local_irq_enable();
-        preempt_enable();
+        __read_unlock_irq(lock);
 }
 EXPORT_SYMBOL(_read_unlock_irq);
+#endif
 
+#ifndef _read_unlock_bh
 void __lockfunc _read_unlock_bh(rwlock_t *lock)
 {
-        rwlock_release(&lock->dep_map, 1, _RET_IP_);
-        _raw_read_unlock(lock);
-        preempt_enable_no_resched();
-        local_bh_enable_ip((unsigned long)__builtin_return_address(0));
+        __read_unlock_bh(lock);
 }
 EXPORT_SYMBOL(_read_unlock_bh);
+#endif
 
+#ifndef _write_unlock_irqrestore
 void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
 {
-        rwlock_release(&lock->dep_map, 1, _RET_IP_);
-        _raw_write_unlock(lock);
-        local_irq_restore(flags);
-        preempt_enable();
+        __write_unlock_irqrestore(lock, flags);
 }
 EXPORT_SYMBOL(_write_unlock_irqrestore);
+#endif
 
+#ifndef _write_unlock_irq
 void __lockfunc _write_unlock_irq(rwlock_t *lock)
 {
-        rwlock_release(&lock->dep_map, 1, _RET_IP_);
-        _raw_write_unlock(lock);
-        local_irq_enable();
-        preempt_enable();
+        __write_unlock_irq(lock);
 }
 EXPORT_SYMBOL(_write_unlock_irq);
+#endif
 
+#ifndef _write_unlock_bh
 void __lockfunc _write_unlock_bh(rwlock_t *lock)
 {
-        rwlock_release(&lock->dep_map, 1, _RET_IP_);
-        _raw_write_unlock(lock);
-        preempt_enable_no_resched();
-        local_bh_enable_ip((unsigned long)__builtin_return_address(0));
+        __write_unlock_bh(lock);
 }
 EXPORT_SYMBOL(_write_unlock_bh);
+#endif
 
+#ifndef _spin_trylock_bh
 int __lockfunc _spin_trylock_bh(spinlock_t *lock)
 {
-        local_bh_disable();
-        preempt_disable();
-        if (_raw_spin_trylock(lock)) {
-                spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
-                return 1;
-        }
-
-        preempt_enable_no_resched();
-        local_bh_enable_ip((unsigned long)__builtin_return_address(0));
-        return 0;
+        return __spin_trylock_bh(lock);
 }
 EXPORT_SYMBOL(_spin_trylock_bh);
+#endif
 
 notrace int in_lock_functions(unsigned long addr)
 {
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index 58be76017fd0..1a631ba684a4 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -49,7 +49,6 @@
 #include <linux/acpi.h>
 #include <linux/reboot.h>
 #include <linux/ftrace.h>
-#include <linux/security.h>
 #include <linux/slow-work.h>
 #include <linux/perf_counter.h>
 
@@ -92,6 +91,9 @@ extern int sysctl_nr_trim_pages;
 #ifdef CONFIG_RCU_TORTURE_TEST
 extern int rcutorture_runnable;
 #endif /* #ifdef CONFIG_RCU_TORTURE_TEST */
+#ifdef CONFIG_BLOCK
+extern int blk_iopoll_enabled;
+#endif
 
 /* Constants used for minimum and  maximum */
 #ifdef CONFIG_DETECT_SOFTLOCKUP
@@ -246,6 +248,14 @@ static int max_wakeup_granularity_ns = NSEC_PER_SEC;	/* 1 second */
 #endif
 
 static struct ctl_table kern_table[] = {
+        {
+                .ctl_name       = CTL_UNNUMBERED,
+                .procname       = "sched_child_runs_first",
+                .data           = &sysctl_sched_child_runs_first,
+                .maxlen         = sizeof(unsigned int),
+                .mode           = 0644,
+                .proc_handler   = &proc_dointvec,
+        },
 #ifdef CONFIG_SCHED_DEBUG
         {
                 .ctl_name       = CTL_UNNUMBERED,
@@ -300,14 +310,6 @@ static struct ctl_table kern_table[] = {
         },
         {
                 .ctl_name       = CTL_UNNUMBERED,
-                .procname       = "sched_child_runs_first",
-                .data           = &sysctl_sched_child_runs_first,
-                .maxlen         = sizeof(unsigned int),
-                .mode           = 0644,
-                .proc_handler   = &proc_dointvec,
-        },
-        {
-                .ctl_name       = CTL_UNNUMBERED,
                 .procname       = "sched_features",
                 .data           = &sysctl_sched_features,
                 .maxlen         = sizeof(unsigned int),
@@ -332,6 +334,14 @@ static struct ctl_table kern_table[] = {
         },
         {
                 .ctl_name       = CTL_UNNUMBERED,
+                .procname       = "sched_time_avg",
+                .data           = &sysctl_sched_time_avg,
+                .maxlen         = sizeof(unsigned int),
+                .mode           = 0644,
+                .proc_handler   = &proc_dointvec,
+        },
+        {
+                .ctl_name       = CTL_UNNUMBERED,
                 .procname       = "timer_migration",
                 .data           = &sysctl_timer_migration,
                 .maxlen         = sizeof(unsigned int),
@@ -990,7 +1000,16 @@ static struct ctl_table kern_table[] = {
                 .proc_handler   = &proc_dointvec,
         },
 #endif
-
+#ifdef CONFIG_BLOCK
+        {
+                .ctl_name       = CTL_UNNUMBERED,
+                .procname       = "blk_iopoll",
+                .data           = &blk_iopoll_enabled,
+                .maxlen         = sizeof(int),
+                .mode           = 0644,
+                .proc_handler   = &proc_dointvec,
+        },
+#endif
 /*
  * NOTE: do not add new entries to this table unless you have read
  * Documentation/sysctl/ctl_unnumbered.txt
diff --git a/kernel/taskstats.c b/kernel/taskstats.c
index 888adbcca30c..ea8384d3caa7 100644
--- a/kernel/taskstats.c
+++ b/kernel/taskstats.c
@@ -108,7 +108,7 @@ static int prepare_reply(struct genl_info *info, u8 cmd, struct sk_buff **skbp,
 /*
  * Send taskstats data in @skb to listener with nl_pid @pid
  */
-static int send_reply(struct sk_buff *skb, pid_t pid)
+static int send_reply(struct sk_buff *skb, struct genl_info *info)
 {
         struct genlmsghdr *genlhdr = nlmsg_data(nlmsg_hdr(skb));
         void *reply = genlmsg_data(genlhdr);
@@ -120,7 +120,7 @@ static int send_reply(struct sk_buff *skb, pid_t pid)
                 return rc;
         }
 
-        return genlmsg_unicast(skb, pid);
+        return genlmsg_reply(skb, info);
 }
 
 /*
@@ -150,7 +150,7 @@ static void send_cpu_listeners(struct sk_buff *skb,
                         if (!skb_next)
                                 break;
                 }
-                rc = genlmsg_unicast(skb_cur, s->pid);
+                rc = genlmsg_unicast(&init_net, skb_cur, s->pid);
                 if (rc == -ECONNREFUSED) {
                         s->valid = 0;
                         delcount++;
@@ -418,7 +418,7 @@ static int cgroupstats_user_cmd(struct sk_buff *skb, struct genl_info *info)
                 goto err;
         }
 
-        rc = send_reply(rep_skb, info->snd_pid);
+        rc = send_reply(rep_skb, info);
 
 err:
         fput_light(file, fput_needed);
@@ -487,7 +487,7 @@ free_return_rc:
         } else
                 goto err;
 
-        return send_reply(rep_skb, info->snd_pid);
+        return send_reply(rep_skb, info);
 err:
         nlmsg_free(rep_skb);
         return rc;
diff --git a/kernel/timer.c b/kernel/timer.c
index a7f07d5a6241..a3d25f415019 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -1156,8 +1156,7 @@ void update_process_times(int user_tick)
         /* Note: this timer irq context must be accounted for as well. */
         account_process_tick(p, user_tick);
         run_local_timers();
-        if (rcu_pending(cpu))
-                rcu_check_callbacks(cpu, user_tick);
+        rcu_check_callbacks(cpu, user_tick);
         printk_tick();
         scheduler_tick();
         run_posix_cpu_timers(p);
diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c
index 787f0fb0994e..56c260b83a9c 100644
--- a/kernel/trace/trace_events.c
+++ b/kernel/trace/trace_events.c
@@ -1411,7 +1411,7 @@ static __init void event_trace_self_tests(void)
 
 #ifdef CONFIG_FUNCTION_TRACER
 
-static DEFINE_PER_CPU(atomic_t, test_event_disable);
+static DEFINE_PER_CPU(atomic_t, ftrace_test_event_disable);
 
 static void
 function_test_events_call(unsigned long ip, unsigned long parent_ip)
@@ -1428,7 +1428,7 @@ function_test_events_call(unsigned long ip, unsigned long parent_ip)
         pc = preempt_count();
         resched = ftrace_preempt_disable();
         cpu = raw_smp_processor_id();
-        disabled = atomic_inc_return(&per_cpu(test_event_disable, cpu));
+        disabled = atomic_inc_return(&per_cpu(ftrace_test_event_disable, cpu));
 
         if (disabled != 1)
                 goto out;
@@ -1447,7 +1447,7 @@ function_test_events_call(unsigned long ip, unsigned long parent_ip)
         trace_nowake_buffer_unlock_commit(buffer, event, flags, pc);
 
  out:
-        atomic_dec(&per_cpu(test_event_disable, cpu));
+        atomic_dec(&per_cpu(ftrace_test_event_disable, cpu));
         ftrace_preempt_enable(resched);
 }
 
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 0668795d8818..addfe2df93b1 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -317,8 +317,6 @@ static int worker_thread(void *__cwq)
         if (cwq->wq->freezeable)
                 set_freezable();
 
-        set_user_nice(current, -5);
-
         for (;;) {
                 prepare_to_wait(&cwq->more_work, &wait, TASK_INTERRUPTIBLE);
                 if (!freezing(current) &&
@@ -600,7 +598,12 @@ static struct workqueue_struct *keventd_wq __read_mostly;
  * schedule_work - put work task in global workqueue
  * @work: job to be done
  *
- * This puts a job in the kernel-global workqueue.
+ * Returns zero if @work was already on the kernel-global workqueue and
+ * non-zero otherwise.
+ *
+ * This puts a job in the kernel-global workqueue if it was not already
+ * queued and leaves it in the same position on the kernel-global
+ * workqueue otherwise.
  */
 int schedule_work(struct work_struct *work)
 {