13 files changed, 559 insertions, 499 deletions

diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt
index a92c5ebf373e..12783fa833c3 100644
--- a/Documentation/kernel-parameters.txt
+++ b/Documentation/kernel-parameters.txt
@@ -2367,6 +2367,11 @@ bytes respectively. Such letter suffixes can also be entirely omitted.
                         Set maximum number of finished RCU callbacks to process
                         in one batch.
 
+        rcutree.fanout_leaf=    [KNL,BOOT]
+                        Increase the number of CPUs assigned to each
+                        leaf rcu_node structure.  Useful for very large
+                        systems.
+
         rcutree.qhimark=        [KNL,BOOT]
                         Set threshold of queued
                         RCU callbacks over which batch limiting is disabled.
diff --git a/include/linux/init_task.h b/include/linux/init_task.h
index 9e65eff6af3b..8a7476186990 100644
--- a/include/linux/init_task.h
+++ b/include/linux/init_task.h
@@ -168,8 +168,8 @@ extern struct cred init_cred;
         .children       = LIST_HEAD_INIT(tsk.children),                 \
         .sibling        = LIST_HEAD_INIT(tsk.sibling),                  \
         .group_leader   = &tsk,                                         \
-        RCU_INIT_POINTER(.real_cred, &init_cred),                       \
-        RCU_INIT_POINTER(.cred, &init_cred),                            \
+        RCU_POINTER_INITIALIZER(real_cred, &init_cred),                 \
+        RCU_POINTER_INITIALIZER(cred, &init_cred),                      \
         .comm           = INIT_TASK_COMM,                               \
         .thread         = INIT_THREAD,                                  \
         .fs             = &init_fs,                                     \
diff --git a/include/linux/key.h b/include/linux/key.h
index 4cd22ed627ef..cef3b315ba7c 100644
--- a/include/linux/key.h
+++ b/include/linux/key.h
@@ -303,7 +303,9 @@ static inline bool key_is_instantiated(const struct key *key)
                                    rwsem_is_locked(&((struct key *)(KEY))->sem)))
 
 #define rcu_assign_keypointer(KEY, PAYLOAD)                             \
-        (rcu_assign_pointer((KEY)->payload.rcudata, PAYLOAD))
+do {                                                                    \
+        rcu_assign_pointer((KEY)->payload.rcudata, (PAYLOAD));          \
+} while (0)
 
 #ifdef CONFIG_SYSCTL
 extern ctl_table key_sysctls[];
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index 9cac722b169c..c2c0d86dd3ac 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -147,6 +147,7 @@ extern void synchronize_sched(void);
 
 extern void __rcu_read_lock(void);
 extern void __rcu_read_unlock(void);
+extern void rcu_read_unlock_special(struct task_struct *t);
 void synchronize_rcu(void);
 
 /*
@@ -255,6 +256,10 @@ static inline void destroy_rcu_head_on_stack(struct rcu_head *head)
 }
 #endif  /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
 
+#if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_SMP)
+extern int rcu_is_cpu_idle(void);
+#endif /* #if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_SMP) */
+
 #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU)
 bool rcu_lockdep_current_cpu_online(void);
 #else /* #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU) */
@@ -266,15 +271,6 @@ static inline bool rcu_lockdep_current_cpu_online(void)
 
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 
-#ifdef CONFIG_PROVE_RCU
-extern int rcu_is_cpu_idle(void);
-#else /* !CONFIG_PROVE_RCU */
-static inline int rcu_is_cpu_idle(void)
-{
-        return 0;
-}
-#endif /* else !CONFIG_PROVE_RCU */
-
 static inline void rcu_lock_acquire(struct lockdep_map *map)
 {
         lock_acquire(map, 0, 0, 2, 1, NULL, _THIS_IP_);
@@ -513,10 +509,10 @@ static inline void rcu_preempt_sleep_check(void)
                 (_________p1); \
         })
 #define __rcu_assign_pointer(p, v, space) \
-        ({ \
+        do { \
                 smp_wmb(); \
                 (p) = (typeof(*v) __force space *)(v); \
-        })
+        } while (0)
 
 
 /**
@@ -851,7 +847,7 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
  *
  * Assigns the specified value to the specified RCU-protected
  * pointer, ensuring that any concurrent RCU readers will see
- * any prior initialization.  Returns the value assigned.
+ * any prior initialization.
  *
  * Inserts memory barriers on architectures that require them
  * (which is most of them), and also prevents the compiler from
@@ -903,25 +899,17 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
  * the reader-accessible portions of the linked structure.
  */
 #define RCU_INIT_POINTER(p, v) \
-                p = (typeof(*v) __force __rcu *)(v)
-
-static __always_inline bool __is_kfree_rcu_offset(unsigned long offset)
-{
-        return offset < 4096;
-}
-
-static __always_inline
-void __kfree_rcu(struct rcu_head *head, unsigned long offset)
-{
-        typedef void (*rcu_callback)(struct rcu_head *);
-
-        BUILD_BUG_ON(!__builtin_constant_p(offset));
-
-        /* See the kfree_rcu() header comment. */
-        BUILD_BUG_ON(!__is_kfree_rcu_offset(offset));
+        do { \
+                p = (typeof(*v) __force __rcu *)(v); \
+        } while (0)
 
-        kfree_call_rcu(head, (rcu_callback)offset);
-}
+/**
+ * RCU_POINTER_INITIALIZER() - statically initialize an RCU protected pointer
+ *
+ * GCC-style initialization for an RCU-protected pointer in a structure field.
+ */
+#define RCU_POINTER_INITIALIZER(p, v) \
+                .p = (typeof(*v) __force __rcu *)(v)
 
 /*
  * Does the specified offset indicate that the corresponding rcu_head
@@ -935,7 +923,7 @@ void __kfree_rcu(struct rcu_head *head, unsigned long offset)
 #define __kfree_rcu(head, offset) \
         do { \
                 BUILD_BUG_ON(!__is_kfree_rcu_offset(offset)); \
-                call_rcu(head, (void (*)(struct rcu_head *))(unsigned long)(offset)); \
+                kfree_call_rcu(head, (void (*)(struct rcu_head *))(unsigned long)(offset)); \
         } while (0)
 
 /**
diff --git a/include/trace/events/rcu.h b/include/trace/events/rcu.h
index d274734b2aa4..5bde94d8585b 100644
--- a/include/trace/events/rcu.h
+++ b/include/trace/events/rcu.h
@@ -541,6 +541,50 @@ TRACE_EVENT(rcu_torture_read,
                   __entry->rcutorturename, __entry->rhp)
 );
 
+/*
+ * Tracepoint for _rcu_barrier() execution.  The string "s" describes
+ * the _rcu_barrier phase:
+ *      "Begin": rcu_barrier_callback() started.
+ *      "Check": rcu_barrier_callback() checking for piggybacking.
+ *      "EarlyExit": rcu_barrier_callback() piggybacked, thus early exit.
+ *      "Inc1": rcu_barrier_callback() piggyback check counter incremented.
+ *      "Offline": rcu_barrier_callback() found offline CPU
+ *      "OnlineQ": rcu_barrier_callback() found online CPU with callbacks.
+ *      "OnlineNQ": rcu_barrier_callback() found online CPU, no callbacks.
+ *      "IRQ": An rcu_barrier_callback() callback posted on remote CPU.
+ *      "CB": An rcu_barrier_callback() invoked a callback, not the last.
+ *      "LastCB": An rcu_barrier_callback() invoked the last callback.
+ *      "Inc2": rcu_barrier_callback() piggyback check counter incremented.
+ * The "cpu" argument is the CPU or -1 if meaningless, the "cnt" argument
+ * is the count of remaining callbacks, and "done" is the piggybacking count.
+ */
+TRACE_EVENT(rcu_barrier,
+
+        TP_PROTO(char *rcuname, char *s, int cpu, int cnt, unsigned long done),
+
+        TP_ARGS(rcuname, s, cpu, cnt, done),
+
+        TP_STRUCT__entry(
+                __field(char *, rcuname)
+                __field(char *, s)
+                __field(int, cpu)
+                __field(int, cnt)
+                __field(unsigned long, done)
+        ),
+
+        TP_fast_assign(
+                __entry->rcuname = rcuname;
+                __entry->s = s;
+                __entry->cpu = cpu;
+                __entry->cnt = cnt;
+                __entry->done = done;
+        ),
+
+        TP_printk("%s %s cpu %d remaining %d # %lu",
+                  __entry->rcuname, __entry->s, __entry->cpu, __entry->cnt,
+                  __entry->done)
+);
+
 #else /* #ifdef CONFIG_RCU_TRACE */
 
 #define trace_rcu_grace_period(rcuname, gpnum, gpevent) do { } while (0)
@@ -564,6 +608,7 @@ TRACE_EVENT(rcu_torture_read,
 #define trace_rcu_batch_end(rcuname, callbacks_invoked, cb, nr, iit, risk) \
         do { } while (0)
 #define trace_rcu_torture_read(rcutorturename, rhp) do { } while (0)
+#define trace_rcu_barrier(name, s, cpu, cnt, done) do { } while (0)
 
 #endif /* #else #ifdef CONFIG_RCU_TRACE */
 
diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c
index 95cba41ce1e9..4e6a61b15e86 100644
--- a/kernel/rcupdate.c
+++ b/kernel/rcupdate.c
@@ -54,6 +54,50 @@
 #ifdef CONFIG_PREEMPT_RCU
 
 /*
+ * Preemptible RCU implementation for rcu_read_lock().
+ * Just increment ->rcu_read_lock_nesting, shared state will be updated
+ * if we block.
+ */
+void __rcu_read_lock(void)
+{
+        current->rcu_read_lock_nesting++;
+        barrier();  /* critical section after entry code. */
+}
+EXPORT_SYMBOL_GPL(__rcu_read_lock);
+
+/*
+ * Preemptible RCU implementation for rcu_read_unlock().
+ * Decrement ->rcu_read_lock_nesting.  If the result is zero (outermost
+ * rcu_read_unlock()) and ->rcu_read_unlock_special is non-zero, then
+ * invoke rcu_read_unlock_special() to clean up after a context switch
+ * in an RCU read-side critical section and other special cases.
+ */
+void __rcu_read_unlock(void)
+{
+        struct task_struct *t = current;
+
+        if (t->rcu_read_lock_nesting != 1) {
+                --t->rcu_read_lock_nesting;
+        } else {
+                barrier();  /* critical section before exit code. */
+                t->rcu_read_lock_nesting = INT_MIN;
+                barrier();  /* assign before ->rcu_read_unlock_special load */
+                if (unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
+                        rcu_read_unlock_special(t);
+                barrier();  /* ->rcu_read_unlock_special load before assign */
+                t->rcu_read_lock_nesting = 0;
+        }
+#ifdef CONFIG_PROVE_LOCKING
+        {
+                int rrln = ACCESS_ONCE(t->rcu_read_lock_nesting);
+
+                WARN_ON_ONCE(rrln < 0 && rrln > INT_MIN / 2);
+        }
+#endif /* #ifdef CONFIG_PROVE_LOCKING */
+}
+EXPORT_SYMBOL_GPL(__rcu_read_unlock);
+
+/*
  * Check for a task exiting while in a preemptible-RCU read-side
  * critical section, clean up if so.  No need to issue warnings,
  * as debug_check_no_locks_held() already does this if lockdep
diff --git a/kernel/rcutiny.c b/kernel/rcutiny.c
index 37a5444204d2..547b1fe5b052 100644
--- a/kernel/rcutiny.c
+++ b/kernel/rcutiny.c
@@ -172,7 +172,7 @@ void rcu_irq_enter(void)
         local_irq_restore(flags);
 }
 
-#ifdef CONFIG_PROVE_RCU
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
 
 /*
  * Test whether RCU thinks that the current CPU is idle.
@@ -183,7 +183,7 @@ int rcu_is_cpu_idle(void)
 }
 EXPORT_SYMBOL(rcu_is_cpu_idle);
 
-#endif /* #ifdef CONFIG_PROVE_RCU */
+#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
 
 /*
  * Test whether the current CPU was interrupted from idle.  Nested
diff --git a/kernel/rcutiny_plugin.h b/kernel/rcutiny_plugin.h
index fc31a2d65100..116725b5edfb 100644
--- a/kernel/rcutiny_plugin.h
+++ b/kernel/rcutiny_plugin.h
@@ -132,7 +132,6 @@ static struct rcu_preempt_ctrlblk rcu_preempt_ctrlblk = {
         RCU_TRACE(.rcb.name = "rcu_preempt")
 };
 
-static void rcu_read_unlock_special(struct task_struct *t);
 static int rcu_preempted_readers_exp(void);
 static void rcu_report_exp_done(void);
 
@@ -527,23 +526,11 @@ void rcu_preempt_note_context_switch(void)
 }
 
 /*
- * Tiny-preemptible RCU implementation for rcu_read_lock().
- * Just increment ->rcu_read_lock_nesting, shared state will be updated
- * if we block.
- */
-void __rcu_read_lock(void)
-{
-        current->rcu_read_lock_nesting++;
-        barrier();  /* needed if we ever invoke rcu_read_lock in rcutiny.c */
-}
-EXPORT_SYMBOL_GPL(__rcu_read_lock);
-
-/*
  * Handle special cases during rcu_read_unlock(), such as needing to
  * notify RCU core processing or task having blocked during the RCU
  * read-side critical section.
  */
-static noinline void rcu_read_unlock_special(struct task_struct *t)
+void rcu_read_unlock_special(struct task_struct *t)
 {
         int empty;
         int empty_exp;
@@ -627,38 +614,6 @@ static noinline void rcu_read_unlock_special(struct task_struct *t)
 }
 
 /*
- * Tiny-preemptible RCU implementation for rcu_read_unlock().
- * Decrement ->rcu_read_lock_nesting.  If the result is zero (outermost
- * rcu_read_unlock()) and ->rcu_read_unlock_special is non-zero, then
- * invoke rcu_read_unlock_special() to clean up after a context switch
- * in an RCU read-side critical section and other special cases.
- */
-void __rcu_read_unlock(void)
-{
-        struct task_struct *t = current;
-
-        barrier();  /* needed if we ever invoke rcu_read_unlock in rcutiny.c */
-        if (t->rcu_read_lock_nesting != 1)
-                --t->rcu_read_lock_nesting;
-        else {
-                t->rcu_read_lock_nesting = INT_MIN;
-                barrier();  /* assign before ->rcu_read_unlock_special load */
-                if (unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
-                        rcu_read_unlock_special(t);
-                barrier();  /* ->rcu_read_unlock_special load before assign */
-                t->rcu_read_lock_nesting = 0;
-        }
-#ifdef CONFIG_PROVE_LOCKING
-        {
-                int rrln = ACCESS_ONCE(t->rcu_read_lock_nesting);
-
-                WARN_ON_ONCE(rrln < 0 && rrln > INT_MIN / 2);
-        }
-#endif /* #ifdef CONFIG_PROVE_LOCKING */
-}
-EXPORT_SYMBOL_GPL(__rcu_read_unlock);
-
-/*
  * Check for a quiescent state from the current CPU.  When a task blocks,
  * the task is recorded in the rcu_preempt_ctrlblk structure, which is
  * checked elsewhere.  This is called from the scheduling-clock interrupt.
@@ -846,8 +801,6 @@ EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
  */
 int rcu_preempt_needs_cpu(void)
 {
-        if (!rcu_preempt_running_reader())
-                rcu_preempt_cpu_qs();
         return rcu_preempt_ctrlblk.rcb.rcucblist != NULL;
 }
 
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index 4b97bba7396e..117218a43724 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -60,36 +60,44 @@
 
 /* Data structures. */
 
-static struct lock_class_key rcu_node_class[NUM_RCU_LVLS];
-
-#define RCU_STATE_INITIALIZER(structname) { \
-        .level = { &structname##_state.node[0] }, \
-        .levelcnt = { \
-                NUM_RCU_LVL_0,  /* root of hierarchy. */ \
-                NUM_RCU_LVL_1, \
-                NUM_RCU_LVL_2, \
-                NUM_RCU_LVL_3, \
-                NUM_RCU_LVL_4, /* == MAX_RCU_LVLS */ \
-        }, \
+static struct lock_class_key rcu_node_class[RCU_NUM_LVLS];
+
+#define RCU_STATE_INITIALIZER(sname, cr) { \
+        .level = { &sname##_state.node[0] }, \
+        .call = cr, \
         .fqs_state = RCU_GP_IDLE, \
         .gpnum = -300, \
         .completed = -300, \
-        .onofflock = __RAW_SPIN_LOCK_UNLOCKED(&structname##_state.onofflock), \
-        .orphan_nxttail = &structname##_state.orphan_nxtlist, \
-        .orphan_donetail = &structname##_state.orphan_donelist, \
-        .fqslock = __RAW_SPIN_LOCK_UNLOCKED(&structname##_state.fqslock), \
-        .n_force_qs = 0, \
-        .n_force_qs_ngp = 0, \
-        .name = #structname, \
+        .onofflock = __RAW_SPIN_LOCK_UNLOCKED(&sname##_state.onofflock), \
+        .orphan_nxttail = &sname##_state.orphan_nxtlist, \
+        .orphan_donetail = &sname##_state.orphan_donelist, \
+        .barrier_mutex = __MUTEX_INITIALIZER(sname##_state.barrier_mutex), \
+        .fqslock = __RAW_SPIN_LOCK_UNLOCKED(&sname##_state.fqslock), \
+        .name = #sname, \
 }
 
-struct rcu_state rcu_sched_state = RCU_STATE_INITIALIZER(rcu_sched);
+struct rcu_state rcu_sched_state =
+        RCU_STATE_INITIALIZER(rcu_sched, call_rcu_sched);
 DEFINE_PER_CPU(struct rcu_data, rcu_sched_data);
 
-struct rcu_state rcu_bh_state = RCU_STATE_INITIALIZER(rcu_bh);
+struct rcu_state rcu_bh_state = RCU_STATE_INITIALIZER(rcu_bh, call_rcu_bh);
 DEFINE_PER_CPU(struct rcu_data, rcu_bh_data);
 
 static struct rcu_state *rcu_state;
+LIST_HEAD(rcu_struct_flavors);
+
+/* Increase (but not decrease) the CONFIG_RCU_FANOUT_LEAF at boot time. */
+static int rcu_fanout_leaf = CONFIG_RCU_FANOUT_LEAF;
+module_param(rcu_fanout_leaf, int, 0);
+int rcu_num_lvls __read_mostly = RCU_NUM_LVLS;
+static int num_rcu_lvl[] = {  /* Number of rcu_nodes at specified level. */
+        NUM_RCU_LVL_0,
+        NUM_RCU_LVL_1,
+        NUM_RCU_LVL_2,
+        NUM_RCU_LVL_3,
+        NUM_RCU_LVL_4,
+};
+int rcu_num_nodes __read_mostly = NUM_RCU_NODES; /* Total # rcu_nodes in use. */
 
 /*
  * The rcu_scheduler_active variable transitions from zero to one just
@@ -147,13 +155,6 @@ static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp);
 unsigned long rcutorture_testseq;
 unsigned long rcutorture_vernum;
 
-/* State information for rcu_barrier() and friends. */
-
-static DEFINE_PER_CPU(struct rcu_head, rcu_barrier_head) = {NULL};
-static atomic_t rcu_barrier_cpu_count;
-static DEFINE_MUTEX(rcu_barrier_mutex);
-static struct completion rcu_barrier_completion;
-
 /*
  * Return true if an RCU grace period is in progress.  The ACCESS_ONCE()s
  * permit this function to be invoked without holding the root rcu_node
@@ -358,7 +359,7 @@ static void rcu_idle_enter_common(struct rcu_dynticks *rdtp, long long oldval)
                 struct task_struct *idle = idle_task(smp_processor_id());
 
                 trace_rcu_dyntick("Error on entry: not idle task", oldval, 0);
-                ftrace_dump(DUMP_ALL);
+                ftrace_dump(DUMP_ORIG);
                 WARN_ONCE(1, "Current pid: %d comm: %s / Idle pid: %d comm: %s",
                           current->pid, current->comm,
                           idle->pid, idle->comm); /* must be idle task! */
@@ -468,7 +469,7 @@ static void rcu_idle_exit_common(struct rcu_dynticks *rdtp, long long oldval)
 
                 trace_rcu_dyntick("Error on exit: not idle task",
                                   oldval, rdtp->dynticks_nesting);
-                ftrace_dump(DUMP_ALL);
+                ftrace_dump(DUMP_ORIG);
                 WARN_ONCE(1, "Current pid: %d comm: %s / Idle pid: %d comm: %s",
                           current->pid, current->comm,
                           idle->pid, idle->comm); /* must be idle task! */
@@ -585,8 +586,6 @@ void rcu_nmi_exit(void)
         WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
 }
 
-#ifdef CONFIG_PROVE_RCU
-
 /**
  * rcu_is_cpu_idle - see if RCU thinks that the current CPU is idle
  *
@@ -604,7 +603,7 @@ int rcu_is_cpu_idle(void)
 }
 EXPORT_SYMBOL(rcu_is_cpu_idle);
 
-#ifdef CONFIG_HOTPLUG_CPU
+#if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU)
 
 /*
  * Is the current CPU online?  Disable preemption to avoid false positives
@@ -645,9 +644,7 @@ bool rcu_lockdep_current_cpu_online(void)
 }
 EXPORT_SYMBOL_GPL(rcu_lockdep_current_cpu_online);
 
-#endif /* #ifdef CONFIG_HOTPLUG_CPU */
-
-#endif /* #ifdef CONFIG_PROVE_RCU */
+#endif /* #if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU) */
 
 /**
  * rcu_is_cpu_rrupt_from_idle - see if idle or immediately interrupted from idle
@@ -733,7 +730,7 @@ static void print_other_cpu_stall(struct rcu_state *rsp)
         int cpu;
         long delta;
         unsigned long flags;
-        int ndetected;
+        int ndetected = 0;
         struct rcu_node *rnp = rcu_get_root(rsp);
 
         /* Only let one CPU complain about others per time interval. */
@@ -774,7 +771,7 @@ static void print_other_cpu_stall(struct rcu_state *rsp)
          */
         rnp = rcu_get_root(rsp);
         raw_spin_lock_irqsave(&rnp->lock, flags);
-        ndetected = rcu_print_task_stall(rnp);
+        ndetected += rcu_print_task_stall(rnp);
         raw_spin_unlock_irqrestore(&rnp->lock, flags);
 
         print_cpu_stall_info_end();
@@ -860,9 +857,10 @@ static int rcu_panic(struct notifier_block *this, unsigned long ev, void *ptr)
  */
 void rcu_cpu_stall_reset(void)
 {
-        rcu_sched_state.jiffies_stall = jiffies + ULONG_MAX / 2;
-        rcu_bh_state.jiffies_stall = jiffies + ULONG_MAX / 2;
-        rcu_preempt_stall_reset();
+        struct rcu_state *rsp;
+
+        for_each_rcu_flavor(rsp)
+                rsp->jiffies_stall = jiffies + ULONG_MAX / 2;
 }
 
 static struct notifier_block rcu_panic_block = {
@@ -937,6 +935,18 @@ check_for_new_grace_period(struct rcu_state *rsp, struct rcu_data *rdp)
 }
 
 /*
+ * Initialize the specified rcu_data structure's callback list to empty.
+ */
+static void init_callback_list(struct rcu_data *rdp)
+{
+        int i;
+
+        rdp->nxtlist = NULL;
+        for (i = 0; i < RCU_NEXT_SIZE; i++)
+                rdp->nxttail[i] = &rdp->nxtlist;
+}
+
+/*
  * Advance this CPU's callbacks, but only if the current grace period
  * has ended.  This may be called only from the CPU to whom the rdp
  * belongs.  In addition, the corresponding leaf rcu_node structure's
@@ -1328,8 +1338,6 @@ static void
 rcu_send_cbs_to_orphanage(int cpu, struct rcu_state *rsp,
                           struct rcu_node *rnp, struct rcu_data *rdp)
 {
-        int i;
-
         /*
          * Orphan the callbacks.  First adjust the counts.  This is safe
          * because ->onofflock excludes _rcu_barrier()'s adoption of
@@ -1340,7 +1348,7 @@ rcu_send_cbs_to_orphanage(int cpu, struct rcu_state *rsp,
                 rsp->qlen += rdp->qlen;
                 rdp->n_cbs_orphaned += rdp->qlen;
                 rdp->qlen_lazy = 0;
-                rdp->qlen = 0;
+                ACCESS_ONCE(rdp->qlen) = 0;
         }
 
         /*
@@ -1369,9 +1377,7 @@ rcu_send_cbs_to_orphanage(int cpu, struct rcu_state *rsp,
         }
 
         /* Finally, initialize the rcu_data structure's list to empty.  */
-        rdp->nxtlist = NULL;
-        for (i = 0; i < RCU_NEXT_SIZE; i++)
-                rdp->nxttail[i] = &rdp->nxtlist;
+        init_callback_list(rdp);
 }
 
 /*
@@ -1505,6 +1511,9 @@ static void rcu_cleanup_dead_cpu(int cpu, struct rcu_state *rsp)
                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
         if (need_report & RCU_OFL_TASKS_EXP_GP)
                 rcu_report_exp_rnp(rsp, rnp, true);
+        WARN_ONCE(rdp->qlen != 0 || rdp->nxtlist != NULL,
+                  "rcu_cleanup_dead_cpu: Callbacks on offline CPU %d: qlen=%lu, nxtlist=%p\n",
+                  cpu, rdp->qlen, rdp->nxtlist);
 }
 
 #else /* #ifdef CONFIG_HOTPLUG_CPU */
@@ -1592,7 +1601,7 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp)
         }
         smp_mb(); /* List handling before counting for rcu_barrier(). */
         rdp->qlen_lazy -= count_lazy;
-        rdp->qlen -= count;
+        ACCESS_ONCE(rdp->qlen) -= count;
         rdp->n_cbs_invoked += count;
 
         /* Reinstate batch limit if we have worked down the excess. */
@@ -1605,6 +1614,7 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp)
                 rdp->n_force_qs_snap = rsp->n_force_qs;
         } else if (rdp->qlen < rdp->qlen_last_fqs_check - qhimark)
                 rdp->qlen_last_fqs_check = rdp->qlen;
+        WARN_ON_ONCE((rdp->nxtlist == NULL) != (rdp->qlen == 0));
 
         local_irq_restore(flags);
 
@@ -1745,8 +1755,6 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed)
                 break; /* grace period idle or initializing, ignore. */
 
         case RCU_SAVE_DYNTICK:
-                if (RCU_SIGNAL_INIT != RCU_SAVE_DYNTICK)
-                        break; /* So gcc recognizes the dead code. */
 
                 raw_spin_unlock(&rnp->lock);  /* irqs remain disabled */
 
@@ -1788,9 +1796,10 @@ unlock_fqs_ret:
  * whom the rdp belongs.
  */
 static void
-__rcu_process_callbacks(struct rcu_state *rsp, struct rcu_data *rdp)
+__rcu_process_callbacks(struct rcu_state *rsp)
 {
         unsigned long flags;
+        struct rcu_data *rdp = __this_cpu_ptr(rsp->rda);
 
         WARN_ON_ONCE(rdp->beenonline == 0);
 
@@ -1826,11 +1835,11 @@ __rcu_process_callbacks(struct rcu_state *rsp, struct rcu_data *rdp)
  */
 static void rcu_process_callbacks(struct softirq_action *unused)
 {
+        struct rcu_state *rsp;
+
         trace_rcu_utilization("Start RCU core");
-        __rcu_process_callbacks(&rcu_sched_state,
-                                &__get_cpu_var(rcu_sched_data));
-        __rcu_process_callbacks(&rcu_bh_state, &__get_cpu_var(rcu_bh_data));
-        rcu_preempt_process_callbacks();
+        for_each_rcu_flavor(rsp)
+                __rcu_process_callbacks(rsp);
         trace_rcu_utilization("End RCU core");
 }
 
@@ -1857,6 +1866,56 @@ static void invoke_rcu_core(void)
         raise_softirq(RCU_SOFTIRQ);
 }
 
+/*
+ * Handle any core-RCU processing required by a call_rcu() invocation.
+ */
+static void __call_rcu_core(struct rcu_state *rsp, struct rcu_data *rdp,
+                            struct rcu_head *head, unsigned long flags)
+{
+        /*
+         * If called from an extended quiescent state, invoke the RCU
+         * core in order to force a re-evaluation of RCU's idleness.
+         */
+        if (rcu_is_cpu_idle() && cpu_online(smp_processor_id()))
+                invoke_rcu_core();
+
+        /* If interrupts were disabled or CPU offline, don't invoke RCU core. */
+        if (irqs_disabled_flags(flags) || cpu_is_offline(smp_processor_id()))
+                return;
+
+        /*
+         * Force the grace period if too many callbacks or too long waiting.
+         * Enforce hysteresis, and don't invoke force_quiescent_state()
+         * if some other CPU has recently done so.  Also, don't bother
+         * invoking force_quiescent_state() if the newly enqueued callback
+         * is the only one waiting for a grace period to complete.
+         */
+        if (unlikely(rdp->qlen > rdp->qlen_last_fqs_check + qhimark)) {
+
+                /* Are we ignoring a completed grace period? */
+                rcu_process_gp_end(rsp, rdp);
+                check_for_new_grace_period(rsp, rdp);
+
+                /* Start a new grace period if one not already started. */
+                if (!rcu_gp_in_progress(rsp)) {
+                        unsigned long nestflag;
+                        struct rcu_node *rnp_root = rcu_get_root(rsp);
+
+                        raw_spin_lock_irqsave(&rnp_root->lock, nestflag);
+                        rcu_start_gp(rsp, nestflag);  /* rlses rnp_root->lock */
+                } else {
+                        /* Give the grace period a kick. */
+                        rdp->blimit = LONG_MAX;
+                        if (rsp->n_force_qs == rdp->n_force_qs_snap &&
+                            *rdp->nxttail[RCU_DONE_TAIL] != head)
+                                force_quiescent_state(rsp, 0);
+                        rdp->n_force_qs_snap = rsp->n_force_qs;
+                        rdp->qlen_last_fqs_check = rdp->qlen;
+                }
+        } else if (ULONG_CMP_LT(ACCESS_ONCE(rsp->jiffies_force_qs), jiffies))
+                force_quiescent_state(rsp, 1);
+}
+
 static void
 __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
            struct rcu_state *rsp, bool lazy)
@@ -1881,7 +1940,7 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
         rdp = this_cpu_ptr(rsp->rda);
 
         /* Add the callback to our list. */
-        rdp->qlen++;
+        ACCESS_ONCE(rdp->qlen)++;
         if (lazy)
                 rdp->qlen_lazy++;
         else
@@ -1896,43 +1955,8 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
         else
                 trace_rcu_callback(rsp->name, head, rdp->qlen_lazy, rdp->qlen);
 
-        /* If interrupts were disabled, don't dive into RCU core. */
-        if (irqs_disabled_flags(flags)) {
-                local_irq_restore(flags);
-                return;
-        }
-
-        /*
-         * Force the grace period if too many callbacks or too long waiting.
-         * Enforce hysteresis, and don't invoke force_quiescent_state()
-         * if some other CPU has recently done so.  Also, don't bother
-         * invoking force_quiescent_state() if the newly enqueued callback
-         * is the only one waiting for a grace period to complete.
-         */
-        if (unlikely(rdp->qlen > rdp->qlen_last_fqs_check + qhimark)) {
-
-                /* Are we ignoring a completed grace period? */
-                rcu_process_gp_end(rsp, rdp);
-                check_for_new_grace_period(rsp, rdp);
-
-                /* Start a new grace period if one not already started. */
-                if (!rcu_gp_in_progress(rsp)) {
-                        unsigned long nestflag;
-                        struct rcu_node *rnp_root = rcu_get_root(rsp);
-
-                        raw_spin_lock_irqsave(&rnp_root->lock, nestflag);
-                        rcu_start_gp(rsp, nestflag);  /* rlses rnp_root->lock */
-                } else {
-                        /* Give the grace period a kick. */
-                        rdp->blimit = LONG_MAX;
-                        if (rsp->n_force_qs == rdp->n_force_qs_snap &&
-                            *rdp->nxttail[RCU_DONE_TAIL] != head)
-                                force_quiescent_state(rsp, 0);
-                        rdp->n_force_qs_snap = rsp->n_force_qs;
-                        rdp->qlen_last_fqs_check = rdp->qlen;
-                }
-        } else if (ULONG_CMP_LT(ACCESS_ONCE(rsp->jiffies_force_qs), jiffies))
-                force_quiescent_state(rsp, 1);
+        /* Go handle any RCU core processing required. */
+        __call_rcu_core(rsp, rdp, head, flags);
         local_irq_restore(flags);
 }
 
@@ -1962,28 +1986,16 @@ EXPORT_SYMBOL_GPL(call_rcu_bh);
  * occasionally incorrectly indicate that there are multiple CPUs online
  * when there was in fact only one the whole time, as this just adds
  * some overhead: RCU still operates correctly.
- *
- * Of course, sampling num_online_cpus() with preemption enabled can
- * give erroneous results if there are concurrent CPU-hotplug operations.
- * For example, given a demonic sequence of preemptions in num_online_cpus()
- * and CPU-hotplug operations, there could be two or more CPUs online at
- * all times, but num_online_cpus() might well return one (or even zero).
- *
- * However, all such demonic sequences require at least one CPU-offline
- * operation.  Furthermore, rcu_blocking_is_gp() giving the wrong answer
- * is only a problem if there is an RCU read-side critical section executing
- * throughout.  But RCU-sched and RCU-bh read-side critical sections
- * disable either preemption or bh, which prevents a CPU from going offline.
- * Therefore, the only way that rcu_blocking_is_gp() can incorrectly return
- * that there is only one CPU when in fact there was more than one throughout
- * is when there were no RCU readers in the system.  If there are no
- * RCU readers, the grace period by definition can be of zero length,
- * regardless of the number of online CPUs.
  */
 static inline int rcu_blocking_is_gp(void)
 {
+        int ret;
+
         might_sleep();  /* Check for RCU read-side critical section. */
-        return num_online_cpus() <= 1;
+        preempt_disable();
+        ret = num_online_cpus() <= 1;
+        preempt_enable();
+        return ret;
 }
 
 /**
@@ -2241,9 +2253,12 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp)
  */
 static int rcu_pending(int cpu)
 {
-        return __rcu_pending(&rcu_sched_state, &per_cpu(rcu_sched_data, cpu)) ||
-               __rcu_pending(&rcu_bh_state, &per_cpu(rcu_bh_data, cpu)) ||
-               rcu_preempt_pending(cpu);
+        struct rcu_state *rsp;
+
+        for_each_rcu_flavor(rsp)
+                if (__rcu_pending(rsp, per_cpu_ptr(rsp->rda, cpu)))
+                        return 1;
+        return 0;
 }
 
 /*
@@ -2253,20 +2268,41 @@ static int rcu_pending(int cpu)
  */
 static int rcu_cpu_has_callbacks(int cpu)
 {
+        struct rcu_state *rsp;
+
         /* RCU callbacks either ready or pending? */
-        return per_cpu(rcu_sched_data, cpu).nxtlist ||
-               per_cpu(rcu_bh_data, cpu).nxtlist ||
-               rcu_preempt_cpu_has_callbacks(cpu);
+        for_each_rcu_flavor(rsp)
+                if (per_cpu_ptr(rsp->rda, cpu)->nxtlist)
+                        return 1;
+        return 0;
+}
+
+/*
+ * Helper function for _rcu_barrier() tracing.  If tracing is disabled,
+ * the compiler is expected to optimize this away.
+ */
+static void _rcu_barrier_trace(struct rcu_state *rsp, char *s,
+                               int cpu, unsigned long done)
+{
+        trace_rcu_barrier(rsp->name, s, cpu,
+                          atomic_read(&rsp->barrier_cpu_count), done);
 }
 
 /*
  * RCU callback function for _rcu_barrier().  If we are last, wake
  * up the task executing _rcu_barrier().
  */
-static void rcu_barrier_callback(struct rcu_head *notused)
+static void rcu_barrier_callback(struct rcu_head *rhp)
 {
-        if (atomic_dec_and_test(&rcu_barrier_cpu_count))
-                complete(&rcu_barrier_completion);
+        struct rcu_data *rdp = container_of(rhp, struct rcu_data, barrier_head);
+        struct rcu_state *rsp = rdp->rsp;
+
+        if (atomic_dec_and_test(&rsp->barrier_cpu_count)) {
+                _rcu_barrier_trace(rsp, "LastCB", -1, rsp->n_barrier_done);
+                complete(&rsp->barrier_completion);
+        } else {
+                _rcu_barrier_trace(rsp, "CB", -1, rsp->n_barrier_done);
+        }
 }
 
 /*
@@ -2274,35 +2310,63 @@ static void rcu_barrier_callback(struct rcu_head *notused)
  */
 static void rcu_barrier_func(void *type)
 {
-        int cpu = smp_processor_id();
-        struct rcu_head *head = &per_cpu(rcu_barrier_head, cpu);
-        void (*call_rcu_func)(struct rcu_head *head,
-                              void (*func)(struct rcu_head *head));
+        struct rcu_state *rsp = type;
+        struct rcu_data *rdp = __this_cpu_ptr(rsp->rda);
 
-        atomic_inc(&rcu_barrier_cpu_count);
-        call_rcu_func = type;
-        call_rcu_func(head, rcu_barrier_callback);
+        _rcu_barrier_trace(rsp, "IRQ", -1, rsp->n_barrier_done);
+        atomic_inc(&rsp->barrier_cpu_count);
+        rsp->call(&rdp->barrier_head, rcu_barrier_callback);
 }
 
 /*
  * Orchestrate the specified type of RCU barrier, waiting for all
  * RCU callbacks of the specified type to complete.
  */
-static void _rcu_barrier(struct rcu_state *rsp,
-                         void (*call_rcu_func)(struct rcu_head *head,
-                                               void (*func)(struct rcu_head *head)))
+static void _rcu_barrier(struct rcu_state *rsp)
 {
         int cpu;
         unsigned long flags;
         struct rcu_data *rdp;
-        struct rcu_head rh;
+        struct rcu_data rd;
+        unsigned long snap = ACCESS_ONCE(rsp->n_barrier_done);
+        unsigned long snap_done;
 
-        init_rcu_head_on_stack(&rh);
+        init_rcu_head_on_stack(&rd.barrier_head);
+        _rcu_barrier_trace(rsp, "Begin", -1, snap);
 
         /* Take mutex to serialize concurrent rcu_barrier() requests. */
-        mutex_lock(&rcu_barrier_mutex);
+        mutex_lock(&rsp->barrier_mutex);
+
+        /*
+         * Ensure that all prior references, including to ->n_barrier_done,
+         * are ordered before the _rcu_barrier() machinery.
+         */
+        smp_mb();  /* See above block comment. */
+
+        /*
+         * Recheck ->n_barrier_done to see if others did our work for us.
+         * This means checking ->n_barrier_done for an even-to-odd-to-even
+         * transition.  The "if" expression below therefore rounds the old
+         * value up to the next even number and adds two before comparing.
+         */
+        snap_done = ACCESS_ONCE(rsp->n_barrier_done);
+        _rcu_barrier_trace(rsp, "Check", -1, snap_done);
+        if (ULONG_CMP_GE(snap_done, ((snap + 1) & ~0x1) + 2)) {
+                _rcu_barrier_trace(rsp, "EarlyExit", -1, snap_done);
+                smp_mb(); /* caller's subsequent code after above check. */
+                mutex_unlock(&rsp->barrier_mutex);
+                return;
+        }
 
-        smp_mb();  /* Prevent any prior operations from leaking in. */
+        /*
+         * Increment ->n_barrier_done to avoid duplicate work.  Use
+         * ACCESS_ONCE() to prevent the compiler from speculating
+         * the increment to precede the early-exit check.
+         */
+        ACCESS_ONCE(rsp->n_barrier_done)++;
+        WARN_ON_ONCE((rsp->n_barrier_done & 0x1) != 1);
+        _rcu_barrier_trace(rsp, "Inc1", -1, rsp->n_barrier_done);
+        smp_mb(); /* Order ->n_barrier_done increment with below mechanism. */
 
         /*
          * Initialize the count to one rather than to zero in order to
@@ -2321,8 +2385,8 @@ static void _rcu_barrier(struct rcu_state *rsp,
          * 6.   Both rcu_barrier_callback() callbacks are invoked, awakening
          *      us -- but before CPU 1's orphaned callbacks are invoked!!!
          */
-        init_completion(&rcu_barrier_completion);
-        atomic_set(&rcu_barrier_cpu_count, 1);
+        init_completion(&rsp->barrier_completion);
+        atomic_set(&rsp->barrier_cpu_count, 1);
         raw_spin_lock_irqsave(&rsp->onofflock, flags);
         rsp->rcu_barrier_in_progress = current;
         raw_spin_unlock_irqrestore(&rsp->onofflock, flags);
@@ -2338,14 +2402,19 @@ static void _rcu_barrier(struct rcu_state *rsp,
                 preempt_disable();
                 rdp = per_cpu_ptr(rsp->rda, cpu);
                 if (cpu_is_offline(cpu)) {
+                        _rcu_barrier_trace(rsp, "Offline", cpu,
+                                           rsp->n_barrier_done);
                         preempt_enable();
                         while (cpu_is_offline(cpu) && ACCESS_ONCE(rdp->qlen))
                                 schedule_timeout_interruptible(1);
                 } else if (ACCESS_ONCE(rdp->qlen)) {
-                        smp_call_function_single(cpu, rcu_barrier_func,
-                                                 (void *)call_rcu_func, 1);
+                        _rcu_barrier_trace(rsp, "OnlineQ", cpu,
+                                           rsp->n_barrier_done);
+                        smp_call_function_single(cpu, rcu_barrier_func, rsp, 1);
                         preempt_enable();
                 } else {
+                        _rcu_barrier_trace(rsp, "OnlineNQ", cpu,
+                                           rsp->n_barrier_done);
                         preempt_enable();
                 }
         }
@@ -2362,24 +2431,32 @@ static void _rcu_barrier(struct rcu_state *rsp,
         rcu_adopt_orphan_cbs(rsp);
         rsp->rcu_barrier_in_progress = NULL;
         raw_spin_unlock_irqrestore(&rsp->onofflock, flags);
-        atomic_inc(&rcu_barrier_cpu_count);
+        atomic_inc(&rsp->barrier_cpu_count);
         smp_mb__after_atomic_inc(); /* Ensure atomic_inc() before callback. */
-        call_rcu_func(&rh, rcu_barrier_callback);
+        rd.rsp = rsp;
+        rsp->call(&rd.barrier_head, rcu_barrier_callback);
 
         /*
          * Now that we have an rcu_barrier_callback() callback on each
          * CPU, and thus each counted, remove the initial count.
          */
-        if (atomic_dec_and_test(&rcu_barrier_cpu_count))
-                complete(&rcu_barrier_completion);
+        if (atomic_dec_and_test(&rsp->barrier_cpu_count))
+                complete(&rsp->barrier_completion);
+
+        /* Increment ->n_barrier_done to prevent duplicate work. */
+        smp_mb(); /* Keep increment after above mechanism. */
+        ACCESS_ONCE(rsp->n_barrier_done)++;
+        WARN_ON_ONCE((rsp->n_barrier_done & 0x1) != 0);
+        _rcu_barrier_trace(rsp, "Inc2", -1, rsp->n_barrier_done);
+        smp_mb(); /* Keep increment before caller's subsequent code. */
 
         /* Wait for all rcu_barrier_callback() callbacks to be invoked. */
-        wait_for_completion(&rcu_barrier_completion);
+        wait_for_completion(&rsp->barrier_completion);
 
         /* Other rcu_barrier() invocations can now safely proceed. */
-        mutex_unlock(&rcu_barrier_mutex);
+        mutex_unlock(&rsp->barrier_mutex);
 
-        destroy_rcu_head_on_stack(&rh);
+        destroy_rcu_head_on_stack(&rd.barrier_head);
 }
 
 /**
@@ -2387,7 +2464,7 @@ static void _rcu_barrier(struct rcu_state *rsp,
  */
 void rcu_barrier_bh(void)
 {
-        _rcu_barrier(&rcu_bh_state, call_rcu_bh);
+        _rcu_barrier(&rcu_bh_state);
 }
 EXPORT_SYMBOL_GPL(rcu_barrier_bh);
 
@@ -2396,7 +2473,7 @@ EXPORT_SYMBOL_GPL(rcu_barrier_bh);
  */
 void rcu_barrier_sched(void)
 {
-        _rcu_barrier(&rcu_sched_state, call_rcu_sched);
+        _rcu_barrier(&rcu_sched_state);
 }
 EXPORT_SYMBOL_GPL(rcu_barrier_sched);
 
@@ -2407,18 +2484,15 @@ static void __init
 rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp)
 {
         unsigned long flags;
-        int i;
         struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
         struct rcu_node *rnp = rcu_get_root(rsp);
 
         /* Set up local state, ensuring consistent view of global state. */
         raw_spin_lock_irqsave(&rnp->lock, flags);
         rdp->grpmask = 1UL << (cpu - rdp->mynode->grplo);
-        rdp->nxtlist = NULL;
-        for (i = 0; i < RCU_NEXT_SIZE; i++)
-                rdp->nxttail[i] = &rdp->nxtlist;
+        init_callback_list(rdp);
         rdp->qlen_lazy = 0;
-        rdp->qlen = 0;
+        ACCESS_ONCE(rdp->qlen) = 0;
         rdp->dynticks = &per_cpu(rcu_dynticks, cpu);
         WARN_ON_ONCE(rdp->dynticks->dynticks_nesting != DYNTICK_TASK_EXIT_IDLE);
         WARN_ON_ONCE(atomic_read(&rdp->dynticks->dynticks) != 1);
@@ -2492,9 +2566,11 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp, int preemptible)
 
 static void __cpuinit rcu_prepare_cpu(int cpu)
 {
-        rcu_init_percpu_data(cpu, &rcu_sched_state, 0);
-        rcu_init_percpu_data(cpu, &rcu_bh_state, 0);
-        rcu_preempt_init_percpu_data(cpu);
+        struct rcu_state *rsp;
+
+        for_each_rcu_flavor(rsp)
+                rcu_init_percpu_data(cpu, rsp,
+                                     strcmp(rsp->name, "rcu_preempt") == 0);
 }
 
 /*
@@ -2506,6 +2582,7 @@ static int __cpuinit rcu_cpu_notify(struct notifier_block *self,
         long cpu = (long)hcpu;
         struct rcu_data *rdp = per_cpu_ptr(rcu_state->rda, cpu);
         struct rcu_node *rnp = rdp->mynode;
+        struct rcu_state *rsp;
 
         trace_rcu_utilization("Start CPU hotplug");
         switch (action) {
@@ -2530,18 +2607,16 @@ static int __cpuinit rcu_cpu_notify(struct notifier_block *self,
                  * touch any data without introducing corruption. We send the
                  * dying CPU's callbacks to an arbitrarily chosen online CPU.
                  */
-                rcu_cleanup_dying_cpu(&rcu_bh_state);
-                rcu_cleanup_dying_cpu(&rcu_sched_state);
-                rcu_preempt_cleanup_dying_cpu();
+                for_each_rcu_flavor(rsp)
+                        rcu_cleanup_dying_cpu(rsp);
                 rcu_cleanup_after_idle(cpu);
                 break;
         case CPU_DEAD:
         case CPU_DEAD_FROZEN:
         case CPU_UP_CANCELED:
         case CPU_UP_CANCELED_FROZEN:
-                rcu_cleanup_dead_cpu(cpu, &rcu_bh_state);
-                rcu_cleanup_dead_cpu(cpu, &rcu_sched_state);
-                rcu_preempt_cleanup_dead_cpu(cpu);
+                for_each_rcu_flavor(rsp)
+                        rcu_cleanup_dead_cpu(cpu, rsp);
                 break;
         default:
                 break;
@@ -2574,9 +2649,9 @@ static void __init rcu_init_levelspread(struct rcu_state *rsp)
 {
         int i;
 
-        for (i = NUM_RCU_LVLS - 1; i > 0; i--)
+        for (i = rcu_num_lvls - 1; i > 0; i--)
                 rsp->levelspread[i] = CONFIG_RCU_FANOUT;
-        rsp->levelspread[0] = CONFIG_RCU_FANOUT_LEAF;
+        rsp->levelspread[0] = rcu_fanout_leaf;
 }
 #else /* #ifdef CONFIG_RCU_FANOUT_EXACT */
 static void __init rcu_init_levelspread(struct rcu_state *rsp)
@@ -2586,7 +2661,7 @@ static void __init rcu_init_levelspread(struct rcu_state *rsp)
         int i;
 
         cprv = NR_CPUS;
-        for (i = NUM_RCU_LVLS - 1; i >= 0; i--) {
+        for (i = rcu_num_lvls - 1; i >= 0; i--) {
                 ccur = rsp->levelcnt[i];
                 rsp->levelspread[i] = (cprv + ccur - 1) / ccur;
                 cprv = ccur;
@@ -2613,13 +2688,15 @@ static void __init rcu_init_one(struct rcu_state *rsp,
 
         /* Initialize the level-tracking arrays. */
 
-        for (i = 1; i < NUM_RCU_LVLS; i++)
+        for (i = 0; i < rcu_num_lvls; i++)
+                rsp->levelcnt[i] = num_rcu_lvl[i];
+        for (i = 1; i < rcu_num_lvls; i++)
                 rsp->level[i] = rsp->level[i - 1] + rsp->levelcnt[i - 1];
         rcu_init_levelspread(rsp);
 
         /* Initialize the elements themselves, starting from the leaves. */
 
-        for (i = NUM_RCU_LVLS - 1; i >= 0; i--) {
+        for (i = rcu_num_lvls - 1; i >= 0; i--) {
                 cpustride *= rsp->levelspread[i];
                 rnp = rsp->level[i];
                 for (j = 0; j < rsp->levelcnt[i]; j++, rnp++) {
@@ -2649,13 +2726,74 @@ static void __init rcu_init_one(struct rcu_state *rsp,
         }
 
         rsp->rda = rda;
-        rnp = rsp->level[NUM_RCU_LVLS - 1];
+        rnp = rsp->level[rcu_num_lvls - 1];
         for_each_possible_cpu(i) {
                 while (i > rnp->grphi)
                         rnp++;
                 per_cpu_ptr(rsp->rda, i)->mynode = rnp;
                 rcu_boot_init_percpu_data(i, rsp);
         }
+        list_add(&rsp->flavors, &rcu_struct_flavors);
+}
+
+/*
+ * Compute the rcu_node tree geometry from kernel parameters.  This cannot
+ * replace the definitions in rcutree.h because those are needed to size
+ * the ->node array in the rcu_state structure.
+ */
+static void __init rcu_init_geometry(void)
+{
+        int i;
+        int j;
+        int n = nr_cpu_ids;
+        int rcu_capacity[MAX_RCU_LVLS + 1];
+
+        /* If the compile-time values are accurate, just leave. */
+        if (rcu_fanout_leaf == CONFIG_RCU_FANOUT_LEAF)
+                return;
+
+        /*
+         * Compute number of nodes that can be handled an rcu_node tree
+         * with the given number of levels.  Setting rcu_capacity[0] makes
+         * some of the arithmetic easier.
+         */
+        rcu_capacity[0] = 1;
+        rcu_capacity[1] = rcu_fanout_leaf;
+        for (i = 2; i <= MAX_RCU_LVLS; i++)
+                rcu_capacity[i] = rcu_capacity[i - 1] * CONFIG_RCU_FANOUT;
+
+        /*
+         * The boot-time rcu_fanout_leaf parameter is only permitted
+         * to increase the leaf-level fanout, not decrease it.  Of course,
+         * the leaf-level fanout cannot exceed the number of bits in
+         * the rcu_node masks.  Finally, the tree must be able to accommodate
+         * the configured number of CPUs.  Complain and fall back to the
+         * compile-time values if these limits are exceeded.
+         */
+        if (rcu_fanout_leaf < CONFIG_RCU_FANOUT_LEAF ||
+            rcu_fanout_leaf > sizeof(unsigned long) * 8 ||
+            n > rcu_capacity[MAX_RCU_LVLS]) {
+                WARN_ON(1);
+                return;
+        }
+
+        /* Calculate the number of rcu_nodes at each level of the tree. */
+        for (i = 1; i <= MAX_RCU_LVLS; i++)
+                if (n <= rcu_capacity[i]) {
+                        for (j = 0; j <= i; j++)
+                                num_rcu_lvl[j] =
+                                        DIV_ROUND_UP(n, rcu_capacity[i - j]);
+                        rcu_num_lvls = i;
+                        for (j = i + 1; j <= MAX_RCU_LVLS; j++)
+                                num_rcu_lvl[j] = 0;
+                        break;
+                }
+
+        /* Calculate the total number of rcu_node structures. */
+        rcu_num_nodes = 0;
+        for (i = 0; i <= MAX_RCU_LVLS; i++)
+                rcu_num_nodes += num_rcu_lvl[i];
+        rcu_num_nodes -= n;
 }
 
 void __init rcu_init(void)
@@ -2663,6 +2801,7 @@ void __init rcu_init(void)
         int cpu;
 
         rcu_bootup_announce();
+        rcu_init_geometry();
         rcu_init_one(&rcu_sched_state, &rcu_sched_data);
         rcu_init_one(&rcu_bh_state, &rcu_bh_data);
         __rcu_init_preempt();
diff --git a/kernel/rcutree.h b/kernel/rcutree.h
index 19b61ac1079f..4d29169f2124 100644
--- a/kernel/rcutree.h
+++ b/kernel/rcutree.h
@@ -42,28 +42,28 @@
 #define RCU_FANOUT_4          (RCU_FANOUT_3 * CONFIG_RCU_FANOUT)
 
 #if NR_CPUS <= RCU_FANOUT_1
-#  define NUM_RCU_LVLS        1
+#  define RCU_NUM_LVLS        1
 #  define NUM_RCU_LVL_0       1
 #  define NUM_RCU_LVL_1       (NR_CPUS)
 #  define NUM_RCU_LVL_2       0
 #  define NUM_RCU_LVL_3       0
 #  define NUM_RCU_LVL_4       0
 #elif NR_CPUS <= RCU_FANOUT_2
-#  define NUM_RCU_LVLS        2
+#  define RCU_NUM_LVLS        2
 #  define NUM_RCU_LVL_0       1
 #  define NUM_RCU_LVL_1       DIV_ROUND_UP(NR_CPUS, RCU_FANOUT_1)
 #  define NUM_RCU_LVL_2       (NR_CPUS)
 #  define NUM_RCU_LVL_3       0
 #  define NUM_RCU_LVL_4       0
 #elif NR_CPUS <= RCU_FANOUT_3
-#  define NUM_RCU_LVLS        3
+#  define RCU_NUM_LVLS        3
 #  define NUM_RCU_LVL_0       1
 #  define NUM_RCU_LVL_1       DIV_ROUND_UP(NR_CPUS, RCU_FANOUT_2)
 #  define NUM_RCU_LVL_2       DIV_ROUND_UP(NR_CPUS, RCU_FANOUT_1)
 #  define NUM_RCU_LVL_3       (NR_CPUS)
 #  define NUM_RCU_LVL_4       0
 #elif NR_CPUS <= RCU_FANOUT_4
-#  define NUM_RCU_LVLS        4
+#  define RCU_NUM_LVLS        4
 #  define NUM_RCU_LVL_0       1
 #  define NUM_RCU_LVL_1       DIV_ROUND_UP(NR_CPUS, RCU_FANOUT_3)
 #  define NUM_RCU_LVL_2       DIV_ROUND_UP(NR_CPUS, RCU_FANOUT_2)
@@ -76,6 +76,9 @@
 #define RCU_SUM (NUM_RCU_LVL_0 + NUM_RCU_LVL_1 + NUM_RCU_LVL_2 + NUM_RCU_LVL_3 + NUM_RCU_LVL_4)
 #define NUM_RCU_NODES (RCU_SUM - NR_CPUS)
 
+extern int rcu_num_lvls;
+extern int rcu_num_nodes;
+
 /*
  * Dynticks per-CPU state.
  */
@@ -97,6 +100,7 @@ struct rcu_dynticks {
                                     /* # times non-lazy CBs posted to CPU. */
         unsigned long nonlazy_posted_snap;
                                     /* idle-period nonlazy_posted snapshot. */
+        int tick_nohz_enabled_snap; /* Previously seen value from sysfs. */
 #endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */
 };
 
@@ -206,7 +210,7 @@ struct rcu_node {
  */
 #define rcu_for_each_node_breadth_first(rsp, rnp) \
         for ((rnp) = &(rsp)->node[0]; \
-             (rnp) < &(rsp)->node[NUM_RCU_NODES]; (rnp)++)
+             (rnp) < &(rsp)->node[rcu_num_nodes]; (rnp)++)
 
 /*
  * Do a breadth-first scan of the non-leaf rcu_node structures for the
@@ -215,7 +219,7 @@ struct rcu_node {
  */
 #define rcu_for_each_nonleaf_node_breadth_first(rsp, rnp) \
         for ((rnp) = &(rsp)->node[0]; \
-             (rnp) < (rsp)->level[NUM_RCU_LVLS - 1]; (rnp)++)
+             (rnp) < (rsp)->level[rcu_num_lvls - 1]; (rnp)++)
 
 /*
  * Scan the leaves of the rcu_node hierarchy for the specified rcu_state
@@ -224,8 +228,8 @@ struct rcu_node {
  * It is still a leaf node, even if it is also the root node.
  */
 #define rcu_for_each_leaf_node(rsp, rnp) \
-        for ((rnp) = (rsp)->level[NUM_RCU_LVLS - 1]; \
-             (rnp) < &(rsp)->node[NUM_RCU_NODES]; (rnp)++)
+        for ((rnp) = (rsp)->level[rcu_num_lvls - 1]; \
+             (rnp) < &(rsp)->node[rcu_num_nodes]; (rnp)++)
 
 /* Index values for nxttail array in struct rcu_data. */
 #define RCU_DONE_TAIL           0       /* Also RCU_WAIT head. */
@@ -311,6 +315,9 @@ struct rcu_data {
         unsigned long n_rp_need_fqs;
         unsigned long n_rp_need_nothing;
 
+        /* 6) _rcu_barrier() callback. */
+        struct rcu_head barrier_head;
+
         int cpu;
         struct rcu_state *rsp;
 };
@@ -357,10 +364,12 @@ do {									\
  */
 struct rcu_state {
         struct rcu_node node[NUM_RCU_NODES];    /* Hierarchy. */
-        struct rcu_node *level[NUM_RCU_LVLS];   /* Hierarchy levels. */
+        struct rcu_node *level[RCU_NUM_LVLS];   /* Hierarchy levels. */
         u32 levelcnt[MAX_RCU_LVLS + 1];         /* # nodes in each level. */
-        u8 levelspread[NUM_RCU_LVLS];           /* kids/node in each level. */
+        u8 levelspread[RCU_NUM_LVLS];           /* kids/node in each level. */
         struct rcu_data __percpu *rda;          /* pointer of percu rcu_data. */
+        void (*call)(struct rcu_head *head,     /* call_rcu() flavor. */
+                     void (*func)(struct rcu_head *head));
 
         /* The following fields are guarded by the root rcu_node's lock. */
 
@@ -392,6 +401,11 @@ struct rcu_state {
         struct task_struct *rcu_barrier_in_progress;
                                                 /* Task doing rcu_barrier(), */
                                                 /*  or NULL if no barrier. */
+        struct mutex barrier_mutex;             /* Guards barrier fields. */
+        atomic_t barrier_cpu_count;             /* # CPUs waiting on. */
+        struct completion barrier_completion;   /* Wake at barrier end. */
+        unsigned long n_barrier_done;           /* ++ at start and end of */
+                                                /*  _rcu_barrier(). */
         raw_spinlock_t fqslock;                 /* Only one task forcing */
                                                 /*  quiescent states. */
         unsigned long jiffies_force_qs;         /* Time at which to invoke */
@@ -409,8 +423,13 @@ struct rcu_state {
         unsigned long gp_max;                   /* Maximum GP duration in */
                                                 /*  jiffies. */
         char *name;                             /* Name of structure. */
+        struct list_head flavors;               /* List of RCU flavors. */
 };
 
+extern struct list_head rcu_struct_flavors;
+#define for_each_rcu_flavor(rsp) \
+        list_for_each_entry((rsp), &rcu_struct_flavors, flavors)
+
 /* Return values for rcu_preempt_offline_tasks(). */
 
 #define RCU_OFL_TASKS_NORM_GP   0x1             /* Tasks blocking normal */
@@ -453,25 +472,18 @@ static void rcu_stop_cpu_kthread(int cpu);
 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
 static void rcu_print_detail_task_stall(struct rcu_state *rsp);
 static int rcu_print_task_stall(struct rcu_node *rnp);
-static void rcu_preempt_stall_reset(void);
 static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp);
 #ifdef CONFIG_HOTPLUG_CPU
 static int rcu_preempt_offline_tasks(struct rcu_state *rsp,
                                      struct rcu_node *rnp,
                                      struct rcu_data *rdp);
 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
-static void rcu_preempt_cleanup_dead_cpu(int cpu);
 static void rcu_preempt_check_callbacks(int cpu);
-static void rcu_preempt_process_callbacks(void);
 void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu));
 #if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_TREE_PREEMPT_RCU)
 static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
                                bool wake);
 #endif /* #if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_TREE_PREEMPT_RCU) */
-static int rcu_preempt_pending(int cpu);
-static int rcu_preempt_cpu_has_callbacks(int cpu);
-static void __cpuinit rcu_preempt_init_percpu_data(int cpu);
-static void rcu_preempt_cleanup_dying_cpu(void);
 static void __init __rcu_init_preempt(void);
 static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags);
 static void rcu_preempt_boost_start_gp(struct rcu_node *rnp);
diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h
index 3e4899459f3d..a9194d5606c4 100644
--- a/kernel/rcutree_plugin.h
+++ b/kernel/rcutree_plugin.h
@@ -68,17 +68,21 @@ static void __init rcu_bootup_announce_oddness(void)
         printk(KERN_INFO "\tAdditional per-CPU info printed with stalls.\n");
 #endif
 #if NUM_RCU_LVL_4 != 0
-        printk(KERN_INFO "\tExperimental four-level hierarchy is enabled.\n");
+        printk(KERN_INFO "\tFour-level hierarchy is enabled.\n");
 #endif
+        if (rcu_fanout_leaf != CONFIG_RCU_FANOUT_LEAF)
+                printk(KERN_INFO "\tExperimental boot-time adjustment of leaf fanout to %d.\n", rcu_fanout_leaf);
+        if (nr_cpu_ids != NR_CPUS)
+                printk(KERN_INFO "\tRCU restricting CPUs from NR_CPUS=%d to nr_cpu_ids=%d.\n", NR_CPUS, nr_cpu_ids);
 }
 
 #ifdef CONFIG_TREE_PREEMPT_RCU
 
-struct rcu_state rcu_preempt_state = RCU_STATE_INITIALIZER(rcu_preempt);
+struct rcu_state rcu_preempt_state =
+        RCU_STATE_INITIALIZER(rcu_preempt, call_rcu);
 DEFINE_PER_CPU(struct rcu_data, rcu_preempt_data);
 static struct rcu_state *rcu_state = &rcu_preempt_state;
 
-static void rcu_read_unlock_special(struct task_struct *t);
 static int rcu_preempted_readers_exp(struct rcu_node *rnp);
 
 /*
@@ -233,18 +237,6 @@ static void rcu_preempt_note_context_switch(int cpu)
 }
 
 /*
- * Tree-preemptible RCU implementation for rcu_read_lock().
- * Just increment ->rcu_read_lock_nesting, shared state will be updated
- * if we block.
- */
-void __rcu_read_lock(void)
-{
-        current->rcu_read_lock_nesting++;
-        barrier();  /* needed if we ever invoke rcu_read_lock in rcutree.c */
-}
-EXPORT_SYMBOL_GPL(__rcu_read_lock);
-
-/*
  * Check for preempted RCU readers blocking the current grace period
  * for the specified rcu_node structure.  If the caller needs a reliable
  * answer, it must hold the rcu_node's ->lock.
@@ -310,7 +302,7 @@ static struct list_head *rcu_next_node_entry(struct task_struct *t,
  * notify RCU core processing or task having blocked during the RCU
  * read-side critical section.
  */
-static noinline void rcu_read_unlock_special(struct task_struct *t)
+void rcu_read_unlock_special(struct task_struct *t)
 {
         int empty;
         int empty_exp;
@@ -418,38 +410,6 @@ static noinline void rcu_read_unlock_special(struct task_struct *t)
         }
 }
 
-/*
- * Tree-preemptible RCU implementation for rcu_read_unlock().
- * Decrement ->rcu_read_lock_nesting.  If the result is zero (outermost
- * rcu_read_unlock()) and ->rcu_read_unlock_special is non-zero, then
- * invoke rcu_read_unlock_special() to clean up after a context switch
- * in an RCU read-side critical section and other special cases.
- */
-void __rcu_read_unlock(void)
-{
-        struct task_struct *t = current;
-
-        if (t->rcu_read_lock_nesting != 1)
-                --t->rcu_read_lock_nesting;
-        else {
-                barrier();  /* critical section before exit code. */
-                t->rcu_read_lock_nesting = INT_MIN;
-                barrier();  /* assign before ->rcu_read_unlock_special load */
-                if (unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
-                        rcu_read_unlock_special(t);
-                barrier();  /* ->rcu_read_unlock_special load before assign */
-                t->rcu_read_lock_nesting = 0;
-        }
-#ifdef CONFIG_PROVE_LOCKING
-        {
-                int rrln = ACCESS_ONCE(t->rcu_read_lock_nesting);
-
-                WARN_ON_ONCE(rrln < 0 && rrln > INT_MIN / 2);
-        }
-#endif /* #ifdef CONFIG_PROVE_LOCKING */
-}
-EXPORT_SYMBOL_GPL(__rcu_read_unlock);
-
 #ifdef CONFIG_RCU_CPU_STALL_VERBOSE
 
 /*
@@ -540,16 +500,6 @@ static int rcu_print_task_stall(struct rcu_node *rnp)
 }
 
 /*
- * Suppress preemptible RCU's CPU stall warnings by pushing the
- * time of the next stall-warning message comfortably far into the
- * future.
- */
-static void rcu_preempt_stall_reset(void)
-{
-        rcu_preempt_state.jiffies_stall = jiffies + ULONG_MAX / 2;
-}
-
-/*
  * Check that the list of blocked tasks for the newly completed grace
  * period is in fact empty.  It is a serious bug to complete a grace
  * period that still has RCU readers blocked!  This function must be
@@ -650,14 +600,6 @@ static int rcu_preempt_offline_tasks(struct rcu_state *rsp,
 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
 
 /*
- * Do CPU-offline processing for preemptible RCU.
- */
-static void rcu_preempt_cleanup_dead_cpu(int cpu)
-{
-        rcu_cleanup_dead_cpu(cpu, &rcu_preempt_state);
-}
-
-/*
  * Check for a quiescent state from the current CPU.  When a task blocks,
  * the task is recorded in the corresponding CPU's rcu_node structure,
  * which is checked elsewhere.
@@ -677,15 +619,6 @@ static void rcu_preempt_check_callbacks(int cpu)
                 t->rcu_read_unlock_special |= RCU_READ_UNLOCK_NEED_QS;
 }
 
-/*
- * Process callbacks for preemptible RCU.
- */
-static void rcu_preempt_process_callbacks(void)
-{
-        __rcu_process_callbacks(&rcu_preempt_state,
-                                &__get_cpu_var(rcu_preempt_data));
-}
-
 #ifdef CONFIG_RCU_BOOST
 
 static void rcu_preempt_do_callbacks(void)
@@ -917,51 +850,16 @@ mb_ret:
 }
 EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
 
-/*
- * Check to see if there is any immediate preemptible-RCU-related work
- * to be done.
- */
-static int rcu_preempt_pending(int cpu)
-{
-        return __rcu_pending(&rcu_preempt_state,
-                             &per_cpu(rcu_preempt_data, cpu));
-}
-
-/*
- * Does preemptible RCU have callbacks on this CPU?
- */
-static int rcu_preempt_cpu_has_callbacks(int cpu)
-{
-        return !!per_cpu(rcu_preempt_data, cpu).nxtlist;
-}
-
 /**
  * rcu_barrier - Wait until all in-flight call_rcu() callbacks complete.
  */
 void rcu_barrier(void)
 {
-        _rcu_barrier(&rcu_preempt_state, call_rcu);
+        _rcu_barrier(&rcu_preempt_state);
 }
 EXPORT_SYMBOL_GPL(rcu_barrier);
 
 /*
- * Initialize preemptible RCU's per-CPU data.
- */
-static void __cpuinit rcu_preempt_init_percpu_data(int cpu)
-{
-        rcu_init_percpu_data(cpu, &rcu_preempt_state, 1);
-}
-
-/*
- * Move preemptible RCU's callbacks from dying CPU to other online CPU
- * and record a quiescent state.
- */
-static void rcu_preempt_cleanup_dying_cpu(void)
-{
-        rcu_cleanup_dying_cpu(&rcu_preempt_state);
-}
-
-/*
  * Initialize preemptible RCU's state structures.
  */
 static void __init __rcu_init_preempt(void)
@@ -1046,14 +944,6 @@ static int rcu_print_task_stall(struct rcu_node *rnp)
 }
 
 /*
- * Because preemptible RCU does not exist, there is no need to suppress
- * its CPU stall warnings.
- */
-static void rcu_preempt_stall_reset(void)
-{
-}
-
-/*
  * Because there is no preemptible RCU, there can be no readers blocked,
  * so there is no need to check for blocked tasks.  So check only for
  * bogus qsmask values.
@@ -1081,14 +971,6 @@ static int rcu_preempt_offline_tasks(struct rcu_state *rsp,
 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
 
 /*
- * Because preemptible RCU does not exist, it never needs CPU-offline
- * processing.
- */
-static void rcu_preempt_cleanup_dead_cpu(int cpu)
-{
-}
-
-/*
  * Because preemptible RCU does not exist, it never has any callbacks
  * to check.
  */
@@ -1097,14 +979,6 @@ static void rcu_preempt_check_callbacks(int cpu)
 }
 
 /*
- * Because preemptible RCU does not exist, it never has any callbacks
- * to process.
- */
-static void rcu_preempt_process_callbacks(void)
-{
-}
-
-/*
  * Queue an RCU callback for lazy invocation after a grace period.
  * This will likely be later named something like "call_rcu_lazy()",
  * but this change will require some way of tagging the lazy RCU
@@ -1145,22 +1019,6 @@ static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
 
 /*
- * Because preemptible RCU does not exist, it never has any work to do.
- */
-static int rcu_preempt_pending(int cpu)
-{
-        return 0;
-}
-
-/*
- * Because preemptible RCU does not exist, it never has callbacks
- */
-static int rcu_preempt_cpu_has_callbacks(int cpu)
-{
-        return 0;
-}
-
-/*
  * Because preemptible RCU does not exist, rcu_barrier() is just
  * another name for rcu_barrier_sched().
  */
@@ -1171,21 +1029,6 @@ void rcu_barrier(void)
 EXPORT_SYMBOL_GPL(rcu_barrier);
 
 /*
- * Because preemptible RCU does not exist, there is no per-CPU
- * data to initialize.
- */
-static void __cpuinit rcu_preempt_init_percpu_data(int cpu)
-{
-}
-
-/*
- * Because there is no preemptible RCU, there is no cleanup to do.
- */
-static void rcu_preempt_cleanup_dying_cpu(void)
-{
-}
-
-/*
  * Because preemptible RCU does not exist, it need not be initialized.
  */
 static void __init __rcu_init_preempt(void)
@@ -1968,9 +1811,11 @@ static void rcu_idle_count_callbacks_posted(void)
  */
 #define RCU_IDLE_FLUSHES 5              /* Number of dyntick-idle tries. */
 #define RCU_IDLE_OPT_FLUSHES 3          /* Optional dyntick-idle tries. */
-#define RCU_IDLE_GP_DELAY 6             /* Roughly one grace period. */
+#define RCU_IDLE_GP_DELAY 4             /* Roughly one grace period. */
 #define RCU_IDLE_LAZY_GP_DELAY (6 * HZ) /* Roughly six seconds. */
 
+extern int tick_nohz_enabled;
+
 /*
  * Does the specified flavor of RCU have non-lazy callbacks pending on
  * the specified CPU?  Both RCU flavor and CPU are specified by the
@@ -2047,10 +1892,13 @@ int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies)
                 return 1;
         }
         /* Set up for the possibility that RCU will post a timer. */
-        if (rcu_cpu_has_nonlazy_callbacks(cpu))
-                *delta_jiffies = RCU_IDLE_GP_DELAY;
-        else
-                *delta_jiffies = RCU_IDLE_LAZY_GP_DELAY;
+        if (rcu_cpu_has_nonlazy_callbacks(cpu)) {
+                *delta_jiffies = round_up(RCU_IDLE_GP_DELAY + jiffies,
+                                          RCU_IDLE_GP_DELAY) - jiffies;
+        } else {
+                *delta_jiffies = jiffies + RCU_IDLE_LAZY_GP_DELAY;
+                *delta_jiffies = round_jiffies(*delta_jiffies) - jiffies;
+        }
         return 0;
 }
 
@@ -2109,6 +1957,7 @@ static void rcu_cleanup_after_idle(int cpu)
 
         del_timer(&rdtp->idle_gp_timer);
         trace_rcu_prep_idle("Cleanup after idle");
+        rdtp->tick_nohz_enabled_snap = ACCESS_ONCE(tick_nohz_enabled);
 }
 
 /*
@@ -2134,6 +1983,18 @@ static void rcu_prepare_for_idle(int cpu)
 {
         struct timer_list *tp;
         struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
+        int tne;
+
+        /* Handle nohz enablement switches conservatively. */
+        tne = ACCESS_ONCE(tick_nohz_enabled);
+        if (tne != rdtp->tick_nohz_enabled_snap) {
+                if (rcu_cpu_has_callbacks(cpu))
+                        invoke_rcu_core(); /* force nohz to see update. */
+                rdtp->tick_nohz_enabled_snap = tne;
+                return;
+        }
+        if (!tne)
+                return;
 
         /*
          * If this is an idle re-entry, for example, due to use of
@@ -2187,10 +2048,11 @@ static void rcu_prepare_for_idle(int cpu)
                 if (rcu_cpu_has_nonlazy_callbacks(cpu)) {
                         trace_rcu_prep_idle("Dyntick with callbacks");
                         rdtp->idle_gp_timer_expires =
-                                           jiffies + RCU_IDLE_GP_DELAY;
+                                round_up(jiffies + RCU_IDLE_GP_DELAY,
+                                         RCU_IDLE_GP_DELAY);
                 } else {
                         rdtp->idle_gp_timer_expires =
-                                           jiffies + RCU_IDLE_LAZY_GP_DELAY;
+                                round_jiffies(jiffies + RCU_IDLE_LAZY_GP_DELAY);
                         trace_rcu_prep_idle("Dyntick with lazy callbacks");
                 }
                 tp = &rdtp->idle_gp_timer;
@@ -2269,6 +2131,7 @@ static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
 
 static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
 {
+        *cp = '\0';
 }
 
 #endif /* #else #ifdef CONFIG_RCU_FAST_NO_HZ */
diff --git a/kernel/rcutree_trace.c b/kernel/rcutree_trace.c
index d4bc16ddd1d4..a16ddbd6fdc4 100644
--- a/kernel/rcutree_trace.c
+++ b/kernel/rcutree_trace.c
@@ -46,6 +46,31 @@
 #define RCU_TREE_NONCORE
 #include "rcutree.h"
 
+static int show_rcubarrier(struct seq_file *m, void *unused)
+{
+        struct rcu_state *rsp;
+
+        for_each_rcu_flavor(rsp)
+                seq_printf(m, "%s: %c bcc: %d nbd: %lu\n",
+                           rsp->name, rsp->rcu_barrier_in_progress ? 'B' : '.',
+                           atomic_read(&rsp->barrier_cpu_count),
+                           rsp->n_barrier_done);
+        return 0;
+}
+
+static int rcubarrier_open(struct inode *inode, struct file *file)
+{
+        return single_open(file, show_rcubarrier, NULL);
+}
+
+static const struct file_operations rcubarrier_fops = {
+        .owner = THIS_MODULE,
+        .open = rcubarrier_open,
+        .read = seq_read,
+        .llseek = seq_lseek,
+        .release = single_release,
+};
+
 #ifdef CONFIG_RCU_BOOST
 
 static char convert_kthread_status(unsigned int kthread_status)
@@ -95,24 +120,16 @@ static void print_one_rcu_data(struct seq_file *m, struct rcu_data *rdp)
                    rdp->n_cbs_invoked, rdp->n_cbs_orphaned, rdp->n_cbs_adopted);
 }
 
-#define PRINT_RCU_DATA(name, func, m) \
-        do { \
-                int _p_r_d_i; \
-                \
-                for_each_possible_cpu(_p_r_d_i) \
-                        func(m, &per_cpu(name, _p_r_d_i)); \
-        } while (0)
-
 static int show_rcudata(struct seq_file *m, void *unused)
 {
-#ifdef CONFIG_TREE_PREEMPT_RCU
-        seq_puts(m, "rcu_preempt:\n");
-        PRINT_RCU_DATA(rcu_preempt_data, print_one_rcu_data, m);
-#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
-        seq_puts(m, "rcu_sched:\n");
-        PRINT_RCU_DATA(rcu_sched_data, print_one_rcu_data, m);
-        seq_puts(m, "rcu_bh:\n");
-        PRINT_RCU_DATA(rcu_bh_data, print_one_rcu_data, m);
+        int cpu;
+        struct rcu_state *rsp;
+
+        for_each_rcu_flavor(rsp) {
+                seq_printf(m, "%s:\n", rsp->name);
+                for_each_possible_cpu(cpu)
+                        print_one_rcu_data(m, per_cpu_ptr(rsp->rda, cpu));
+        }
         return 0;
 }
 
@@ -166,6 +183,9 @@ static void print_one_rcu_data_csv(struct seq_file *m, struct rcu_data *rdp)
 
 static int show_rcudata_csv(struct seq_file *m, void *unused)
 {
+        int cpu;
+        struct rcu_state *rsp;
+
         seq_puts(m, "\"CPU\",\"Online?\",\"c\",\"g\",\"pq\",\"pgp\",\"pq\",");
         seq_puts(m, "\"dt\",\"dt nesting\",\"dt NMI nesting\",\"df\",");
         seq_puts(m, "\"of\",\"qll\",\"ql\",\"qs\"");
@@ -173,14 +193,11 @@ static int show_rcudata_csv(struct seq_file *m, void *unused)
         seq_puts(m, "\"kt\",\"ktl\"");
 #endif /* #ifdef CONFIG_RCU_BOOST */
         seq_puts(m, ",\"b\",\"ci\",\"co\",\"ca\"\n");
-#ifdef CONFIG_TREE_PREEMPT_RCU
-        seq_puts(m, "\"rcu_preempt:\"\n");
-        PRINT_RCU_DATA(rcu_preempt_data, print_one_rcu_data_csv, m);
-#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
-        seq_puts(m, "\"rcu_sched:\"\n");
-        PRINT_RCU_DATA(rcu_sched_data, print_one_rcu_data_csv, m);
-        seq_puts(m, "\"rcu_bh:\"\n");
-        PRINT_RCU_DATA(rcu_bh_data, print_one_rcu_data_csv, m);
+        for_each_rcu_flavor(rsp) {
+                seq_printf(m, "\"%s:\"\n", rsp->name);
+                for_each_possible_cpu(cpu)
+                        print_one_rcu_data_csv(m, per_cpu_ptr(rsp->rda, cpu));
+        }
         return 0;
 }
 
@@ -270,15 +287,15 @@ static void print_one_rcu_state(struct seq_file *m, struct rcu_state *rsp)
         struct rcu_node *rnp;
 
         gpnum = rsp->gpnum;
-        seq_printf(m, "c=%lu g=%lu s=%d jfq=%ld j=%x "
+        seq_printf(m, "%s: c=%lu g=%lu s=%d jfq=%ld j=%x "
                       "nfqs=%lu/nfqsng=%lu(%lu) fqlh=%lu oqlen=%ld/%ld\n",
-                   rsp->completed, gpnum, rsp->fqs_state,
+                   rsp->name, rsp->completed, gpnum, rsp->fqs_state,
                    (long)(rsp->jiffies_force_qs - jiffies),
                    (int)(jiffies & 0xffff),
                    rsp->n_force_qs, rsp->n_force_qs_ngp,
                    rsp->n_force_qs - rsp->n_force_qs_ngp,
                    rsp->n_force_qs_lh, rsp->qlen_lazy, rsp->qlen);
-        for (rnp = &rsp->node[0]; rnp - &rsp->node[0] < NUM_RCU_NODES; rnp++) {
+        for (rnp = &rsp->node[0]; rnp - &rsp->node[0] < rcu_num_nodes; rnp++) {
                 if (rnp->level != level) {
                         seq_puts(m, "\n");
                         level = rnp->level;
@@ -295,14 +312,10 @@ static void print_one_rcu_state(struct seq_file *m, struct rcu_state *rsp)
 
 static int show_rcuhier(struct seq_file *m, void *unused)
 {
-#ifdef CONFIG_TREE_PREEMPT_RCU
-        seq_puts(m, "rcu_preempt:\n");
-        print_one_rcu_state(m, &rcu_preempt_state);
-#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
-        seq_puts(m, "rcu_sched:\n");
-        print_one_rcu_state(m, &rcu_sched_state);
-        seq_puts(m, "rcu_bh:\n");
-        print_one_rcu_state(m, &rcu_bh_state);
+        struct rcu_state *rsp;
+
+        for_each_rcu_flavor(rsp)
+                print_one_rcu_state(m, rsp);
         return 0;
 }
 
@@ -343,11 +356,10 @@ static void show_one_rcugp(struct seq_file *m, struct rcu_state *rsp)
 
 static int show_rcugp(struct seq_file *m, void *unused)
 {
-#ifdef CONFIG_TREE_PREEMPT_RCU
-        show_one_rcugp(m, &rcu_preempt_state);
-#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
-        show_one_rcugp(m, &rcu_sched_state);
-        show_one_rcugp(m, &rcu_bh_state);
+        struct rcu_state *rsp;
+
+        for_each_rcu_flavor(rsp)
+                show_one_rcugp(m, rsp);
         return 0;
 }
 
@@ -382,28 +394,20 @@ static void print_one_rcu_pending(struct seq_file *m, struct rcu_data *rdp)
                    rdp->n_rp_need_nothing);
 }
 
-static void print_rcu_pendings(struct seq_file *m, struct rcu_state *rsp)
+static int show_rcu_pending(struct seq_file *m, void *unused)
 {
         int cpu;
         struct rcu_data *rdp;
-
-        for_each_possible_cpu(cpu) {
-                rdp = per_cpu_ptr(rsp->rda, cpu);
-                if (rdp->beenonline)
-                        print_one_rcu_pending(m, rdp);
+        struct rcu_state *rsp;
+
+        for_each_rcu_flavor(rsp) {
+                seq_printf(m, "%s:\n", rsp->name);
+                for_each_possible_cpu(cpu) {
+                        rdp = per_cpu_ptr(rsp->rda, cpu);
+                        if (rdp->beenonline)
+                                print_one_rcu_pending(m, rdp);
+                }
         }
-}
-
-static int show_rcu_pending(struct seq_file *m, void *unused)
-{
-#ifdef CONFIG_TREE_PREEMPT_RCU
-        seq_puts(m, "rcu_preempt:\n");
-        print_rcu_pendings(m, &rcu_preempt_state);
-#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
-        seq_puts(m, "rcu_sched:\n");
-        print_rcu_pendings(m, &rcu_sched_state);
-        seq_puts(m, "rcu_bh:\n");
-        print_rcu_pendings(m, &rcu_bh_state);
         return 0;
 }
 
@@ -453,6 +457,11 @@ static int __init rcutree_trace_init(void)
         if (!rcudir)
                 goto free_out;
 
+        retval = debugfs_create_file("rcubarrier", 0444, rcudir,
+                                                NULL, &rcubarrier_fops);
+        if (!retval)
+                goto free_out;
+
         retval = debugfs_create_file("rcudata", 0444, rcudir,
                                                 NULL, &rcudata_fops);
         if (!retval)
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 869997833928..66ff07f6184c 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -105,7 +105,7 @@ static ktime_t tick_init_jiffy_update(void)
 /*
  * NO HZ enabled ?
  */
-static int tick_nohz_enabled __read_mostly  = 1;
+int tick_nohz_enabled __read_mostly  = 1;
 
 /*
  * Enable / Disable tickless mode