17 files changed, 357 insertions, 222 deletions

diff --git a/Documentation/atomic_ops.txt b/Documentation/atomic_ops.txt
index 183e41bdcb69..dab6da3382d9 100644
--- a/Documentation/atomic_ops.txt
+++ b/Documentation/atomic_ops.txt
@@ -201,11 +201,11 @@ These routines add 1 and subtract 1, respectively, from the given
 atomic_t and return the new counter value after the operation is
 performed.
 
-Unlike the above routines, it is required that explicit memory
-barriers are performed before and after the operation.  It must be
-done such that all memory operations before and after the atomic
-operation calls are strongly ordered with respect to the atomic
-operation itself.
+Unlike the above routines, it is required that these primitives
+include explicit memory barriers that are performed before and after
+the operation.  It must be done such that all memory operations before
+and after the atomic operation calls are strongly ordered with respect
+to the atomic operation itself.
 
 For example, it should behave as if a smp_mb() call existed both
 before and after the atomic operation.
@@ -233,21 +233,21 @@ These two routines increment and decrement by 1, respectively, the
 given atomic counter.  They return a boolean indicating whether the
 resulting counter value was zero or not.
 
-It requires explicit memory barrier semantics around the operation as
-above.
+Again, these primitives provide explicit memory barrier semantics around
+the atomic operation.
 
         int atomic_sub_and_test(int i, atomic_t *v);
 
 This is identical to atomic_dec_and_test() except that an explicit
-decrement is given instead of the implicit "1".  It requires explicit
-memory barrier semantics around the operation.
+decrement is given instead of the implicit "1".  This primitive must
+provide explicit memory barrier semantics around the operation.
 
         int atomic_add_negative(int i, atomic_t *v);
 
-The given increment is added to the given atomic counter value.  A
-boolean is return which indicates whether the resulting counter value
-is negative.  It requires explicit memory barrier semantics around the
-operation.
+The given increment is added to the given atomic counter value.  A boolean
+is return which indicates whether the resulting counter value is negative.
+This primitive must provide explicit memory barrier semantics around
+the operation.
 
 Then:
 
@@ -257,7 +257,7 @@ This performs an atomic exchange operation on the atomic variable v, setting
 the given new value.  It returns the old value that the atomic variable v had
 just before the operation.
 
-atomic_xchg requires explicit memory barriers around the operation.
+atomic_xchg must provide explicit memory barriers around the operation.
 
         int atomic_cmpxchg(atomic_t *v, int old, int new);
 
@@ -266,7 +266,7 @@ with the given old and new values. Like all atomic_xxx operations,
 atomic_cmpxchg will only satisfy its atomicity semantics as long as all
 other accesses of *v are performed through atomic_xxx operations.
 
-atomic_cmpxchg requires explicit memory barriers around the operation.
+atomic_cmpxchg must provide explicit memory barriers around the operation.
 
 The semantics for atomic_cmpxchg are the same as those defined for 'cas'
 below.
@@ -279,8 +279,8 @@ If the atomic value v is not equal to u, this function adds a to v, and
 returns non zero. If v is equal to u then it returns zero. This is done as
 an atomic operation.
 
-atomic_add_unless requires explicit memory barriers around the operation
-unless it fails (returns 0).
+atomic_add_unless must provide explicit memory barriers around the
+operation unless it fails (returns 0).
 
 atomic_inc_not_zero, equivalent to atomic_add_unless(v, 1, 0)
 
@@ -460,9 +460,9 @@ the return value into an int.  There are other places where things
 like this occur as well.
 
 These routines, like the atomic_t counter operations returning values,
-require explicit memory barrier semantics around their execution.  All
-memory operations before the atomic bit operation call must be made
-visible globally before the atomic bit operation is made visible.
+must provide explicit memory barrier semantics around their execution.
+All memory operations before the atomic bit operation call must be
+made visible globally before the atomic bit operation is made visible.
 Likewise, the atomic bit operation must be visible globally before any
 subsequent memory operation is made visible.  For example:
 
@@ -536,8 +536,9 @@ except that two underscores are prefixed to the interface name.
 These non-atomic variants also do not require any special memory
 barrier semantics.
 
-The routines xchg() and cmpxchg() need the same exact memory barriers
-as the atomic and bit operations returning values.
+The routines xchg() and cmpxchg() must provide the same exact
+memory-barrier semantics as the atomic and bit operations returning
+values.
 
 Spinlocks and rwlocks have memory barrier expectations as well.
 The rule to follow is simple:
diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt
index 94de410ec341..5368ba701de2 100644
--- a/Documentation/kernel-parameters.txt
+++ b/Documentation/kernel-parameters.txt
@@ -2997,11 +2997,15 @@ bytes respectively. Such letter suffixes can also be entirely omitted.
                         value is one, and maximum value is HZ.
 
         rcutree.kthread_prio=    [KNL,BOOT]
-                        Set the SCHED_FIFO priority of the RCU
-                        per-CPU kthreads (rcuc/N). This value is also
-                        used for the priority of the RCU boost threads
-                        (rcub/N). Valid values are 1-99 and the default
-                        is 1 (the least-favored priority).
+                        Set the SCHED_FIFO priority of the RCU per-CPU
+                        kthreads (rcuc/N). This value is also used for
+                        the priority of the RCU boost threads (rcub/N)
+                        and for the RCU grace-period kthreads (rcu_bh,
+                        rcu_preempt, and rcu_sched). If RCU_BOOST is
+                        set, valid values are 1-99 and the default is 1
+                        (the least-favored priority).  Otherwise, when
+                        RCU_BOOST is not set, valid values are 0-99 and
+                        the default is zero (non-realtime operation).
 
         rcutree.rcu_nocb_leader_stride= [KNL]
                         Set the number of NOCB kthread groups, which
diff --git a/Documentation/kernel-per-CPU-kthreads.txt b/Documentation/kernel-per-CPU-kthreads.txt
index f3cd299fcc41..f4cbfe0ba108 100644
--- a/Documentation/kernel-per-CPU-kthreads.txt
+++ b/Documentation/kernel-per-CPU-kthreads.txt
@@ -190,20 +190,24 @@ To reduce its OS jitter, do any of the following:
                 on each CPU, including cs_dbs_timer() and od_dbs_timer().
                 WARNING:  Please check your CPU specifications to
                 make sure that this is safe on your particular system.
-        d.      It is not possible to entirely get rid of OS jitter
-                from vmstat_update() on CONFIG_SMP=y systems, but you
-                can decrease its frequency by writing a large value
-                to /proc/sys/vm/stat_interval.  The default value is
-                HZ, for an interval of one second.  Of course, larger
-                values will make your virtual-memory statistics update
-                more slowly.  Of course, you can also run your workload
-                at a real-time priority, thus preempting vmstat_update(),
+        d.      As of v3.18, Christoph Lameter's on-demand vmstat workers
+                commit prevents OS jitter due to vmstat_update() on
+                CONFIG_SMP=y systems.  Before v3.18, is not possible
+                to entirely get rid of the OS jitter, but you can
+                decrease its frequency by writing a large value to
+                /proc/sys/vm/stat_interval.  The default value is HZ,
+                for an interval of one second.  Of course, larger values
+                will make your virtual-memory statistics update more
+                slowly.  Of course, you can also run your workload at
+                a real-time priority, thus preempting vmstat_update(),
                 but if your workload is CPU-bound, this is a bad idea.
                 However, there is an RFC patch from Christoph Lameter
                 (based on an earlier one from Gilad Ben-Yossef) that
                 reduces or even eliminates vmstat overhead for some
                 workloads at https://lkml.org/lkml/2013/9/4/379.
-        e.      If running on high-end powerpc servers, build with
+        e.      Boot with "elevator=noop" to avoid workqueue use by
+                the block layer.
+        f.      If running on high-end powerpc servers, build with
                 CONFIG_PPC_RTAS_DAEMON=n.  This prevents the RTAS
                 daemon from running on each CPU every second or so.
                 (This will require editing Kconfig files and will defeat
@@ -211,12 +215,12 @@ To reduce its OS jitter, do any of the following:
                 due to the rtas_event_scan() function.
                 WARNING:  Please check your CPU specifications to
                 make sure that this is safe on your particular system.
-        f.      If running on Cell Processor, build your kernel with
+        g.      If running on Cell Processor, build your kernel with
                 CBE_CPUFREQ_SPU_GOVERNOR=n to avoid OS jitter from
                 spu_gov_work().
                 WARNING:  Please check your CPU specifications to
                 make sure that this is safe on your particular system.
-        g.      If running on PowerMAC, build your kernel with
+        h.      If running on PowerMAC, build your kernel with
                 CONFIG_PMAC_RACKMETER=n to disable the CPU-meter,
                 avoiding OS jitter from rackmeter_do_timer().
 
@@ -258,8 +262,12 @@ Purpose: Detect software lockups on each CPU.
 To reduce its OS jitter, do at least one of the following:
 1.      Build with CONFIG_LOCKUP_DETECTOR=n, which will prevent these
         kthreads from being created in the first place.
-2.      Echo a zero to /proc/sys/kernel/watchdog to disable the
+2.      Boot with "nosoftlockup=0", which will also prevent these kthreads
+        from being created.  Other related watchdog and softlockup boot
+        parameters may be found in Documentation/kernel-parameters.txt
+        and Documentation/watchdog/watchdog-parameters.txt.
+3.      Echo a zero to /proc/sys/kernel/watchdog to disable the
         watchdog timer.
-3.      Echo a large number of /proc/sys/kernel/watchdog_thresh in
+4.      Echo a large number of /proc/sys/kernel/watchdog_thresh in
         order to reduce the frequency of OS jitter due to the watchdog
         timer down to a level that is acceptable for your workload.
diff --git a/Documentation/memory-barriers.txt b/Documentation/memory-barriers.txt
index ca2387ef27ab..6974f1c2b4e1 100644
--- a/Documentation/memory-barriers.txt
+++ b/Documentation/memory-barriers.txt
@@ -592,9 +592,9 @@ See also the subsection on "Cache Coherency" for a more thorough example.
 CONTROL DEPENDENCIES
 --------------------
 
-A control dependency requires a full read memory barrier, not simply a data
-dependency barrier to make it work correctly.  Consider the following bit of
-code:
+A load-load control dependency requires a full read memory barrier, not
+simply a data dependency barrier to make it work correctly.  Consider the
+following bit of code:
 
         q = ACCESS_ONCE(a);
         if (q) {
@@ -615,14 +615,15 @@ case what's actually required is:
         }
 
 However, stores are not speculated.  This means that ordering -is- provided
-in the following example:
+for load-store control dependencies, as in the following example:
 
         q = ACCESS_ONCE(a);
         if (q) {
                 ACCESS_ONCE(b) = p;
         }
 
-Please note that ACCESS_ONCE() is not optional!  Without the
+Control dependencies pair normally with other types of barriers.
+That said, please note that ACCESS_ONCE() is not optional!  Without the
 ACCESS_ONCE(), might combine the load from 'a' with other loads from
 'a', and the store to 'b' with other stores to 'b', with possible highly
 counterintuitive effects on ordering.
@@ -813,6 +814,8 @@ In summary:
       barrier() can help to preserve your control dependency.  Please
       see the Compiler Barrier section for more information.
 
+  (*) Control dependencies pair normally with other types of barriers.
+
   (*) Control dependencies do -not- provide transitivity.  If you
       need transitivity, use smp_mb().
 
@@ -823,14 +826,14 @@ SMP BARRIER PAIRING
 When dealing with CPU-CPU interactions, certain types of memory barrier should
 always be paired.  A lack of appropriate pairing is almost certainly an error.
 
-General barriers pair with each other, though they also pair with
-most other types of barriers, albeit without transitivity.  An acquire
-barrier pairs with a release barrier, but both may also pair with other
-barriers, including of course general barriers.  A write barrier pairs
-with a data dependency barrier, an acquire barrier, a release barrier,
-a read barrier, or a general barrier.  Similarly a read barrier or a
-data dependency barrier pairs with a write barrier, an acquire barrier,
-a release barrier, or a general barrier:
+General barriers pair with each other, though they also pair with most
+other types of barriers, albeit without transitivity.  An acquire barrier
+pairs with a release barrier, but both may also pair with other barriers,
+including of course general barriers.  A write barrier pairs with a data
+dependency barrier, a control dependency, an acquire barrier, a release
+barrier, a read barrier, or a general barrier.  Similarly a read barrier,
+control dependency, or a data dependency barrier pairs with a write
+barrier, an acquire barrier, a release barrier, or a general barrier:
 
         CPU 1                 CPU 2
         ===============       ===============
@@ -850,6 +853,19 @@ Or:
                               <data dependency barrier>
                               y = *x;
 
+Or even:
+
+        CPU 1                 CPU 2
+        ===============       ===============================
+        r1 = ACCESS_ONCE(y);
+        <general barrier>
+        ACCESS_ONCE(y) = 1;   if (r2 = ACCESS_ONCE(x)) {
+                                 <implicit control dependency>
+                                 ACCESS_ONCE(y) = 1;
+                              }
+
+        assert(r1 == 0 || r2 == 0);
+
 Basically, the read barrier always has to be there, even though it can be of
 the "weaker" type.
 
diff --git a/Documentation/timers/NO_HZ.txt b/Documentation/timers/NO_HZ.txt
index cca122f25120..6eaf576294f3 100644
--- a/Documentation/timers/NO_HZ.txt
+++ b/Documentation/timers/NO_HZ.txt
@@ -158,13 +158,9 @@ not come for free:
         to the need to inform kernel subsystems (such as RCU) about
         the change in mode.
 
-3.      POSIX CPU timers on adaptive-tick CPUs may miss their deadlines
-        (perhaps indefinitely) because they currently rely on
-        scheduling-tick interrupts.  This will likely be fixed in
-        one of two ways: (1) Prevent CPUs with POSIX CPU timers from
-        entering adaptive-tick mode, or (2) Use hrtimers or other
-        adaptive-ticks-immune mechanism to cause the POSIX CPU timer to
-        fire properly.
+3.      POSIX CPU timers prevent CPUs from entering adaptive-tick mode.
+        Real-time applications needing to take actions based on CPU time
+        consumption need to use other means of doing so.
 
 4.      If there are more perf events pending than the hardware can
         accommodate, they are normally round-robined so as to collect
diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h
index 74ab23176e9b..066ba4157541 100644
--- a/include/linux/lockdep.h
+++ b/include/linux/lockdep.h
@@ -531,8 +531,13 @@ do {									\
 # define might_lock_read(lock) do { } while (0)
 #endif
 
-#ifdef CONFIG_PROVE_RCU
+#ifdef CONFIG_LOCKDEP
 void lockdep_rcu_suspicious(const char *file, const int line, const char *s);
+#else
+static inline void
+lockdep_rcu_suspicious(const char *file, const int line, const char *s)
+{
+}
 #endif
 
 #endif /* __LINUX_LOCKDEP_H */
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index 762022f07afd..573a5afd5ed8 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -48,6 +48,26 @@
 
 extern int rcu_expedited; /* for sysctl */
 
+#ifdef CONFIG_TINY_RCU
+/* Tiny RCU doesn't expedite, as its purpose in life is instead to be tiny. */
+static inline bool rcu_gp_is_expedited(void)  /* Internal RCU use. */
+{
+        return false;
+}
+
+static inline void rcu_expedite_gp(void)
+{
+}
+
+static inline void rcu_unexpedite_gp(void)
+{
+}
+#else /* #ifdef CONFIG_TINY_RCU */
+bool rcu_gp_is_expedited(void);  /* Internal RCU use. */
+void rcu_expedite_gp(void);
+void rcu_unexpedite_gp(void);
+#endif /* #else #ifdef CONFIG_TINY_RCU */
+
 enum rcutorture_type {
         RCU_FLAVOR,
         RCU_BH_FLAVOR,
@@ -195,6 +215,15 @@ void call_rcu_sched(struct rcu_head *head,
 
 void synchronize_sched(void);
 
+/*
+ * Structure allowing asynchronous waiting on RCU.
+ */
+struct rcu_synchronize {
+        struct rcu_head head;
+        struct completion completion;
+};
+void wakeme_after_rcu(struct rcu_head *head);
+
 /**
  * call_rcu_tasks() - Queue an RCU for invocation task-based grace period
  * @head: structure to be used for queueing the RCU updates.
@@ -258,6 +287,7 @@ static inline int rcu_preempt_depth(void)
 
 /* Internal to kernel */
 void rcu_init(void);
+void rcu_end_inkernel_boot(void);
 void rcu_sched_qs(void);
 void rcu_bh_qs(void);
 void rcu_check_callbacks(int user);
@@ -722,7 +752,7 @@ static inline void rcu_preempt_sleep_check(void)
  * annotated as __rcu.
  */
 #define rcu_dereference_check(p, c) \
-        __rcu_dereference_check((p), rcu_read_lock_held() || (c), __rcu)
+        __rcu_dereference_check((p), (c) || rcu_read_lock_held(), __rcu)
 
 /**
  * rcu_dereference_bh_check() - rcu_dereference_bh with debug checking
@@ -732,7 +762,7 @@ static inline void rcu_preempt_sleep_check(void)
  * This is the RCU-bh counterpart to rcu_dereference_check().
  */
 #define rcu_dereference_bh_check(p, c) \
-        __rcu_dereference_check((p), rcu_read_lock_bh_held() || (c), __rcu)
+        __rcu_dereference_check((p), (c) || rcu_read_lock_bh_held(), __rcu)
 
 /**
  * rcu_dereference_sched_check() - rcu_dereference_sched with debug checking
@@ -742,7 +772,7 @@ static inline void rcu_preempt_sleep_check(void)
  * This is the RCU-sched counterpart to rcu_dereference_check().
  */
 #define rcu_dereference_sched_check(p, c) \
-        __rcu_dereference_check((p), rcu_read_lock_sched_held() || (c), \
+        __rcu_dereference_check((p), (c) || rcu_read_lock_sched_held(), \
                                 __rcu)
 
 #define rcu_dereference_raw(p) rcu_dereference_check(p, 1) /*@@@ needed? @@@*/
@@ -935,9 +965,9 @@ static inline void rcu_read_unlock(void)
 {
         rcu_lockdep_assert(rcu_is_watching(),
                            "rcu_read_unlock() used illegally while idle");
-        rcu_lock_release(&rcu_lock_map);
         __release(RCU);
         __rcu_read_unlock();
+        rcu_lock_release(&rcu_lock_map); /* Keep acq info for rls diags. */
 }
 
 /**
diff --git a/include/linux/srcu.h b/include/linux/srcu.h
index 9cfd9623fb03..bdeb4567b71e 100644
--- a/include/linux/srcu.h
+++ b/include/linux/srcu.h
@@ -182,7 +182,7 @@ static inline int srcu_read_lock_held(struct srcu_struct *sp)
  * lockdep_is_held() calls.
  */
 #define srcu_dereference_check(p, sp, c) \
-        __rcu_dereference_check((p), srcu_read_lock_held(sp) || (c), __rcu)
+        __rcu_dereference_check((p), (c) || srcu_read_lock_held(sp), __rcu)
 
 /**
  * srcu_dereference - fetch SRCU-protected pointer for later dereferencing
diff --git a/init/Kconfig b/init/Kconfig
index f5dbc6d4261b..9a0592516f48 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -791,6 +791,19 @@ config RCU_NOCB_CPU_ALL
 
 endchoice
 
+config RCU_EXPEDITE_BOOT
+        bool
+        default n
+        help
+          This option enables expedited grace periods at boot time,
+          as if rcu_expedite_gp() had been invoked early in boot.
+          The corresponding rcu_unexpedite_gp() is invoked from
+          rcu_end_inkernel_boot(), which is intended to be invoked
+          at the end of the kernel-only boot sequence, just before
+          init is exec'ed.
+
+          Accept the default if unsure.
+
 endmenu # "RCU Subsystem"
 
 config BUILD_BIN2C
diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c
index 30d42aa55d83..8dbe27611ec3 100644
--- a/kernel/rcu/rcutorture.c
+++ b/kernel/rcu/rcutorture.c
@@ -853,6 +853,8 @@ rcu_torture_fqs(void *arg)
 static int
 rcu_torture_writer(void *arg)
 {
+        bool can_expedite = !rcu_gp_is_expedited();
+        int expediting = 0;
         unsigned long gp_snap;
         bool gp_cond1 = gp_cond, gp_exp1 = gp_exp, gp_normal1 = gp_normal;
         bool gp_sync1 = gp_sync;
@@ -865,9 +867,15 @@ rcu_torture_writer(void *arg)
         int nsynctypes = 0;
 
         VERBOSE_TOROUT_STRING("rcu_torture_writer task started");
+        pr_alert("%s" TORTURE_FLAG
+                 " Grace periods expedited from boot/sysfs for %s,\n",
+                 torture_type, cur_ops->name);
+        pr_alert("%s" TORTURE_FLAG
+                 " Testing of dynamic grace-period expediting diabled.\n",
+                 torture_type);
 
         /* Initialize synctype[] array.  If none set, take default. */
-        if (!gp_cond1 && !gp_exp1 && !gp_normal1 && !gp_sync)
+        if (!gp_cond1 && !gp_exp1 && !gp_normal1 && !gp_sync1)
                 gp_cond1 = gp_exp1 = gp_normal1 = gp_sync1 = true;
         if (gp_cond1 && cur_ops->get_state && cur_ops->cond_sync)
                 synctype[nsynctypes++] = RTWS_COND_GET;
@@ -949,9 +957,26 @@ rcu_torture_writer(void *arg)
                         }
                 }
                 rcutorture_record_progress(++rcu_torture_current_version);
+                /* Cycle through nesting levels of rcu_expedite_gp() calls. */
+                if (can_expedite &&
+                    !(torture_random(&rand) & 0xff & (!!expediting - 1))) {
+                        WARN_ON_ONCE(expediting == 0 && rcu_gp_is_expedited());
+                        if (expediting >= 0)
+                                rcu_expedite_gp();
+                        else
+                                rcu_unexpedite_gp();
+                        if (++expediting > 3)
+                                expediting = -expediting;
+                }
                 rcu_torture_writer_state = RTWS_STUTTER;
                 stutter_wait("rcu_torture_writer");
         } while (!torture_must_stop());
+        /* Reset expediting back to unexpedited. */
+        if (expediting > 0)
+                expediting = -expediting;
+        while (can_expedite && expediting++ < 0)
+                rcu_unexpedite_gp();
+        WARN_ON_ONCE(can_expedite && rcu_gp_is_expedited());
         rcu_torture_writer_state = RTWS_STOPPING;
         torture_kthread_stopping("rcu_torture_writer");
         return 0;
diff --git a/kernel/rcu/srcu.c b/kernel/rcu/srcu.c
index 445bf8ffe3fb..cad76e76b4e7 100644
--- a/kernel/rcu/srcu.c
+++ b/kernel/rcu/srcu.c
@@ -402,23 +402,6 @@ void call_srcu(struct srcu_struct *sp, struct rcu_head *head,
 }
 EXPORT_SYMBOL_GPL(call_srcu);
 
-struct rcu_synchronize {
-        struct rcu_head head;
-        struct completion completion;
-};
-
-/*
- * Awaken the corresponding synchronize_srcu() instance now that a
- * grace period has elapsed.
- */
-static void wakeme_after_rcu(struct rcu_head *head)
-{
-        struct rcu_synchronize *rcu;
-
-        rcu = container_of(head, struct rcu_synchronize, head);
-        complete(&rcu->completion);
-}
-
 static void srcu_advance_batches(struct srcu_struct *sp, int trycount);
 static void srcu_reschedule(struct srcu_struct *sp);
 
@@ -507,7 +490,7 @@ static void __synchronize_srcu(struct srcu_struct *sp, int trycount)
  */
 void synchronize_srcu(struct srcu_struct *sp)
 {
-        __synchronize_srcu(sp, rcu_expedited
+        __synchronize_srcu(sp, rcu_gp_is_expedited()
                            ? SYNCHRONIZE_SRCU_EXP_TRYCOUNT
                            : SYNCHRONIZE_SRCU_TRYCOUNT);
 }
diff --git a/kernel/rcu/tiny.c b/kernel/rcu/tiny.c
index cc9ceca7bde1..069742d61c68 100644
--- a/kernel/rcu/tiny.c
+++ b/kernel/rcu/tiny.c
@@ -103,8 +103,7 @@ EXPORT_SYMBOL(__rcu_is_watching);
 static int rcu_qsctr_help(struct rcu_ctrlblk *rcp)
 {
         RCU_TRACE(reset_cpu_stall_ticks(rcp));
-        if (rcp->rcucblist != NULL &&
-            rcp->donetail != rcp->curtail) {
+        if (rcp->donetail != rcp->curtail) {
                 rcp->donetail = rcp->curtail;
                 return 1;
         }
@@ -169,17 +168,6 @@ static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp)
         unsigned long flags;
         RCU_TRACE(int cb_count = 0);
 
-        /* If no RCU callbacks ready to invoke, just return. */
-        if (&rcp->rcucblist == rcp->donetail) {
-                RCU_TRACE(trace_rcu_batch_start(rcp->name, 0, 0, -1));
-                RCU_TRACE(trace_rcu_batch_end(rcp->name, 0,
-                                              !!ACCESS_ONCE(rcp->rcucblist),
-                                              need_resched(),
-                                              is_idle_task(current),
-                                              false));
-                return;
-        }
-
         /* Move the ready-to-invoke callbacks to a local list. */
         local_irq_save(flags);
         RCU_TRACE(trace_rcu_batch_start(rcp->name, 0, rcp->qlen, -1));
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 8fcc64ed858c..233165da782f 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -91,8 +91,10 @@ static const char *tp_##sname##_varname __used __tracepoint_string = sname##_var
 
 #define RCU_STATE_INITIALIZER(sname, sabbr, cr) \
 DEFINE_RCU_TPS(sname) \
+DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, sname##_data); \
 struct rcu_state sname##_state = { \
         .level = { &sname##_state.node[0] }, \
+        .rda = &sname##_data, \
         .call = cr, \
         .fqs_state = RCU_GP_IDLE, \
         .gpnum = 0UL - 300UL, \
@@ -103,8 +105,7 @@ struct rcu_state sname##_state = { \
         .barrier_mutex = __MUTEX_INITIALIZER(sname##_state.barrier_mutex), \
         .name = RCU_STATE_NAME(sname), \
         .abbr = sabbr, \
-}; \
-DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, sname##_data)
+}
 
 RCU_STATE_INITIALIZER(rcu_sched, 's', call_rcu_sched);
 RCU_STATE_INITIALIZER(rcu_bh, 'b', call_rcu_bh);
@@ -310,10 +311,10 @@ void rcu_note_context_switch(void)
 EXPORT_SYMBOL_GPL(rcu_note_context_switch);
 
 /*
- * Register a quiesecent state for all RCU flavors.  If there is an
+ * Register a quiescent state for all RCU flavors.  If there is an
  * emergency, invoke rcu_momentary_dyntick_idle() to do a heavy-weight
  * dyntick-idle quiescent state visible to other CPUs (but only for those
- * RCU flavors in desparate need of a quiescent state, which will normally
+ * RCU flavors in desperate need of a quiescent state, which will normally
  * be none of them).  Either way, do a lightweight quiescent state for
  * all RCU flavors.
  */
@@ -428,6 +429,15 @@ void rcu_bh_force_quiescent_state(void)
 EXPORT_SYMBOL_GPL(rcu_bh_force_quiescent_state);
 
 /*
+ * Force a quiescent state for RCU-sched.
+ */
+void rcu_sched_force_quiescent_state(void)
+{
+        force_quiescent_state(&rcu_sched_state);
+}
+EXPORT_SYMBOL_GPL(rcu_sched_force_quiescent_state);
+
+/*
  * Show the state of the grace-period kthreads.
  */
 void show_rcu_gp_kthreads(void)
@@ -501,15 +511,6 @@ void rcutorture_record_progress(unsigned long vernum)
 EXPORT_SYMBOL_GPL(rcutorture_record_progress);
 
 /*
- * Force a quiescent state for RCU-sched.
- */
-void rcu_sched_force_quiescent_state(void)
-{
-        force_quiescent_state(&rcu_sched_state);
-}
-EXPORT_SYMBOL_GPL(rcu_sched_force_quiescent_state);
-
-/*
  * Does the CPU have callbacks ready to be invoked?
  */
 static int
@@ -1347,20 +1348,30 @@ void rcu_cpu_stall_reset(void)
 }
 
 /*
- * Initialize the specified rcu_data structure's callback list to empty.
+ * Initialize the specified rcu_data structure's default callback list
+ * to empty.  The default callback list is the one that is not used by
+ * no-callbacks CPUs.
  */
-static void init_callback_list(struct rcu_data *rdp)
+static void init_default_callback_list(struct rcu_data *rdp)
 {
         int i;
 
-        if (init_nocb_callback_list(rdp))
-                return;
         rdp->nxtlist = NULL;
         for (i = 0; i < RCU_NEXT_SIZE; i++)
                 rdp->nxttail[i] = &rdp->nxtlist;
 }
 
 /*
+ * Initialize the specified rcu_data structure's callback list to empty.
+ */
+static void init_callback_list(struct rcu_data *rdp)
+{
+        if (init_nocb_callback_list(rdp))
+                return;
+        init_default_callback_list(rdp);
+}
+
+/*
  * Determine the value that ->completed will have at the end of the
  * next subsequent grace period.  This is used to tag callbacks so that
  * a CPU can invoke callbacks in a timely fashion even if that CPU has
@@ -1727,7 +1738,6 @@ static int rcu_gp_init(struct rcu_state *rsp)
         struct rcu_node *rnp = rcu_get_root(rsp);
 
         ACCESS_ONCE(rsp->gp_activity) = jiffies;
-        rcu_bind_gp_kthread();
         raw_spin_lock_irq(&rnp->lock);
         smp_mb__after_unlock_lock();
         if (!ACCESS_ONCE(rsp->gp_flags)) {
@@ -1822,8 +1832,8 @@ static int rcu_gp_init(struct rcu_state *rsp)
                 rcu_preempt_check_blocked_tasks(rnp);
                 rnp->qsmask = rnp->qsmaskinit;
                 ACCESS_ONCE(rnp->gpnum) = rsp->gpnum;
-                WARN_ON_ONCE(rnp->completed != rsp->completed);
-                ACCESS_ONCE(rnp->completed) = rsp->completed;
+                if (WARN_ON_ONCE(rnp->completed != rsp->completed))
+                        ACCESS_ONCE(rnp->completed) = rsp->completed;
                 if (rnp == rdp->mynode)
                         (void)__note_gp_changes(rsp, rnp, rdp);
                 rcu_preempt_boost_start_gp(rnp);
@@ -1866,7 +1876,7 @@ static int rcu_gp_fqs(struct rcu_state *rsp, int fqs_state_in)
                 fqs_state = RCU_FORCE_QS;
         } else {
                 /* Handle dyntick-idle and offline CPUs. */
-                isidle = false;
+                isidle = true;
                 force_qs_rnp(rsp, rcu_implicit_dynticks_qs, &isidle, &maxj);
         }
         /* Clear flag to prevent immediate re-entry. */
@@ -1965,6 +1975,7 @@ static int __noreturn rcu_gp_kthread(void *arg)
         struct rcu_state *rsp = arg;
         struct rcu_node *rnp = rcu_get_root(rsp);
 
+        rcu_bind_gp_kthread();
         for (;;) {
 
                 /* Handle grace-period start. */
@@ -2750,8 +2761,8 @@ static void force_qs_rnp(struct rcu_state *rsp,
                 bit = 1;
                 for (; cpu <= rnp->grphi; cpu++, bit <<= 1) {
                         if ((rnp->qsmask & bit) != 0) {
-                                if ((rnp->qsmaskinit & bit) != 0)
-                                        *isidle = false;
+                                if ((rnp->qsmaskinit & bit) == 0)
+                                        *isidle = false; /* Pending hotplug. */
                                 if (f(per_cpu_ptr(rsp->rda, cpu), isidle, maxj))
                                         mask |= bit;
                         }
@@ -2895,7 +2906,7 @@ static void __call_rcu_core(struct rcu_state *rsp, struct rcu_data *rdp,
          * If called from an extended quiescent state, invoke the RCU
          * core in order to force a re-evaluation of RCU's idleness.
          */
-        if (!rcu_is_watching() && cpu_online(smp_processor_id()))
+        if (!rcu_is_watching())
                 invoke_rcu_core();
 
         /* If interrupts were disabled or CPU offline, don't invoke RCU core. */
@@ -2981,11 +2992,22 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
 
                 if (cpu != -1)
                         rdp = per_cpu_ptr(rsp->rda, cpu);
-                offline = !__call_rcu_nocb(rdp, head, lazy, flags);
-                WARN_ON_ONCE(offline);
-                /* _call_rcu() is illegal on offline CPU; leak the callback. */
-                local_irq_restore(flags);
-                return;
+                if (likely(rdp->mynode)) {
+                        /* Post-boot, so this should be for a no-CBs CPU. */
+                        offline = !__call_rcu_nocb(rdp, head, lazy, flags);
+                        WARN_ON_ONCE(offline);
+                        /* Offline CPU, _call_rcu() illegal, leak callback.  */
+                        local_irq_restore(flags);
+                        return;
+                }
+                /*
+                 * Very early boot, before rcu_init().  Initialize if needed
+                 * and then drop through to queue the callback.
+                 */
+                BUG_ON(cpu != -1);
+                WARN_ON_ONCE(!rcu_is_watching());
+                if (!likely(rdp->nxtlist))
+                        init_default_callback_list(rdp);
         }
         ACCESS_ONCE(rdp->qlen) = rdp->qlen + 1;
         if (lazy)
@@ -3108,7 +3130,7 @@ void synchronize_sched(void)
                            "Illegal synchronize_sched() in RCU-sched read-side critical section");
         if (rcu_blocking_is_gp())
                 return;
-        if (rcu_expedited)
+        if (rcu_gp_is_expedited())
                 synchronize_sched_expedited();
         else
                 wait_rcu_gp(call_rcu_sched);
@@ -3135,7 +3157,7 @@ void synchronize_rcu_bh(void)
                            "Illegal synchronize_rcu_bh() in RCU-bh read-side critical section");
         if (rcu_blocking_is_gp())
                 return;
-        if (rcu_expedited)
+        if (rcu_gp_is_expedited())
                 synchronize_rcu_bh_expedited();
         else
                 wait_rcu_gp(call_rcu_bh);
@@ -3735,7 +3757,8 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
         rdp->qlen_last_fqs_check = 0;
         rdp->n_force_qs_snap = rsp->n_force_qs;
         rdp->blimit = blimit;
-        init_callback_list(rdp);  /* Re-enable callbacks on this CPU. */
+        if (!rdp->nxtlist)
+                init_callback_list(rdp);  /* Re-enable callbacks on this CPU. */
         rdp->dynticks->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
         rcu_sysidle_init_percpu_data(rdp->dynticks);
         atomic_set(&rdp->dynticks->dynticks,
@@ -3826,11 +3849,12 @@ static int rcu_pm_notify(struct notifier_block *self,
         case PM_HIBERNATION_PREPARE:
         case PM_SUSPEND_PREPARE:
                 if (nr_cpu_ids <= 256) /* Expediting bad for large systems. */
-                        rcu_expedited = 1;
+                        rcu_expedite_gp();
                 break;
         case PM_POST_HIBERNATION:
         case PM_POST_SUSPEND:
-                rcu_expedited = 0;
+                if (nr_cpu_ids <= 256) /* Expediting bad for large systems. */
+                        rcu_unexpedite_gp();
                 break;
         default:
                 break;
@@ -3900,30 +3924,26 @@ void rcu_scheduler_starting(void)
  * Compute the per-level fanout, either using the exact fanout specified
  * or balancing the tree, depending on CONFIG_RCU_FANOUT_EXACT.
  */
-#ifdef CONFIG_RCU_FANOUT_EXACT
 static void __init rcu_init_levelspread(struct rcu_state *rsp)
 {
         int i;
 
-        rsp->levelspread[rcu_num_lvls - 1] = rcu_fanout_leaf;
-        for (i = rcu_num_lvls - 2; i >= 0; i--)
-                rsp->levelspread[i] = CONFIG_RCU_FANOUT;
-}
-#else /* #ifdef CONFIG_RCU_FANOUT_EXACT */
-static void __init rcu_init_levelspread(struct rcu_state *rsp)
-{
-        int ccur;
-        int cprv;
-        int i;
-
-        cprv = nr_cpu_ids;
-        for (i = rcu_num_lvls - 1; i >= 0; i--) {
-                ccur = rsp->levelcnt[i];
-                rsp->levelspread[i] = (cprv + ccur - 1) / ccur;
-                cprv = ccur;
+        if (IS_ENABLED(CONFIG_RCU_FANOUT_EXACT)) {
+                rsp->levelspread[rcu_num_lvls - 1] = rcu_fanout_leaf;
+                for (i = rcu_num_lvls - 2; i >= 0; i--)
+                        rsp->levelspread[i] = CONFIG_RCU_FANOUT;
+        } else {
+                int ccur;
+                int cprv;
+
+                cprv = nr_cpu_ids;
+                for (i = rcu_num_lvls - 1; i >= 0; i--) {
+                        ccur = rsp->levelcnt[i];
+                        rsp->levelspread[i] = (cprv + ccur - 1) / ccur;
+                        cprv = ccur;
+                }
         }
 }
-#endif /* #else #ifdef CONFIG_RCU_FANOUT_EXACT */
 
 /*
  * Helper function for rcu_init() that initializes one rcu_state structure.
@@ -3999,7 +4019,6 @@ static void __init rcu_init_one(struct rcu_state *rsp,
                 }
         }
 
-        rsp->rda = rda;
         init_waitqueue_head(&rsp->gp_wq);
         rnp = rsp->level[rcu_num_lvls - 1];
         for_each_possible_cpu(i) {
@@ -4092,6 +4111,8 @@ void __init rcu_init(void)
 {
         int cpu;
 
+        rcu_early_boot_tests();
+
         rcu_bootup_announce();
         rcu_init_geometry();
         rcu_init_one(&rcu_bh_state, &rcu_bh_data);
@@ -4108,8 +4129,6 @@ void __init rcu_init(void)
         pm_notifier(rcu_pm_notify, 0);
         for_each_online_cpu(cpu)
                 rcu_cpu_notify(NULL, CPU_UP_PREPARE, (void *)(long)cpu);
-
-        rcu_early_boot_tests();
 }
 
 #include "tree_plugin.h"
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index d45e961515c1..8c0ec0f5a027 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -58,38 +58,33 @@ static bool __read_mostly rcu_nocb_poll;    /* Offload kthread are to poll. */
  */
 static void __init rcu_bootup_announce_oddness(void)
 {
-#ifdef CONFIG_RCU_TRACE
-        pr_info("\tRCU debugfs-based tracing is enabled.\n");
-#endif
-#if (defined(CONFIG_64BIT) && CONFIG_RCU_FANOUT != 64) || (!defined(CONFIG_64BIT) && CONFIG_RCU_FANOUT != 32)
-        pr_info("\tCONFIG_RCU_FANOUT set to non-default value of %d\n",
-               CONFIG_RCU_FANOUT);
-#endif
-#ifdef CONFIG_RCU_FANOUT_EXACT
-        pr_info("\tHierarchical RCU autobalancing is disabled.\n");
-#endif
-#ifdef CONFIG_RCU_FAST_NO_HZ
-        pr_info("\tRCU dyntick-idle grace-period acceleration is enabled.\n");
-#endif
-#ifdef CONFIG_PROVE_RCU
-        pr_info("\tRCU lockdep checking is enabled.\n");
-#endif
-#ifdef CONFIG_RCU_TORTURE_TEST_RUNNABLE
-        pr_info("\tRCU torture testing starts during boot.\n");
-#endif
-#if defined(CONFIG_RCU_CPU_STALL_INFO)
-        pr_info("\tAdditional per-CPU info printed with stalls.\n");
-#endif
-#if NUM_RCU_LVL_4 != 0
-        pr_info("\tFour-level hierarchy is enabled.\n");
-#endif
+        if (IS_ENABLED(CONFIG_RCU_TRACE))
+                pr_info("\tRCU debugfs-based tracing is enabled.\n");
+        if ((IS_ENABLED(CONFIG_64BIT) && CONFIG_RCU_FANOUT != 64) ||
+            (!IS_ENABLED(CONFIG_64BIT) && CONFIG_RCU_FANOUT != 32))
+                pr_info("\tCONFIG_RCU_FANOUT set to non-default value of %d\n",
+                       CONFIG_RCU_FANOUT);
+        if (IS_ENABLED(CONFIG_RCU_FANOUT_EXACT))
+                pr_info("\tHierarchical RCU autobalancing is disabled.\n");
+        if (IS_ENABLED(CONFIG_RCU_FAST_NO_HZ))
+                pr_info("\tRCU dyntick-idle grace-period acceleration is enabled.\n");
+        if (IS_ENABLED(CONFIG_PROVE_RCU))
+                pr_info("\tRCU lockdep checking is enabled.\n");
+        if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_RUNNABLE))
+                pr_info("\tRCU torture testing starts during boot.\n");
+        if (IS_ENABLED(CONFIG_RCU_CPU_STALL_INFO))
+                pr_info("\tAdditional per-CPU info printed with stalls.\n");
+        if (NUM_RCU_LVL_4 != 0)
+                pr_info("\tFour-level hierarchy is enabled.\n");
+        if (CONFIG_RCU_FANOUT_LEAF != 16)
+                pr_info("\tBuild-time adjustment of leaf fanout to %d.\n",
+                        CONFIG_RCU_FANOUT_LEAF);
         if (rcu_fanout_leaf != CONFIG_RCU_FANOUT_LEAF)
                 pr_info("\tBoot-time adjustment of leaf fanout to %d.\n", rcu_fanout_leaf);
         if (nr_cpu_ids != NR_CPUS)
                 pr_info("\tRCU restricting CPUs from NR_CPUS=%d to nr_cpu_ids=%d.\n", NR_CPUS, nr_cpu_ids);
-#ifdef CONFIG_RCU_BOOST
-        pr_info("\tRCU kthread priority: %d.\n", kthread_prio);
-#endif
+        if (IS_ENABLED(CONFIG_RCU_BOOST))
+                pr_info("\tRCU kthread priority: %d.\n", kthread_prio);
 }
 
 #ifdef CONFIG_PREEMPT_RCU
@@ -296,7 +291,13 @@ void rcu_read_unlock_special(struct task_struct *t)
         }
 
         /* Hardware IRQ handlers cannot block, complain if they get here. */
-        if (WARN_ON_ONCE(in_irq() || in_serving_softirq())) {
+        if (in_irq() || in_serving_softirq()) {
+                lockdep_rcu_suspicious(__FILE__, __LINE__,
+                                       "rcu_read_unlock() from irq or softirq with blocking in critical section!!!\n");
+                pr_alert("->rcu_read_unlock_special: %#x (b: %d, nq: %d)\n",
+                         t->rcu_read_unlock_special.s,
+                         t->rcu_read_unlock_special.b.blocked,
+                         t->rcu_read_unlock_special.b.need_qs);
                 local_irq_restore(flags);
                 return;
         }
@@ -535,7 +536,7 @@ void synchronize_rcu(void)
                            "Illegal synchronize_rcu() in RCU read-side critical section");
         if (!rcu_scheduler_active)
                 return;
-        if (rcu_expedited)
+        if (rcu_gp_is_expedited())
                 synchronize_rcu_expedited();
         else
                 wait_rcu_gp(call_rcu);
@@ -1940,7 +1941,8 @@ static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu)
                 rhp = ACCESS_ONCE(rdp->nocb_follower_head);
 
         /* Having no rcuo kthread but CBs after scheduler starts is bad! */
-        if (!ACCESS_ONCE(rdp->nocb_kthread) && rhp) {
+        if (!ACCESS_ONCE(rdp->nocb_kthread) && rhp &&
+            rcu_scheduler_fully_active) {
                 /* RCU callback enqueued before CPU first came online??? */
                 pr_err("RCU: Never-onlined no-CBs CPU %d has CB %p\n",
                        cpu, rhp->func);
@@ -2387,18 +2389,8 @@ void __init rcu_init_nohz(void)
                 pr_info("\tPoll for callbacks from no-CBs CPUs.\n");
 
         for_each_rcu_flavor(rsp) {
-                for_each_cpu(cpu, rcu_nocb_mask) {
-                        struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
-
-                        /*
-                         * If there are early callbacks, they will need
-                         * to be moved to the nocb lists.
-                         */
-                        WARN_ON_ONCE(rdp->nxttail[RCU_NEXT_TAIL] !=
-                                     &rdp->nxtlist &&
-                                     rdp->nxttail[RCU_NEXT_TAIL] != NULL);
-                        init_nocb_callback_list(rdp);
-                }
+                for_each_cpu(cpu, rcu_nocb_mask)
+                        init_nocb_callback_list(per_cpu_ptr(rsp->rda, cpu));
                 rcu_organize_nocb_kthreads(rsp);
         }
 }
@@ -2535,6 +2527,16 @@ static bool init_nocb_callback_list(struct rcu_data *rdp)
         if (!rcu_is_nocb_cpu(rdp->cpu))
                 return false;
 
+        /* If there are early-boot callbacks, move them to nocb lists. */
+        if (rdp->nxtlist) {
+                rdp->nocb_head = rdp->nxtlist;
+                rdp->nocb_tail = rdp->nxttail[RCU_NEXT_TAIL];
+                atomic_long_set(&rdp->nocb_q_count, rdp->qlen);
+                atomic_long_set(&rdp->nocb_q_count_lazy, rdp->qlen_lazy);
+                rdp->nxtlist = NULL;
+                rdp->qlen = 0;
+                rdp->qlen_lazy = 0;
+        }
         rdp->nxttail[RCU_NEXT_TAIL] = NULL;
         return true;
 }
@@ -2758,7 +2760,8 @@ static void rcu_sysidle_exit(int irq)
 
 /*
  * Check to see if the current CPU is idle.  Note that usermode execution
- * does not count as idle.  The caller must have disabled interrupts.
+ * does not count as idle.  The caller must have disabled interrupts,
+ * and must be running on tick_do_timer_cpu.
  */
 static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
                                   unsigned long *maxj)
@@ -2779,8 +2782,8 @@ static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
         if (!*isidle || rdp->rsp != rcu_state_p ||
             cpu_is_offline(rdp->cpu) || rdp->cpu == tick_do_timer_cpu)
                 return;
-        if (rcu_gp_in_progress(rdp->rsp))
-                WARN_ON_ONCE(smp_processor_id() != tick_do_timer_cpu);
+        /* Verify affinity of current kthread. */
+        WARN_ON_ONCE(smp_processor_id() != tick_do_timer_cpu);
 
         /* Pick up current idle and NMI-nesting counter and check. */
         cur = atomic_read(&rdtp->dynticks_idle);
@@ -3063,11 +3066,10 @@ static void rcu_bind_gp_kthread(void)
                 return;
 #ifdef CONFIG_NO_HZ_FULL_SYSIDLE
         cpu = tick_do_timer_cpu;
-        if (cpu >= 0 && cpu < nr_cpu_ids && raw_smp_processor_id() != cpu)
+        if (cpu >= 0 && cpu < nr_cpu_ids)
                 set_cpus_allowed_ptr(current, cpumask_of(cpu));
 #else /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
-        if (!is_housekeeping_cpu(raw_smp_processor_id()))
-                housekeeping_affine(current);
+        housekeeping_affine(current);
 #endif /* #else #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
 }
 
diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c
index e0d31a345ee6..1f133350da01 100644
--- a/kernel/rcu/update.c
+++ b/kernel/rcu/update.c
@@ -62,6 +62,63 @@ MODULE_ALIAS("rcupdate");
 
 module_param(rcu_expedited, int, 0);
 
+#ifndef CONFIG_TINY_RCU
+
+static atomic_t rcu_expedited_nesting =
+        ATOMIC_INIT(IS_ENABLED(CONFIG_RCU_EXPEDITE_BOOT) ? 1 : 0);
+
+/*
+ * Should normal grace-period primitives be expedited?  Intended for
+ * use within RCU.  Note that this function takes the rcu_expedited
+ * sysfs/boot variable into account as well as the rcu_expedite_gp()
+ * nesting.  So looping on rcu_unexpedite_gp() until rcu_gp_is_expedited()
+ * returns false is a -really- bad idea.
+ */
+bool rcu_gp_is_expedited(void)
+{
+        return rcu_expedited || atomic_read(&rcu_expedited_nesting);
+}
+EXPORT_SYMBOL_GPL(rcu_gp_is_expedited);
+
+/**
+ * rcu_expedite_gp - Expedite future RCU grace periods
+ *
+ * After a call to this function, future calls to synchronize_rcu() and
+ * friends act as the corresponding synchronize_rcu_expedited() function
+ * had instead been called.
+ */
+void rcu_expedite_gp(void)
+{
+        atomic_inc(&rcu_expedited_nesting);
+}
+EXPORT_SYMBOL_GPL(rcu_expedite_gp);
+
+/**
+ * rcu_unexpedite_gp - Cancel prior rcu_expedite_gp() invocation
+ *
+ * Undo a prior call to rcu_expedite_gp().  If all prior calls to
+ * rcu_expedite_gp() are undone by a subsequent call to rcu_unexpedite_gp(),
+ * and if the rcu_expedited sysfs/boot parameter is not set, then all
+ * subsequent calls to synchronize_rcu() and friends will return to
+ * their normal non-expedited behavior.
+ */
+void rcu_unexpedite_gp(void)
+{
+        atomic_dec(&rcu_expedited_nesting);
+}
+EXPORT_SYMBOL_GPL(rcu_unexpedite_gp);
+
+#endif /* #ifndef CONFIG_TINY_RCU */
+
+/*
+ * Inform RCU of the end of the in-kernel boot sequence.
+ */
+void rcu_end_inkernel_boot(void)
+{
+        if (IS_ENABLED(CONFIG_RCU_EXPEDITE_BOOT))
+                rcu_unexpedite_gp();
+}
+
 #ifdef CONFIG_PREEMPT_RCU
 
 /*
@@ -199,16 +256,13 @@ EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held);
 
 #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
 
-struct rcu_synchronize {
-        struct rcu_head head;
-        struct completion completion;
-};
-
-/*
- * Awaken the corresponding synchronize_rcu() instance now that a
- * grace period has elapsed.
+/**
+ * wakeme_after_rcu() - Callback function to awaken a task after grace period
+ * @head: Pointer to rcu_head member within rcu_synchronize structure
+ *
+ * Awaken the corresponding task now that a grace period has elapsed.
  */
-static void wakeme_after_rcu(struct rcu_head  *head)
+void wakeme_after_rcu(struct rcu_head *head)
 {
         struct rcu_synchronize *rcu;
 
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 1173afc308ad..1ad74c0df01f 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -1180,16 +1180,7 @@ config DEBUG_CREDENTIALS
 menu "RCU Debugging"
 
 config PROVE_RCU
-        bool "RCU debugging: prove RCU correctness"
-        depends on PROVE_LOCKING
-        default n
-        help
-         This feature enables lockdep extensions that check for correct
-         use of RCU APIs.  This is currently under development.  Say Y
-         if you want to debug RCU usage or help work on the PROVE_RCU
-         feature.
-
-         Say N if you are unsure.
+        def_bool PROVE_LOCKING
 
 config PROVE_RCU_REPEATEDLY
         bool "RCU debugging: don't disable PROVE_RCU on first splat"
diff --git a/tools/testing/selftests/rcutorture/bin/kvm.sh b/tools/testing/selftests/rcutorture/bin/kvm.sh
index 368d64ac779e..dd2812ceb0ba 100755
--- a/tools/testing/selftests/rcutorture/bin/kvm.sh
+++ b/tools/testing/selftests/rcutorture/bin/kvm.sh
@@ -310,7 +310,7 @@ function dump(first, pastlast)
                         cfr[jn] = cf[j] "." cfrep[cf[j]];
                 }
                 if (cpusr[jn] > ncpus && ncpus != 0)
-                        ovf = "(!)";
+                        ovf = "-ovf";
                 else
                         ovf = "";
                 print "echo ", cfr[jn], cpusr[jn] ovf ": Starting build. `date`";