8 files changed, 545 insertions, 306 deletions
diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c
index 30d42aa55d83..8dbe27611ec3 100644
--- a/kernel/rcu/rcutorture.c
+++ b/kernel/rcu/rcutorture.c
@@ -853,6 +853,8 @@ rcu_torture_fqs(void *arg)
 static int
 rcu_torture_writer(void *arg)
 {
+        bool can_expedite = !rcu_gp_is_expedited();
+        int expediting = 0;
        unsigned long gp_snap;
        bool gp_cond1 = gp_cond, gp_exp1 = gp_exp, gp_normal1 = gp_normal;
        bool gp_sync1 = gp_sync;
@@ -865,9 +867,15 @@ rcu_torture_writer(void *arg)
        int nsynctypes = 0;
        VERBOSE_TOROUT_STRING("rcu_torture_writer task started");
+        pr_alert("%s" TORTURE_FLAG
+                 " Grace periods expedited from boot/sysfs for %s,\n",
+                 torture_type, cur_ops->name);
+        pr_alert("%s" TORTURE_FLAG
+                 " Testing of dynamic grace-period expediting diabled.\n",
+                 torture_type);
        /* Initialize synctype[] array.  If none set, take default. */
-        if (!gp_cond1 && !gp_exp1 && !gp_normal1 && !gp_sync)
+        if (!gp_cond1 && !gp_exp1 && !gp_normal1 && !gp_sync1)
                gp_cond1 = gp_exp1 = gp_normal1 = gp_sync1 = true;
        if (gp_cond1 && cur_ops->get_state && cur_ops->cond_sync)
                synctype[nsynctypes++] = RTWS_COND_GET;
@@ -949,9 +957,26 @@ rcu_torture_writer(void *arg)
                        }
                }
                rcutorture_record_progress(++rcu_torture_current_version);
+                /* Cycle through nesting levels of rcu_expedite_gp() calls. */
+                if (can_expedite &&
+                    !(torture_random(&rand) & 0xff & (!!expediting - 1))) {
+                        WARN_ON_ONCE(expediting == 0 && rcu_gp_is_expedited());
+                        if (expediting >= 0)
+                                rcu_expedite_gp();
+                        else
+                                rcu_unexpedite_gp();
+                        if (++expediting > 3)
+                                expediting = -expediting;
+                }
                rcu_torture_writer_state = RTWS_STUTTER;
                stutter_wait("rcu_torture_writer");
        } while (!torture_must_stop());
+        /* Reset expediting back to unexpedited. */
+        if (expediting > 0)
+                expediting = -expediting;
+        while (can_expedite && expediting++ < 0)
+                rcu_unexpedite_gp();
+        WARN_ON_ONCE(can_expedite && rcu_gp_is_expedited());
        rcu_torture_writer_state = RTWS_STOPPING;
        torture_kthread_stopping("rcu_torture_writer");
        return 0;
diff --git a/kernel/rcu/srcu.c b/kernel/rcu/srcu.c
index 445bf8ffe3fb..cad76e76b4e7 100644
--- a/kernel/rcu/srcu.c
+++ b/kernel/rcu/srcu.c
@@ -402,23 +402,6 @@ void call_srcu(struct srcu_struct *sp, struct rcu_head *head,
 }
 EXPORT_SYMBOL_GPL(call_srcu);
-struct rcu_synchronize {
-        struct rcu_head head;
-        struct completion completion;
-};
-/*
- * Awaken the corresponding synchronize_srcu() instance now that a
- * grace period has elapsed.
- */
-static void wakeme_after_rcu(struct rcu_head *head)
-{
-        struct rcu_synchronize *rcu;
-        rcu = container_of(head, struct rcu_synchronize, head);
-        complete(&rcu->completion);
-}
 static void srcu_advance_batches(struct srcu_struct *sp, int trycount);
 static void srcu_reschedule(struct srcu_struct *sp);
@@ -507,7 +490,7 @@ static void __synchronize_srcu(struct srcu_struct *sp, int trycount)
 */
 void synchronize_srcu(struct srcu_struct *sp)
 {
-        __synchronize_srcu(sp, rcu_expedited
+        __synchronize_srcu(sp, rcu_gp_is_expedited()
                           ? SYNCHRONIZE_SRCU_EXP_TRYCOUNT
                           : SYNCHRONIZE_SRCU_TRYCOUNT);
 }
diff --git a/kernel/rcu/tiny.c b/kernel/rcu/tiny.c
index cc9ceca7bde1..069742d61c68 100644
--- a/kernel/rcu/tiny.c
+++ b/kernel/rcu/tiny.c
@@ -103,8 +103,7 @@ EXPORT_SYMBOL(__rcu_is_watching);
 static int rcu_qsctr_help(struct rcu_ctrlblk *rcp)
 {
        RCU_TRACE(reset_cpu_stall_ticks(rcp));
-        if (rcp->rcucblist != NULL &&
+        if (rcp->donetail != rcp->curtail) {
-            rcp->donetail != rcp->curtail) {
                rcp->donetail = rcp->curtail;
                return 1;
        }
@@ -169,17 +168,6 @@ static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp)
        unsigned long flags;
        RCU_TRACE(int cb_count = 0);
-        /* If no RCU callbacks ready to invoke, just return. */
-        if (&rcp->rcucblist == rcp->donetail) {
-                RCU_TRACE(trace_rcu_batch_start(rcp->name, 0, 0, -1));
-                RCU_TRACE(trace_rcu_batch_end(rcp->name, 0,
-                                              !!ACCESS_ONCE(rcp->rcucblist),
-                                              need_resched(),
-                                              is_idle_task(current),
-                                              false));
-                return;
-        }
        /* Move the ready-to-invoke callbacks to a local list. */
        local_irq_save(flags);
        RCU_TRACE(trace_rcu_batch_start(rcp->name, 0, rcp->qlen, -1));
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 48d640ca1a05..233165da782f 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -91,8 +91,10 @@ static const char *tp_##sname##_varname __used __tracepoint_string = sname##_var
 #define RCU_STATE_INITIALIZER(sname, sabbr, cr) \
 DEFINE_RCU_TPS(sname) \
+DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, sname##_data); \
 struct rcu_state sname##_state = { \
        .level = { &sname##_state.node[0] }, \
+        .rda = &sname##_data, \
        .call = cr, \
        .fqs_state = RCU_GP_IDLE, \
        .gpnum = 0UL - 300UL, \
@@ -101,11 +103,9 @@ struct rcu_state sname##_state = { \
        .orphan_nxttail = &sname##_state.orphan_nxtlist, \
        .orphan_donetail = &sname##_state.orphan_donelist, \
        .barrier_mutex = __MUTEX_INITIALIZER(sname##_state.barrier_mutex), \
-        .onoff_mutex = __MUTEX_INITIALIZER(sname##_state.onoff_mutex), \
        .name = RCU_STATE_NAME(sname), \
        .abbr = sabbr, \
-}; \
+}
-DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, sname##_data)
 RCU_STATE_INITIALIZER(rcu_sched, 's', call_rcu_sched);
 RCU_STATE_INITIALIZER(rcu_bh, 'b', call_rcu_bh);
@@ -152,6 +152,8 @@ EXPORT_SYMBOL_GPL(rcu_scheduler_active);
 */
 static int rcu_scheduler_fully_active __read_mostly;
+static void rcu_init_new_rnp(struct rcu_node *rnp_leaf);
+static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf);
 static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu);
 static void invoke_rcu_core(void);
 static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp);
@@ -160,6 +162,12 @@ static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp);
 static int kthread_prio = CONFIG_RCU_KTHREAD_PRIO;
 module_param(kthread_prio, int, 0644);
+/* Delay in jiffies for grace-period initialization delays. */
+static int gp_init_delay = IS_ENABLED(CONFIG_RCU_TORTURE_TEST_SLOW_INIT)
+                                ? CONFIG_RCU_TORTURE_TEST_SLOW_INIT_DELAY
+                                : 0;
+module_param(gp_init_delay, int, 0644);
 /*
 * Track the rcutorture test sequence number and the update version
 * number within a given test.  The rcutorture_testseq is incremented
@@ -173,6 +181,17 @@ unsigned long rcutorture_testseq;
 unsigned long rcutorture_vernum;
 /*
+ * Compute the mask of online CPUs for the specified rcu_node structure.
+ * This will not be stable unless the rcu_node structure's ->lock is
+ * held, but the bit corresponding to the current CPU will be stable
+ * in most contexts.
+ */
+unsigned long rcu_rnp_online_cpus(struct rcu_node *rnp)
+{
+        return ACCESS_ONCE(rnp->qsmaskinitnext);
+}
+/*
 * Return true if an RCU grace period is in progress.  The ACCESS_ONCE()s
 * permit this function to be invoked without holding the root rcu_node
 * structure's ->lock, but of course results can be subject to change.
@@ -292,10 +311,10 @@ void rcu_note_context_switch(void)
 EXPORT_SYMBOL_GPL(rcu_note_context_switch);
 /*
- * Register a quiesecent state for all RCU flavors.  If there is an
+ * Register a quiescent state for all RCU flavors.  If there is an
 * emergency, invoke rcu_momentary_dyntick_idle() to do a heavy-weight
 * dyntick-idle quiescent state visible to other CPUs (but only for those
- * RCU flavors in desparate need of a quiescent state, which will normally
+ * RCU flavors in desperate need of a quiescent state, which will normally
 * be none of them).  Either way, do a lightweight quiescent state for
 * all RCU flavors.
 */
@@ -410,6 +429,15 @@ void rcu_bh_force_quiescent_state(void)
 EXPORT_SYMBOL_GPL(rcu_bh_force_quiescent_state);
 /*
+ * Force a quiescent state for RCU-sched.
+ */
+void rcu_sched_force_quiescent_state(void)
+{
+        force_quiescent_state(&rcu_sched_state);
+}
+EXPORT_SYMBOL_GPL(rcu_sched_force_quiescent_state);
+/*
 * Show the state of the grace-period kthreads.
 */
 void show_rcu_gp_kthreads(void)
@@ -483,15 +511,6 @@ void rcutorture_record_progress(unsigned long vernum)
 EXPORT_SYMBOL_GPL(rcutorture_record_progress);
 /*
- * Force a quiescent state for RCU-sched.
- */
-void rcu_sched_force_quiescent_state(void)
-{
-        force_quiescent_state(&rcu_sched_state);
-}
-EXPORT_SYMBOL_GPL(rcu_sched_force_quiescent_state);
-/*
 * Does the CPU have callbacks ready to be invoked?
 */
 static int
@@ -954,7 +973,7 @@ bool rcu_lockdep_current_cpu_online(void)
        preempt_disable();
        rdp = this_cpu_ptr(&rcu_sched_data);
        rnp = rdp->mynode;
-        ret = (rdp->grpmask & rnp->qsmaskinit) ||
+        ret = (rdp->grpmask & rcu_rnp_online_cpus(rnp)) ||
              !rcu_scheduler_fully_active;
        preempt_enable();
        return ret;
@@ -1196,9 +1215,10 @@ static void print_other_cpu_stall(struct rcu_state *rsp, unsigned long gpnum)
                } else {
                        j = jiffies;
                        gpa = ACCESS_ONCE(rsp->gp_activity);
-                        pr_err("All QSes seen, last %s kthread activity %ld (%ld-%ld), jiffies_till_next_fqs=%ld\n",
+                        pr_err("All QSes seen, last %s kthread activity %ld (%ld-%ld), jiffies_till_next_fqs=%ld, root ->qsmask %#lx\n",
                               rsp->name, j - gpa, j, gpa,
-                               jiffies_till_next_fqs);
+                               jiffies_till_next_fqs,
+                               rcu_get_root(rsp)->qsmask);
                        /* In this case, the current CPU might be at fault. */
                        sched_show_task(current);
                }
@@ -1328,20 +1348,30 @@ void rcu_cpu_stall_reset(void)
 }
 /*
- * Initialize the specified rcu_data structure's callback list to empty.
+ * Initialize the specified rcu_data structure's default callback list
+ * to empty.  The default callback list is the one that is not used by
+ * no-callbacks CPUs.
 */
-static void init_callback_list(struct rcu_data *rdp)
+static void init_default_callback_list(struct rcu_data *rdp)
 {
        int i;
-        if (init_nocb_callback_list(rdp))
-                return;
        rdp->nxtlist = NULL;
        for (i = 0; i < RCU_NEXT_SIZE; i++)
                rdp->nxttail[i] = &rdp->nxtlist;
 }
 /*
+ * Initialize the specified rcu_data structure's callback list to empty.
+ */
+static void init_callback_list(struct rcu_data *rdp)
+{
+        if (init_nocb_callback_list(rdp))
+                return;
+        init_default_callback_list(rdp);
+}
+/*
 * Determine the value that ->completed will have at the end of the
 * next subsequent grace period.  This is used to tag callbacks so that
 * a CPU can invoke callbacks in a timely fashion even if that CPU has
@@ -1703,11 +1733,11 @@ static void note_gp_changes(struct rcu_state *rsp, struct rcu_data *rdp)
 */
 static int rcu_gp_init(struct rcu_state *rsp)
 {
+        unsigned long oldmask;
        struct rcu_data *rdp;
        struct rcu_node *rnp = rcu_get_root(rsp);
        ACCESS_ONCE(rsp->gp_activity) = jiffies;
-        rcu_bind_gp_kthread();
        raw_spin_lock_irq(&rnp->lock);
        smp_mb__after_unlock_lock();
        if (!ACCESS_ONCE(rsp->gp_flags)) {
@@ -1733,9 +1763,54 @@ static int rcu_gp_init(struct rcu_state *rsp)
        trace_rcu_grace_period(rsp->name, rsp->gpnum, TPS("start"));
        raw_spin_unlock_irq(&rnp->lock);
-        /* Exclude any concurrent CPU-hotplug operations. */
+        /*
-        mutex_lock(&rsp->onoff_mutex);
+         * Apply per-leaf buffered online and offline operations to the
-        smp_mb__after_unlock_lock(); /* ->gpnum increment before GP! */
+         * rcu_node tree.  Note that this new grace period need not wait
+         * for subsequent online CPUs, and that quiescent-state forcing
+         * will handle subsequent offline CPUs.
+         */
+        rcu_for_each_leaf_node(rsp, rnp) {
+                raw_spin_lock_irq(&rnp->lock);
+                smp_mb__after_unlock_lock();
+                if (rnp->qsmaskinit == rnp->qsmaskinitnext &&
+                    !rnp->wait_blkd_tasks) {
+                        /* Nothing to do on this leaf rcu_node structure. */
+                        raw_spin_unlock_irq(&rnp->lock);
+                        continue;
+                }
+                /* Record old state, apply changes to ->qsmaskinit field. */
+                oldmask = rnp->qsmaskinit;
+                rnp->qsmaskinit = rnp->qsmaskinitnext;
+                /* If zero-ness of ->qsmaskinit changed, propagate up tree. */
+                if (!oldmask != !rnp->qsmaskinit) {
+                        if (!oldmask) /* First online CPU for this rcu_node. */
+                                rcu_init_new_rnp(rnp);
+                        else if (rcu_preempt_has_tasks(rnp)) /* blocked tasks */
+                                rnp->wait_blkd_tasks = true;
+                        else /* Last offline CPU and can propagate. */
+                                rcu_cleanup_dead_rnp(rnp);
+                }
+                /*
+                 * If all waited-on tasks from prior grace period are
+                 * done, and if all this rcu_node structure's CPUs are
+                 * still offline, propagate up the rcu_node tree and
+                 * clear ->wait_blkd_tasks.  Otherwise, if one of this
+                 * rcu_node structure's CPUs has since come back online,
+                 * simply clear ->wait_blkd_tasks (but rcu_cleanup_dead_rnp()
+                 * checks for this, so just call it unconditionally).
+                 */
+                if (rnp->wait_blkd_tasks &&
+                    (!rcu_preempt_has_tasks(rnp) ||
+                     rnp->qsmaskinit)) {
+                        rnp->wait_blkd_tasks = false;
+                        rcu_cleanup_dead_rnp(rnp);
+                }
+                raw_spin_unlock_irq(&rnp->lock);
+        }
        /*
         * Set the quiescent-state-needed bits in all the rcu_node
@@ -1757,8 +1832,8 @@ static int rcu_gp_init(struct rcu_state *rsp)
                rcu_preempt_check_blocked_tasks(rnp);
                rnp->qsmask = rnp->qsmaskinit;
                ACCESS_ONCE(rnp->gpnum) = rsp->gpnum;
-                WARN_ON_ONCE(rnp->completed != rsp->completed);
+                if (WARN_ON_ONCE(rnp->completed != rsp->completed))
-                ACCESS_ONCE(rnp->completed) = rsp->completed;
+                        ACCESS_ONCE(rnp->completed) = rsp->completed;
                if (rnp == rdp->mynode)
                        (void)__note_gp_changes(rsp, rnp, rdp);
                rcu_preempt_boost_start_gp(rnp);
@@ -1768,9 +1843,12 @@ static int rcu_gp_init(struct rcu_state *rsp)
                raw_spin_unlock_irq(&rnp->lock);
                cond_resched_rcu_qs();
                ACCESS_ONCE(rsp->gp_activity) = jiffies;
+                if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_SLOW_INIT) &&
+                    gp_init_delay > 0 &&
+                    !(rsp->gpnum % (rcu_num_nodes * 10)))
+                        schedule_timeout_uninterruptible(gp_init_delay);
        }
-        mutex_unlock(&rsp->onoff_mutex);
        return 1;
 }
@@ -1798,7 +1876,7 @@ static int rcu_gp_fqs(struct rcu_state *rsp, int fqs_state_in)
                fqs_state = RCU_FORCE_QS;
        } else {
                /* Handle dyntick-idle and offline CPUs. */
-                isidle = false;
+                isidle = true;
                force_qs_rnp(rsp, rcu_implicit_dynticks_qs, &isidle, &maxj);
        }
        /* Clear flag to prevent immediate re-entry. */
@@ -1852,6 +1930,8 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
        rcu_for_each_node_breadth_first(rsp, rnp) {
                raw_spin_lock_irq(&rnp->lock);
                smp_mb__after_unlock_lock();
+                WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp));
+                WARN_ON_ONCE(rnp->qsmask);
                ACCESS_ONCE(rnp->completed) = rsp->gpnum;
                rdp = this_cpu_ptr(rsp->rda);
                if (rnp == rdp->mynode)
@@ -1895,6 +1975,7 @@ static int __noreturn rcu_gp_kthread(void *arg)
        struct rcu_state *rsp = arg;
        struct rcu_node *rnp = rcu_get_root(rsp);
+        rcu_bind_gp_kthread();
        for (;;) {
                /* Handle grace-period start. */
@@ -2062,25 +2143,32 @@ static void rcu_report_qs_rsp(struct rcu_state *rsp, unsigned long flags)
 * Similar to rcu_report_qs_rdp(), for which it is a helper function.
 * Allows quiescent states for a group of CPUs to be reported at one go
 * to the specified rcu_node structure, though all the CPUs in the group
- * must be represented by the same rcu_node structure (which need not be
+ * must be represented by the same rcu_node structure (which need not be a
- * a leaf rcu_node structure, though it often will be).  That structure's
+ * leaf rcu_node structure, though it often will be).  The gps parameter
- * lock must be held upon entry, and it is released before return.
+ * is the grace-period snapshot, which means that the quiescent states
+ * are valid only if rnp->gpnum is equal to gps.  That structure's lock
+ * must be held upon entry, and it is released before return.
 */
 static void
 rcu_report_qs_rnp(unsigned long mask, struct rcu_state *rsp,
-                  struct rcu_node *rnp, unsigned long flags)
+                  struct rcu_node *rnp, unsigned long gps, unsigned long flags)
        __releases(rnp->lock)
 {
+        unsigned long oldmask = 0;
        struct rcu_node *rnp_c;
        /* Walk up the rcu_node hierarchy. */
        for (;;) {
-                if (!(rnp->qsmask & mask)) {
+                if (!(rnp->qsmask & mask) || rnp->gpnum != gps) {
-                        /* Our bit has already been cleared, so done. */
+                        /*
+                         * Our bit has already been cleared, or the
+                         * relevant grace period is already over, so done.
+                         */
                        raw_spin_unlock_irqrestore(&rnp->lock, flags);
                        return;
                }
+                WARN_ON_ONCE(oldmask); /* Any child must be all zeroed! */
                rnp->qsmask &= ~mask;
                trace_rcu_quiescent_state_report(rsp->name, rnp->gpnum,
                                                 mask, rnp->qsmask, rnp->level,
@@ -2104,7 +2192,7 @@ rcu_report_qs_rnp(unsigned long mask, struct rcu_state *rsp,
                rnp = rnp->parent;
                raw_spin_lock_irqsave(&rnp->lock, flags);
                smp_mb__after_unlock_lock();
-                WARN_ON_ONCE(rnp_c->qsmask);
+                oldmask = rnp_c->qsmask;
        }
        /*
@@ -2116,6 +2204,46 @@ rcu_report_qs_rnp(unsigned long mask, struct rcu_state *rsp,
 }
 /*
+ * Record a quiescent state for all tasks that were previously queued
+ * on the specified rcu_node structure and that were blocking the current
+ * RCU grace period.  The caller must hold the specified rnp->lock with
+ * irqs disabled, and this lock is released upon return, but irqs remain
+ * disabled.
+ */
+static void rcu_report_unblock_qs_rnp(struct rcu_state *rsp,
+                                      struct rcu_node *rnp, unsigned long flags)
+        __releases(rnp->lock)
+{
+        unsigned long gps;
+        unsigned long mask;
+        struct rcu_node *rnp_p;
+        if (rcu_state_p == &rcu_sched_state || rsp != rcu_state_p ||
+            rnp->qsmask != 0 || rcu_preempt_blocked_readers_cgp(rnp)) {
+                raw_spin_unlock_irqrestore(&rnp->lock, flags);
+                return;  /* Still need more quiescent states! */
+        }
+        rnp_p = rnp->parent;
+        if (rnp_p == NULL) {
+                /*
+                 * Only one rcu_node structure in the tree, so don't
+                 * try to report up to its nonexistent parent!
+                 */
+                rcu_report_qs_rsp(rsp, flags);
+                return;
+        }
+        /* Report up the rest of the hierarchy, tracking current ->gpnum. */
+        gps = rnp->gpnum;
+        mask = rnp->grpmask;
+        raw_spin_unlock(&rnp->lock);    /* irqs remain disabled. */
+        raw_spin_lock(&rnp_p->lock);    /* irqs already disabled. */
+        smp_mb__after_unlock_lock();
+        rcu_report_qs_rnp(mask, rsp, rnp_p, gps, flags);
+}
+/*
 * Record a quiescent state for the specified CPU to that CPU's rcu_data
 * structure.  This must be either called from the specified CPU, or
 * called when the specified CPU is known to be offline (and when it is
@@ -2163,7 +2291,8 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp)
                 */
                needwake = rcu_accelerate_cbs(rsp, rnp, rdp);
-                rcu_report_qs_rnp(mask, rsp, rnp, flags); /* rlses rnp->lock */
+                rcu_report_qs_rnp(mask, rsp, rnp, rnp->gpnum, flags);
+                /* ^^^ Released rnp->lock */
                if (needwake)
                        rcu_gp_kthread_wake(rsp);
        }
@@ -2256,8 +2385,12 @@ rcu_send_cbs_to_orphanage(int cpu, struct rcu_state *rsp,
                rsp->orphan_donetail = rdp->nxttail[RCU_DONE_TAIL];
        }
-        /* Finally, initialize the rcu_data structure's list to empty.  */
+        /*
+         * Finally, initialize the rcu_data structure's list to empty and
+         * disallow further callbacks on this CPU.
+         */
        init_callback_list(rdp);
+        rdp->nxttail[RCU_NEXT_TAIL] = NULL;
 }
 /*
@@ -2355,6 +2488,7 @@ static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf)
                raw_spin_lock(&rnp->lock); /* irqs already disabled. */
                smp_mb__after_unlock_lock(); /* GP memory ordering. */
                rnp->qsmaskinit &= ~mask;
+                rnp->qsmask &= ~mask;
                if (rnp->qsmaskinit) {
                        raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
                        return;
@@ -2364,6 +2498,26 @@ static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf)
 }
 /*
+ * The CPU is exiting the idle loop into the arch_cpu_idle_dead()
+ * function.  We now remove it from the rcu_node tree's ->qsmaskinit
+ * bit masks.
+ */
+static void rcu_cleanup_dying_idle_cpu(int cpu, struct rcu_state *rsp)
+{
+        unsigned long flags;
+        unsigned long mask;
+        struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
+        struct rcu_node *rnp = rdp->mynode;  /* Outgoing CPU's rdp & rnp. */
+        /* Remove outgoing CPU from mask in the leaf rcu_node structure. */
+        mask = rdp->grpmask;
+        raw_spin_lock_irqsave(&rnp->lock, flags);
+        smp_mb__after_unlock_lock();    /* Enforce GP memory-order guarantee. */
+        rnp->qsmaskinitnext &= ~mask;
+        raw_spin_unlock_irqrestore(&rnp->lock, flags);
+}
+/*
 * The CPU has been completely removed, and some other CPU is reporting
 * this fact from process context.  Do the remainder of the cleanup,
 * including orphaning the outgoing CPU's RCU callbacks, and also
@@ -2379,29 +2533,15 @@ static void rcu_cleanup_dead_cpu(int cpu, struct rcu_state *rsp)
        /* Adjust any no-longer-needed kthreads. */
        rcu_boost_kthread_setaffinity(rnp, -1);
-        /* Exclude any attempts to start a new grace period. */
-        mutex_lock(&rsp->onoff_mutex);
-        raw_spin_lock_irqsave(&rsp->orphan_lock, flags);
        /* Orphan the dead CPU's callbacks, and adopt them if appropriate. */
+        raw_spin_lock_irqsave(&rsp->orphan_lock, flags);
        rcu_send_cbs_to_orphanage(cpu, rsp, rnp, rdp);
        rcu_adopt_orphan_cbs(rsp, flags);
        raw_spin_unlock_irqrestore(&rsp->orphan_lock, flags);
-        /* Remove outgoing CPU from mask in the leaf rcu_node structure. */
-        raw_spin_lock_irqsave(&rnp->lock, flags);
-        smp_mb__after_unlock_lock();    /* Enforce GP memory-order guarantee. */
-        rnp->qsmaskinit &= ~rdp->grpmask;
-        if (rnp->qsmaskinit == 0 && !rcu_preempt_has_tasks(rnp))
-                rcu_cleanup_dead_rnp(rnp);
-        rcu_report_qs_rnp(rdp->grpmask, rsp, rnp, flags); /* Rlses rnp->lock. */
        WARN_ONCE(rdp->qlen != 0 || rdp->nxtlist != NULL,
                  "rcu_cleanup_dead_cpu: Callbacks on offline CPU %d: qlen=%lu, nxtlist=%p\n",
                  cpu, rdp->qlen, rdp->nxtlist);
-        init_callback_list(rdp);
-        /* Disallow further callbacks on this CPU. */
-        rdp->nxttail[RCU_NEXT_TAIL] = NULL;
-        mutex_unlock(&rsp->onoff_mutex);
 }
 #else /* #ifdef CONFIG_HOTPLUG_CPU */
@@ -2414,6 +2554,10 @@ static void __maybe_unused rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf)
 {
 }
+static void rcu_cleanup_dying_idle_cpu(int cpu, struct rcu_state *rsp)
+{
+}
 static void rcu_cleanup_dead_cpu(int cpu, struct rcu_state *rsp)
 {
 }
@@ -2589,26 +2733,47 @@ static void force_qs_rnp(struct rcu_state *rsp,
                        return;
                }
                if (rnp->qsmask == 0) {
-                        rcu_initiate_boost(rnp, flags); /* releases rnp->lock */
+                        if (rcu_state_p == &rcu_sched_state ||
-                        continue;
+                            rsp != rcu_state_p ||
+                            rcu_preempt_blocked_readers_cgp(rnp)) {
+                                /*
+                                 * No point in scanning bits because they
+                                 * are all zero.  But we might need to
+                                 * priority-boost blocked readers.
+                                 */
+                                rcu_initiate_boost(rnp, flags);
+                                /* rcu_initiate_boost() releases rnp->lock */
+                                continue;
+                        }
+                        if (rnp->parent &&
+                            (rnp->parent->qsmask & rnp->grpmask)) {
+                                /*
+                                 * Race between grace-period
+                                 * initialization and task exiting RCU
+                                 * read-side critical section: Report.
+                                 */
+                                rcu_report_unblock_qs_rnp(rsp, rnp, flags);
+                                /* rcu_report_unblock_qs_rnp() rlses ->lock */
+                                continue;
+                        }
                }
                cpu = rnp->grplo;
                bit = 1;
                for (; cpu <= rnp->grphi; cpu++, bit <<= 1) {
                        if ((rnp->qsmask & bit) != 0) {
-                                if ((rnp->qsmaskinit & bit) != 0)
+                                if ((rnp->qsmaskinit & bit) == 0)
-                                        *isidle = false;
+                                        *isidle = false; /* Pending hotplug. */
                                if (f(per_cpu_ptr(rsp->rda, cpu), isidle, maxj))
                                        mask |= bit;
                        }
                }
                if (mask != 0) {
+                        /* Idle/offline CPUs, report (releases rnp->lock. */
-                        /* rcu_report_qs_rnp() releases rnp->lock. */
+                        rcu_report_qs_rnp(mask, rsp, rnp, rnp->gpnum, flags);
-                        rcu_report_qs_rnp(mask, rsp, rnp, flags);
+                } else {
-                        continue;
+                        /* Nothing to do here, so just drop the lock. */
+                        raw_spin_unlock_irqrestore(&rnp->lock, flags);
                }
-                raw_spin_unlock_irqrestore(&rnp->lock, flags);
        }
 }
@@ -2741,7 +2906,7 @@ static void __call_rcu_core(struct rcu_state *rsp, struct rcu_data *rdp,
         * If called from an extended quiescent state, invoke the RCU
         * core in order to force a re-evaluation of RCU's idleness.
         */
-        if (!rcu_is_watching() && cpu_online(smp_processor_id()))
+        if (!rcu_is_watching())
                invoke_rcu_core();
        /* If interrupts were disabled or CPU offline, don't invoke RCU core. */
@@ -2827,11 +2992,22 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
                if (cpu != -1)
                        rdp = per_cpu_ptr(rsp->rda, cpu);
-                offline = !__call_rcu_nocb(rdp, head, lazy, flags);
+                if (likely(rdp->mynode)) {
-                WARN_ON_ONCE(offline);
+                        /* Post-boot, so this should be for a no-CBs CPU. */
-                /* _call_rcu() is illegal on offline CPU; leak the callback. */
+                        offline = !__call_rcu_nocb(rdp, head, lazy, flags);
-                local_irq_restore(flags);
+                        WARN_ON_ONCE(offline);
-                return;
+                        /* Offline CPU, _call_rcu() illegal, leak callback.  */
+                        local_irq_restore(flags);
+                        return;
+                }
+                /*
+                 * Very early boot, before rcu_init().  Initialize if needed
+                 * and then drop through to queue the callback.
+                 */
+                BUG_ON(cpu != -1);
+                WARN_ON_ONCE(!rcu_is_watching());
+                if (!likely(rdp->nxtlist))
+                        init_default_callback_list(rdp);
        }
        ACCESS_ONCE(rdp->qlen) = rdp->qlen + 1;
        if (lazy)
@@ -2954,7 +3130,7 @@ void synchronize_sched(void)
                           "Illegal synchronize_sched() in RCU-sched read-side critical section");
        if (rcu_blocking_is_gp())
                return;
-        if (rcu_expedited)
+        if (rcu_gp_is_expedited())
                synchronize_sched_expedited();
        else
                wait_rcu_gp(call_rcu_sched);
@@ -2981,7 +3157,7 @@ void synchronize_rcu_bh(void)
                           "Illegal synchronize_rcu_bh() in RCU-bh read-side critical section");
        if (rcu_blocking_is_gp())
                return;
-        if (rcu_expedited)
+        if (rcu_gp_is_expedited())
                synchronize_rcu_bh_expedited();
        else
                wait_rcu_gp(call_rcu_bh);
@@ -3518,6 +3694,28 @@ void rcu_barrier_sched(void)
 EXPORT_SYMBOL_GPL(rcu_barrier_sched);
 /*
+ * Propagate ->qsinitmask bits up the rcu_node tree to account for the
+ * first CPU in a given leaf rcu_node structure coming online.  The caller
+ * must hold the corresponding leaf rcu_node ->lock with interrrupts
+ * disabled.
+ */
+static void rcu_init_new_rnp(struct rcu_node *rnp_leaf)
+{
+        long mask;
+        struct rcu_node *rnp = rnp_leaf;
+        for (;;) {
+                mask = rnp->grpmask;
+                rnp = rnp->parent;
+                if (rnp == NULL)
+                        return;
+                raw_spin_lock(&rnp->lock); /* Interrupts already disabled. */
+                rnp->qsmaskinit |= mask;
+                raw_spin_unlock(&rnp->lock); /* Interrupts remain disabled. */
+        }
+}
+/*
 * Do boot-time initialization of a CPU's per-CPU RCU data.
 */
 static void __init
@@ -3553,49 +3751,37 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
        struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
        struct rcu_node *rnp = rcu_get_root(rsp);
-        /* Exclude new grace periods. */
-        mutex_lock(&rsp->onoff_mutex);
        /* Set up local state, ensuring consistent view of global state. */
        raw_spin_lock_irqsave(&rnp->lock, flags);
        rdp->beenonline = 1;     /* We have now been online. */
        rdp->qlen_last_fqs_check = 0;
        rdp->n_force_qs_snap = rsp->n_force_qs;
        rdp->blimit = blimit;
-        init_callback_list(rdp);  /* Re-enable callbacks on this CPU. */
+        if (!rdp->nxtlist)
+                init_callback_list(rdp);  /* Re-enable callbacks on this CPU. */
        rdp->dynticks->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
        rcu_sysidle_init_percpu_data(rdp->dynticks);
        atomic_set(&rdp->dynticks->dynticks,
                   (atomic_read(&rdp->dynticks->dynticks) & ~0x1) + 1);
        raw_spin_unlock(&rnp->lock);            /* irqs remain disabled. */
-        /* Add CPU to rcu_node bitmasks. */
+        /*
+         * Add CPU to leaf rcu_node pending-online bitmask.  Any needed
+         * propagation up the rcu_node tree will happen at the beginning
+         * of the next grace period.
+         */
        rnp = rdp->mynode;
        mask = rdp->grpmask;
-        do {
+        raw_spin_lock(&rnp->lock);              /* irqs already disabled. */
-                /* Exclude any attempts to start a new GP on small systems. */
+        smp_mb__after_unlock_lock();
-                raw_spin_lock(&rnp->lock);      /* irqs already disabled. */
+        rnp->qsmaskinitnext |= mask;
-                rnp->qsmaskinit |= mask;
+        rdp->gpnum = rnp->completed; /* Make CPU later note any new GP. */
-                mask = rnp->grpmask;
+        rdp->completed = rnp->completed;
-                if (rnp == rdp->mynode) {
+        rdp->passed_quiesce = false;
-                        /*
+        rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr);
-                         * If there is a grace period in progress, we will
+        rdp->qs_pending = false;
-                         * set up to wait for it next time we run the
+        trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuonl"));
-                         * RCU core code.
+        raw_spin_unlock_irqrestore(&rnp->lock, flags);
-                         */
-                        rdp->gpnum = rnp->completed;
-                        rdp->completed = rnp->completed;
-                        rdp->passed_quiesce = 0;
-                        rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr);
-                        rdp->qs_pending = 0;
-                        trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuonl"));
-                }
-                raw_spin_unlock(&rnp->lock); /* irqs already disabled. */
-                rnp = rnp->parent;
-        } while (rnp != NULL && !(rnp->qsmaskinit & mask));
-        local_irq_restore(flags);
-        mutex_unlock(&rsp->onoff_mutex);
 }
 static void rcu_prepare_cpu(int cpu)
@@ -3609,15 +3795,14 @@ static void rcu_prepare_cpu(int cpu)
 /*
 * Handle CPU online/offline notification events.
 */
-static int rcu_cpu_notify(struct notifier_block *self,
+int rcu_cpu_notify(struct notifier_block *self,
-                                    unsigned long action, void *hcpu)
+                   unsigned long action, void *hcpu)
 {
        long cpu = (long)hcpu;
        struct rcu_data *rdp = per_cpu_ptr(rcu_state_p->rda, cpu);
        struct rcu_node *rnp = rdp->mynode;
        struct rcu_state *rsp;
-        trace_rcu_utilization(TPS("Start CPU hotplug"));
        switch (action) {
        case CPU_UP_PREPARE:
        case CPU_UP_PREPARE_FROZEN:
@@ -3637,6 +3822,11 @@ static int rcu_cpu_notify(struct notifier_block *self,
                for_each_rcu_flavor(rsp)
                        rcu_cleanup_dying_cpu(rsp);
                break;
+        case CPU_DYING_IDLE:
+                for_each_rcu_flavor(rsp) {
+                        rcu_cleanup_dying_idle_cpu(cpu, rsp);
+                }
+                break;
        case CPU_DEAD:
        case CPU_DEAD_FROZEN:
        case CPU_UP_CANCELED:
@@ -3649,7 +3839,6 @@ static int rcu_cpu_notify(struct notifier_block *self,
        default:
                break;
        }
-        trace_rcu_utilization(TPS("End CPU hotplug"));
        return NOTIFY_OK;
 }
@@ -3660,11 +3849,12 @@ static int rcu_pm_notify(struct notifier_block *self,
        case PM_HIBERNATION_PREPARE:
        case PM_SUSPEND_PREPARE:
                if (nr_cpu_ids <= 256) /* Expediting bad for large systems. */
-                        rcu_expedited = 1;
+                        rcu_expedite_gp();
                break;
        case PM_POST_HIBERNATION:
        case PM_POST_SUSPEND:
-                rcu_expedited = 0;
+                if (nr_cpu_ids <= 256) /* Expediting bad for large systems. */
+                        rcu_unexpedite_gp();
                break;
        default:
                break;
@@ -3734,30 +3924,26 @@ void rcu_scheduler_starting(void)
 * Compute the per-level fanout, either using the exact fanout specified
 * or balancing the tree, depending on CONFIG_RCU_FANOUT_EXACT.
 */
-#ifdef CONFIG_RCU_FANOUT_EXACT
-static void __init rcu_init_levelspread(struct rcu_state *rsp)
-{
-        int i;
-        rsp->levelspread[rcu_num_lvls - 1] = rcu_fanout_leaf;
-        for (i = rcu_num_lvls - 2; i >= 0; i--)
-                rsp->levelspread[i] = CONFIG_RCU_FANOUT;
-}
-#else /* #ifdef CONFIG_RCU_FANOUT_EXACT */
 static void __init rcu_init_levelspread(struct rcu_state *rsp)
 {
-        int ccur;
-        int cprv;
        int i;
-        cprv = nr_cpu_ids;
+        if (IS_ENABLED(CONFIG_RCU_FANOUT_EXACT)) {
-        for (i = rcu_num_lvls - 1; i >= 0; i--) {
+                rsp->levelspread[rcu_num_lvls - 1] = rcu_fanout_leaf;
-                ccur = rsp->levelcnt[i];
+                for (i = rcu_num_lvls - 2; i >= 0; i--)
-                rsp->levelspread[i] = (cprv + ccur - 1) / ccur;
+                        rsp->levelspread[i] = CONFIG_RCU_FANOUT;
-                cprv = ccur;
+        } else {
+                int ccur;
+                int cprv;
+                cprv = nr_cpu_ids;
+                for (i = rcu_num_lvls - 1; i >= 0; i--) {
+                        ccur = rsp->levelcnt[i];
+                        rsp->levelspread[i] = (cprv + ccur - 1) / ccur;
+                        cprv = ccur;
+                }
        }
 }
-#endif /* #else #ifdef CONFIG_RCU_FANOUT_EXACT */
 /*
 * Helper function for rcu_init() that initializes one rcu_state structure.
@@ -3833,7 +4019,6 @@ static void __init rcu_init_one(struct rcu_state *rsp,
                }
        }
-        rsp->rda = rda;
        init_waitqueue_head(&rsp->gp_wq);
        rnp = rsp->level[rcu_num_lvls - 1];
        for_each_possible_cpu(i) {
@@ -3926,6 +4111,8 @@ void __init rcu_init(void)
 {
        int cpu;
+        rcu_early_boot_tests();
        rcu_bootup_announce();
        rcu_init_geometry();
        rcu_init_one(&rcu_bh_state, &rcu_bh_data);
@@ -3942,8 +4129,6 @@ void __init rcu_init(void)
        pm_notifier(rcu_pm_notify, 0);
        for_each_online_cpu(cpu)
                rcu_cpu_notify(NULL, CPU_UP_PREPARE, (void *)(long)cpu);
-        rcu_early_boot_tests();
 }
 #include "tree_plugin.h"
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index 119de399eb2f..a69d3dab2ec4 100644
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -141,12 +141,20 @@ struct rcu_node {
                                /*  complete (only for PREEMPT_RCU). */
        unsigned long qsmaskinit;
                                /* Per-GP initial value for qsmask & expmask. */
+                                /*  Initialized from ->qsmaskinitnext at the */
+                                /*  beginning of each grace period. */
+        unsigned long qsmaskinitnext;
+                                /* Online CPUs for next grace period. */
        unsigned long grpmask;  /* Mask to apply to parent qsmask. */
                                /*  Only one bit will be set in this mask. */
        int     grplo;          /* lowest-numbered CPU or group here. */
        int     grphi;          /* highest-numbered CPU or group here. */
        u8      grpnum;         /* CPU/group number for next level up. */
        u8      level;          /* root is at level 0. */
+        bool    wait_blkd_tasks;/* Necessary to wait for blocked tasks to */
+                                /*  exit RCU read-side critical sections */
+                                /*  before propagating offline up the */
+                                /*  rcu_node tree? */
        struct rcu_node *parent;
        struct list_head blkd_tasks;
                                /* Tasks blocked in RCU read-side critical */
@@ -448,8 +456,6 @@ struct rcu_state {
        long qlen;                              /* Total number of callbacks. */
        /* End of fields guarded by orphan_lock. */
-        struct mutex onoff_mutex;               /* Coordinate hotplug & GPs. */
        struct mutex barrier_mutex;             /* Guards barrier fields. */
        atomic_t barrier_cpu_count;             /* # CPUs waiting on. */
        struct completion barrier_completion;   /* Wake at barrier end. */
@@ -559,6 +565,7 @@ static void rcu_prepare_kthreads(int cpu);
 static void rcu_cleanup_after_idle(void);
 static void rcu_prepare_for_idle(void);
 static void rcu_idle_count_callbacks_posted(void);
+static bool rcu_preempt_has_tasks(struct rcu_node *rnp);
 static void print_cpu_stall_info_begin(void);
 static void print_cpu_stall_info(struct rcu_state *rsp, int cpu);
 static void print_cpu_stall_info_end(void);
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 0a571e9a0f1d..8c0ec0f5a027 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -58,38 +58,33 @@ static bool __read_mostly rcu_nocb_poll;    /* Offload kthread are to poll. */
 */
 static void __init rcu_bootup_announce_oddness(void)
 {
-#ifdef CONFIG_RCU_TRACE
+        if (IS_ENABLED(CONFIG_RCU_TRACE))
-        pr_info("\tRCU debugfs-based tracing is enabled.\n");
+                pr_info("\tRCU debugfs-based tracing is enabled.\n");
-#endif
+        if ((IS_ENABLED(CONFIG_64BIT) && CONFIG_RCU_FANOUT != 64) ||
-#if (defined(CONFIG_64BIT) && CONFIG_RCU_FANOUT != 64) || (!defined(CONFIG_64BIT) && CONFIG_RCU_FANOUT != 32)
+            (!IS_ENABLED(CONFIG_64BIT) && CONFIG_RCU_FANOUT != 32))
-        pr_info("\tCONFIG_RCU_FANOUT set to non-default value of %d\n",
+                pr_info("\tCONFIG_RCU_FANOUT set to non-default value of %d\n",
-               CONFIG_RCU_FANOUT);
+                       CONFIG_RCU_FANOUT);
-#endif
+        if (IS_ENABLED(CONFIG_RCU_FANOUT_EXACT))
-#ifdef CONFIG_RCU_FANOUT_EXACT
+                pr_info("\tHierarchical RCU autobalancing is disabled.\n");
-        pr_info("\tHierarchical RCU autobalancing is disabled.\n");
+        if (IS_ENABLED(CONFIG_RCU_FAST_NO_HZ))
-#endif
+                pr_info("\tRCU dyntick-idle grace-period acceleration is enabled.\n");
-#ifdef CONFIG_RCU_FAST_NO_HZ
+        if (IS_ENABLED(CONFIG_PROVE_RCU))
-        pr_info("\tRCU dyntick-idle grace-period acceleration is enabled.\n");
+                pr_info("\tRCU lockdep checking is enabled.\n");
-#endif
+        if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_RUNNABLE))
-#ifdef CONFIG_PROVE_RCU
+                pr_info("\tRCU torture testing starts during boot.\n");
-        pr_info("\tRCU lockdep checking is enabled.\n");
+        if (IS_ENABLED(CONFIG_RCU_CPU_STALL_INFO))
-#endif
+                pr_info("\tAdditional per-CPU info printed with stalls.\n");
-#ifdef CONFIG_RCU_TORTURE_TEST_RUNNABLE
+        if (NUM_RCU_LVL_4 != 0)
-        pr_info("\tRCU torture testing starts during boot.\n");
+                pr_info("\tFour-level hierarchy is enabled.\n");
-#endif
+        if (CONFIG_RCU_FANOUT_LEAF != 16)
-#if defined(CONFIG_RCU_CPU_STALL_INFO)
+                pr_info("\tBuild-time adjustment of leaf fanout to %d.\n",
-        pr_info("\tAdditional per-CPU info printed with stalls.\n");
+                        CONFIG_RCU_FANOUT_LEAF);
-#endif
-#if NUM_RCU_LVL_4 != 0
-        pr_info("\tFour-level hierarchy is enabled.\n");
-#endif
        if (rcu_fanout_leaf != CONFIG_RCU_FANOUT_LEAF)
                pr_info("\tBoot-time adjustment of leaf fanout to %d.\n", rcu_fanout_leaf);
        if (nr_cpu_ids != NR_CPUS)
                pr_info("\tRCU restricting CPUs from NR_CPUS=%d to nr_cpu_ids=%d.\n", NR_CPUS, nr_cpu_ids);
-#ifdef CONFIG_RCU_BOOST
+        if (IS_ENABLED(CONFIG_RCU_BOOST))
-        pr_info("\tRCU kthread priority: %d.\n", kthread_prio);
+                pr_info("\tRCU kthread priority: %d.\n", kthread_prio);
-#endif
 }
 #ifdef CONFIG_PREEMPT_RCU
@@ -180,7 +175,7 @@ static void rcu_preempt_note_context_switch(void)
                 * But first, note that the current CPU must still be
                 * on line!
                 */
-                WARN_ON_ONCE((rdp->grpmask & rnp->qsmaskinit) == 0);
+                WARN_ON_ONCE((rdp->grpmask & rcu_rnp_online_cpus(rnp)) == 0);
                WARN_ON_ONCE(!list_empty(&t->rcu_node_entry));
                if ((rnp->qsmask & rdp->grpmask) && rnp->gp_tasks != NULL) {
                        list_add(&t->rcu_node_entry, rnp->gp_tasks->prev);
@@ -233,43 +228,6 @@ static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
 }
 /*
- * Record a quiescent state for all tasks that were previously queued
- * on the specified rcu_node structure and that were blocking the current
- * RCU grace period.  The caller must hold the specified rnp->lock with
- * irqs disabled, and this lock is released upon return, but irqs remain
- * disabled.
- */
-static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp, unsigned long flags)
-        __releases(rnp->lock)
-{
-        unsigned long mask;
-        struct rcu_node *rnp_p;
-        if (rnp->qsmask != 0 || rcu_preempt_blocked_readers_cgp(rnp)) {
-                raw_spin_unlock_irqrestore(&rnp->lock, flags);
-                return;  /* Still need more quiescent states! */
-        }
-        rnp_p = rnp->parent;
-        if (rnp_p == NULL) {
-                /*
-                 * Either there is only one rcu_node in the tree,
-                 * or tasks were kicked up to root rcu_node due to
-                 * CPUs going offline.
-                 */
-                rcu_report_qs_rsp(&rcu_preempt_state, flags);
-                return;
-        }
-        /* Report up the rest of the hierarchy. */
-        mask = rnp->grpmask;
-        raw_spin_unlock(&rnp->lock);    /* irqs remain disabled. */
-        raw_spin_lock(&rnp_p->lock);    /* irqs already disabled. */
-        smp_mb__after_unlock_lock();
-        rcu_report_qs_rnp(mask, &rcu_preempt_state, rnp_p, flags);
-}
-/*
 * Advance a ->blkd_tasks-list pointer to the next entry, instead
 * returning NULL if at the end of the list.
 */
@@ -300,7 +258,6 @@ static bool rcu_preempt_has_tasks(struct rcu_node *rnp)
 */
 void rcu_read_unlock_special(struct task_struct *t)
 {
-        bool empty;
        bool empty_exp;
        bool empty_norm;
        bool empty_exp_now;
@@ -334,7 +291,13 @@ void rcu_read_unlock_special(struct task_struct *t)
        }
        /* Hardware IRQ handlers cannot block, complain if they get here. */
-        if (WARN_ON_ONCE(in_irq() || in_serving_softirq())) {
+        if (in_irq() || in_serving_softirq()) {
+                lockdep_rcu_suspicious(__FILE__, __LINE__,
+                                       "rcu_read_unlock() from irq or softirq with blocking in critical section!!!\n");
+                pr_alert("->rcu_read_unlock_special: %#x (b: %d, nq: %d)\n",
+                         t->rcu_read_unlock_special.s,
+                         t->rcu_read_unlock_special.b.blocked,
+                         t->rcu_read_unlock_special.b.need_qs);
                local_irq_restore(flags);
                return;
        }
@@ -356,7 +319,6 @@ void rcu_read_unlock_special(struct task_struct *t)
                                break;
                        raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
                }
-                empty = !rcu_preempt_has_tasks(rnp);
                empty_norm = !rcu_preempt_blocked_readers_cgp(rnp);
                empty_exp = !rcu_preempted_readers_exp(rnp);
                smp_mb(); /* ensure expedited fastpath sees end of RCU c-s. */
@@ -377,14 +339,6 @@ void rcu_read_unlock_special(struct task_struct *t)
 #endif /* #ifdef CONFIG_RCU_BOOST */
                /*
-                 * If this was the last task on the list, go see if we
-                 * need to propagate ->qsmaskinit bit clearing up the
-                 * rcu_node tree.
-                 */
-                if (!empty && !rcu_preempt_has_tasks(rnp))
-                        rcu_cleanup_dead_rnp(rnp);
-                /*
                 * If this was the last task on the current list, and if
                 * we aren't waiting on any CPUs, report the quiescent state.
                 * Note that rcu_report_unblock_qs_rnp() releases rnp->lock,
@@ -399,7 +353,8 @@ void rcu_read_unlock_special(struct task_struct *t)
                                                         rnp->grplo,
                                                         rnp->grphi,
                                                         !!rnp->gp_tasks);
-                        rcu_report_unblock_qs_rnp(rnp, flags);
+                        rcu_report_unblock_qs_rnp(&rcu_preempt_state,
+                                                  rnp, flags);
                } else {
                        raw_spin_unlock_irqrestore(&rnp->lock, flags);
                }
@@ -520,10 +475,6 @@ static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
        WARN_ON_ONCE(rnp->qsmask);
 }
-#ifdef CONFIG_HOTPLUG_CPU
-#endif /* #ifdef CONFIG_HOTPLUG_CPU */
 /*
 * Check for a quiescent state from the current CPU.  When a task blocks,
 * the task is recorded in the corresponding CPU's rcu_node structure,
@@ -585,7 +536,7 @@ void synchronize_rcu(void)
                           "Illegal synchronize_rcu() in RCU read-side critical section");
        if (!rcu_scheduler_active)
                return;
-        if (rcu_expedited)
+        if (rcu_gp_is_expedited())
                synchronize_rcu_expedited();
        else
                wait_rcu_gp(call_rcu);
@@ -630,9 +581,6 @@ static int sync_rcu_preempt_exp_done(struct rcu_node *rnp)
 * recursively up the tree.  (Calm down, calm down, we do the recursion
 * iteratively!)
 *
- * Most callers will set the "wake" flag, but the task initiating the
- * expedited grace period need not wake itself.
- *
 * Caller must hold sync_rcu_preempt_exp_mutex.
 */
 static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
@@ -667,29 +615,85 @@ static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
 /*
 * Snapshot the tasks blocking the newly started preemptible-RCU expedited
- * grace period for the specified rcu_node structure.  If there are no such
+ * grace period for the specified rcu_node structure, phase 1.  If there
- * tasks, report it up the rcu_node hierarchy.
+ * are such tasks, set the ->expmask bits up the rcu_node tree and also
+ * set the ->expmask bits on the leaf rcu_node structures to tell phase 2
+ * that work is needed here.
 *
- * Caller must hold sync_rcu_preempt_exp_mutex and must exclude
+ * Caller must hold sync_rcu_preempt_exp_mutex.
- * CPU hotplug operations.
 */
 static void
-sync_rcu_preempt_exp_init(struct rcu_state *rsp, struct rcu_node *rnp)
+sync_rcu_preempt_exp_init1(struct rcu_state *rsp, struct rcu_node *rnp)
 {
        unsigned long flags;
-        int must_wait = 0;
+        unsigned long mask;
+        struct rcu_node *rnp_up;
        raw_spin_lock_irqsave(&rnp->lock, flags);
        smp_mb__after_unlock_lock();
+        WARN_ON_ONCE(rnp->expmask);
+        WARN_ON_ONCE(rnp->exp_tasks);
        if (!rcu_preempt_has_tasks(rnp)) {
+                /* No blocked tasks, nothing to do. */
                raw_spin_unlock_irqrestore(&rnp->lock, flags);
-        } else {
+                return;
+        }
+        /* Call for Phase 2 and propagate ->expmask bits up the tree. */
+        rnp->expmask = 1;
+        rnp_up = rnp;
+        while (rnp_up->parent) {
+                mask = rnp_up->grpmask;
+                rnp_up = rnp_up->parent;
+                if (rnp_up->expmask & mask)
+                        break;
+                raw_spin_lock(&rnp_up->lock); /* irqs already off */
+                smp_mb__after_unlock_lock();
+                rnp_up->expmask |= mask;
+                raw_spin_unlock(&rnp_up->lock); /* irqs still off */
+        }
+        raw_spin_unlock_irqrestore(&rnp->lock, flags);
+}
+/*
+ * Snapshot the tasks blocking the newly started preemptible-RCU expedited
+ * grace period for the specified rcu_node structure, phase 2.  If the
+ * leaf rcu_node structure has its ->expmask field set, check for tasks.
+ * If there are some, clear ->expmask and set ->exp_tasks accordingly,
+ * then initiate RCU priority boosting.  Otherwise, clear ->expmask and
+ * invoke rcu_report_exp_rnp() to clear out the upper-level ->expmask bits,
+ * enabling rcu_read_unlock_special() to do the bit-clearing.
+ *
+ * Caller must hold sync_rcu_preempt_exp_mutex.
+ */
+static void
+sync_rcu_preempt_exp_init2(struct rcu_state *rsp, struct rcu_node *rnp)
+{
+        unsigned long flags;
+        raw_spin_lock_irqsave(&rnp->lock, flags);
+        smp_mb__after_unlock_lock();
+        if (!rnp->expmask) {
+                /* Phase 1 didn't do anything, so Phase 2 doesn't either. */
+                raw_spin_unlock_irqrestore(&rnp->lock, flags);
+                return;
+        }
+        /* Phase 1 is over. */
+        rnp->expmask = 0;
+        /*
+         * If there are still blocked tasks, set up ->exp_tasks so that
+         * rcu_read_unlock_special() will wake us and then boost them.
+         */
+        if (rcu_preempt_has_tasks(rnp)) {
                rnp->exp_tasks = rnp->blkd_tasks.next;
                rcu_initiate_boost(rnp, flags);  /* releases rnp->lock */
-                must_wait = 1;
+                return;
        }
-        if (!must_wait)
-                rcu_report_exp_rnp(rsp, rnp, false); /* Don't wake self. */
+        /* No longer any blocked tasks, so undo bit setting. */
+        raw_spin_unlock_irqrestore(&rnp->lock, flags);
+        rcu_report_exp_rnp(rsp, rnp, false);
 }
 /**
@@ -706,7 +710,6 @@ sync_rcu_preempt_exp_init(struct rcu_state *rsp, struct rcu_node *rnp)
 */
 void synchronize_rcu_expedited(void)
 {
-        unsigned long flags;
        struct rcu_node *rnp;
        struct rcu_state *rsp = &rcu_preempt_state;
        unsigned long snap;
@@ -757,19 +760,16 @@ void synchronize_rcu_expedited(void)
        /* force all RCU readers onto ->blkd_tasks lists. */
        synchronize_sched_expedited();
-        /* Initialize ->expmask for all non-leaf rcu_node structures. */
+        /*
-        rcu_for_each_nonleaf_node_breadth_first(rsp, rnp) {
+         * Snapshot current state of ->blkd_tasks lists into ->expmask.
-                raw_spin_lock_irqsave(&rnp->lock, flags);
+         * Phase 1 sets bits and phase 2 permits rcu_read_unlock_special()
-                smp_mb__after_unlock_lock();
+         * to start clearing them.  Doing this in one phase leads to
-                rnp->expmask = rnp->qsmaskinit;
+         * strange races between setting and clearing bits, so just say "no"!
-                raw_spin_unlock_irqrestore(&rnp->lock, flags);
+         */
-        }
+        rcu_for_each_leaf_node(rsp, rnp)
+                sync_rcu_preempt_exp_init1(rsp, rnp);
-        /* Snapshot current state of ->blkd_tasks lists. */
        rcu_for_each_leaf_node(rsp, rnp)
-                sync_rcu_preempt_exp_init(rsp, rnp);
+                sync_rcu_preempt_exp_init2(rsp, rnp);
-        if (NUM_RCU_NODES > 1)
-                sync_rcu_preempt_exp_init(rsp, rcu_get_root(rsp));
        put_online_cpus();
@@ -859,8 +859,6 @@ static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
        return 0;
 }
-#ifdef CONFIG_HOTPLUG_CPU
 /*
 * Because there is no preemptible RCU, there can be no readers blocked.
 */
@@ -869,8 +867,6 @@ static bool rcu_preempt_has_tasks(struct rcu_node *rnp)
        return false;
 }
-#endif /* #ifdef CONFIG_HOTPLUG_CPU */
 /*
 * Because preemptible RCU does not exist, we never have to check for
 * tasks blocked within RCU read-side critical sections.
@@ -1170,7 +1166,7 @@ static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
 * Returns zero if all is well, a negated errno otherwise.
 */
 static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
-                                                 struct rcu_node *rnp)
+                                       struct rcu_node *rnp)
 {
        int rnp_index = rnp - &rsp->node[0];
        unsigned long flags;
@@ -1180,7 +1176,7 @@ static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
        if (&rcu_preempt_state != rsp)
                return 0;
-        if (!rcu_scheduler_fully_active || rnp->qsmaskinit == 0)
+        if (!rcu_scheduler_fully_active || rcu_rnp_online_cpus(rnp) == 0)
                return 0;
        rsp->boost = 1;
@@ -1273,7 +1269,7 @@ static void rcu_cpu_kthread(unsigned int cpu)
 static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
 {
        struct task_struct *t = rnp->boost_kthread_task;
-        unsigned long mask = rnp->qsmaskinit;
+        unsigned long mask = rcu_rnp_online_cpus(rnp);
        cpumask_var_t cm;
        int cpu;
@@ -1945,7 +1941,8 @@ static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu)
                rhp = ACCESS_ONCE(rdp->nocb_follower_head);
        /* Having no rcuo kthread but CBs after scheduler starts is bad! */
-        if (!ACCESS_ONCE(rdp->nocb_kthread) && rhp) {
+        if (!ACCESS_ONCE(rdp->nocb_kthread) && rhp &&
+            rcu_scheduler_fully_active) {
                /* RCU callback enqueued before CPU first came online??? */
                pr_err("RCU: Never-onlined no-CBs CPU %d has CB %p\n",
                       cpu, rhp->func);
@@ -2392,18 +2389,8 @@ void __init rcu_init_nohz(void)
                pr_info("\tPoll for callbacks from no-CBs CPUs.\n");
        for_each_rcu_flavor(rsp) {
-                for_each_cpu(cpu, rcu_nocb_mask) {
+                for_each_cpu(cpu, rcu_nocb_mask)
-                        struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
+                        init_nocb_callback_list(per_cpu_ptr(rsp->rda, cpu));
-                        /*
-                         * If there are early callbacks, they will need
-                         * to be moved to the nocb lists.
-                         */
-                        WARN_ON_ONCE(rdp->nxttail[RCU_NEXT_TAIL] !=
-                                     &rdp->nxtlist &&
-                                     rdp->nxttail[RCU_NEXT_TAIL] != NULL);
-                        init_nocb_callback_list(rdp);
-                }
                rcu_organize_nocb_kthreads(rsp);
        }
 }
@@ -2540,6 +2527,16 @@ static bool init_nocb_callback_list(struct rcu_data *rdp)
        if (!rcu_is_nocb_cpu(rdp->cpu))
                return false;
+        /* If there are early-boot callbacks, move them to nocb lists. */
+        if (rdp->nxtlist) {
+                rdp->nocb_head = rdp->nxtlist;
+                rdp->nocb_tail = rdp->nxttail[RCU_NEXT_TAIL];
+                atomic_long_set(&rdp->nocb_q_count, rdp->qlen);
+                atomic_long_set(&rdp->nocb_q_count_lazy, rdp->qlen_lazy);
+                rdp->nxtlist = NULL;
+                rdp->qlen = 0;
+                rdp->qlen_lazy = 0;
+        }
        rdp->nxttail[RCU_NEXT_TAIL] = NULL;
        return true;
 }
@@ -2763,7 +2760,8 @@ static void rcu_sysidle_exit(int irq)
 /*
 * Check to see if the current CPU is idle.  Note that usermode execution
- * does not count as idle.  The caller must have disabled interrupts.
+ * does not count as idle.  The caller must have disabled interrupts,
+ * and must be running on tick_do_timer_cpu.
 */
 static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
                                  unsigned long *maxj)
@@ -2784,8 +2782,8 @@ static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
        if (!*isidle || rdp->rsp != rcu_state_p ||
            cpu_is_offline(rdp->cpu) || rdp->cpu == tick_do_timer_cpu)
                return;
-        if (rcu_gp_in_progress(rdp->rsp))
+        /* Verify affinity of current kthread. */
-                WARN_ON_ONCE(smp_processor_id() != tick_do_timer_cpu);
+        WARN_ON_ONCE(smp_processor_id() != tick_do_timer_cpu);
        /* Pick up current idle and NMI-nesting counter and check. */
        cur = atomic_read(&rdtp->dynticks_idle);
@@ -3068,11 +3066,10 @@ static void rcu_bind_gp_kthread(void)
                return;
 #ifdef CONFIG_NO_HZ_FULL_SYSIDLE
        cpu = tick_do_timer_cpu;
-        if (cpu >= 0 && cpu < nr_cpu_ids && raw_smp_processor_id() != cpu)
+        if (cpu >= 0 && cpu < nr_cpu_ids)
                set_cpus_allowed_ptr(current, cpumask_of(cpu));
 #else /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
-        if (!is_housekeeping_cpu(raw_smp_processor_id()))
+        housekeeping_affine(current);
-                housekeeping_affine(current);
 #endif /* #else #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
 }
diff --git a/kernel/rcu/tree_trace.c b/kernel/rcu/tree_trace.c
index fbb6240509ea..f92361efd0f5 100644
--- a/kernel/rcu/tree_trace.c
+++ b/kernel/rcu/tree_trace.c
@@ -283,8 +283,8 @@ static void print_one_rcu_state(struct seq_file *m, struct rcu_state *rsp)
                        seq_puts(m, "\n");
                        level = rnp->level;
                }
-                seq_printf(m, "%lx/%lx %c%c>%c %d:%d ^%d    ",
+                seq_printf(m, "%lx/%lx->%lx %c%c>%c %d:%d ^%d    ",
-                           rnp->qsmask, rnp->qsmaskinit,
+                           rnp->qsmask, rnp->qsmaskinit, rnp->qsmaskinitnext,
                           ".G"[rnp->gp_tasks != NULL],
                           ".E"[rnp->exp_tasks != NULL],
                           ".T"[!list_empty(&rnp->blkd_tasks)],
diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c
index e0d31a345ee6..1f133350da01 100644
--- a/kernel/rcu/update.c
+++ b/kernel/rcu/update.c
@@ -62,6 +62,63 @@ MODULE_ALIAS("rcupdate");
 module_param(rcu_expedited, int, 0);
+#ifndef CONFIG_TINY_RCU
+static atomic_t rcu_expedited_nesting =
+        ATOMIC_INIT(IS_ENABLED(CONFIG_RCU_EXPEDITE_BOOT) ? 1 : 0);
+/*
+ * Should normal grace-period primitives be expedited?  Intended for
+ * use within RCU.  Note that this function takes the rcu_expedited
+ * sysfs/boot variable into account as well as the rcu_expedite_gp()
+ * nesting.  So looping on rcu_unexpedite_gp() until rcu_gp_is_expedited()
+ * returns false is a -really- bad idea.
+ */
+bool rcu_gp_is_expedited(void)
+{
+        return rcu_expedited || atomic_read(&rcu_expedited_nesting);
+}
+EXPORT_SYMBOL_GPL(rcu_gp_is_expedited);
+/**
+ * rcu_expedite_gp - Expedite future RCU grace periods
+ *
+ * After a call to this function, future calls to synchronize_rcu() and
+ * friends act as the corresponding synchronize_rcu_expedited() function
+ * had instead been called.
+ */
+void rcu_expedite_gp(void)
+{
+        atomic_inc(&rcu_expedited_nesting);
+}
+EXPORT_SYMBOL_GPL(rcu_expedite_gp);
+/**
+ * rcu_unexpedite_gp - Cancel prior rcu_expedite_gp() invocation
+ *
+ * Undo a prior call to rcu_expedite_gp().  If all prior calls to
+ * rcu_expedite_gp() are undone by a subsequent call to rcu_unexpedite_gp(),
+ * and if the rcu_expedited sysfs/boot parameter is not set, then all
+ * subsequent calls to synchronize_rcu() and friends will return to
+ * their normal non-expedited behavior.
+ */
+void rcu_unexpedite_gp(void)
+{
+        atomic_dec(&rcu_expedited_nesting);
+}
+EXPORT_SYMBOL_GPL(rcu_unexpedite_gp);
+#endif /* #ifndef CONFIG_TINY_RCU */
+/*
+ * Inform RCU of the end of the in-kernel boot sequence.
+ */
+void rcu_end_inkernel_boot(void)
+{
+        if (IS_ENABLED(CONFIG_RCU_EXPEDITE_BOOT))
+                rcu_unexpedite_gp();
+}
 #ifdef CONFIG_PREEMPT_RCU
 /*
@@ -199,16 +256,13 @@ EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held);
 #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
-struct rcu_synchronize {
+/**
-        struct rcu_head head;
+ * wakeme_after_rcu() - Callback function to awaken a task after grace period
-        struct completion completion;
+ * @head: Pointer to rcu_head member within rcu_synchronize structure
-};
+ *
+ * Awaken the corresponding task now that a grace period has elapsed.
-/*
- * Awaken the corresponding synchronize_rcu() instance now that a
- * grace period has elapsed.
 */
-static void wakeme_after_rcu(struct rcu_head  *head)
+void wakeme_after_rcu(struct rcu_head *head)
 {
        struct rcu_synchronize *rcu;