3 files changed, 19 insertions, 28 deletions

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index aeea697d6f9f..4ea28e820f4a 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -3899,11 +3899,6 @@ static void rcu_adopt_orphan_cbs(struct rcu_state *rsp, unsigned long flags)
 
         lockdep_assert_held(&rsp->orphan_lock);
 
-        /* No-CBs CPUs are handled specially. */
-        if (!IS_ENABLED(CONFIG_HOTPLUG_CPU) ||
-            rcu_nocb_adopt_orphan_cbs(rsp, rdp, flags))
-                return;
-
         /* Do the accounting first. */
         if (rsp->orphan_done.len_lazy != rsp->orphan_done.len)
                 rcu_idle_count_callbacks_posted();
@@ -3928,13 +3923,20 @@ static void rcu_adopt_orphan_cbs(struct rcu_state *rsp, unsigned long flags)
 static void rcu_migrate_callbacks(int cpu, struct rcu_state *rsp)
 {
         unsigned long flags;
+        struct rcu_data *my_rdp;
         struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
         struct rcu_node *rnp = rdp->mynode;  /* Outgoing CPU's rdp & rnp. */
 
         if (rcu_is_nocb_cpu(cpu) || rcu_segcblist_empty(&rdp->cblist))
                 return;  /* No callbacks to migrate. */
 
-        raw_spin_lock_irqsave(&rsp->orphan_lock, flags);
+        local_irq_save(flags);
+        my_rdp = this_cpu_ptr(rsp->rda);
+        if (rcu_nocb_adopt_orphan_cbs(my_rdp, rdp, flags)) {
+                local_irq_restore(flags);
+                return;
+        }
+        raw_spin_lock(&rsp->orphan_lock); /* irqs already disabled. */
         rcu_send_cbs_to_orphanage(cpu, rsp, rnp, rdp);
         rcu_adopt_orphan_cbs(rsp, flags);
         raw_spin_unlock_irqrestore(&rsp->orphan_lock, flags);
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index aec53c4d4aec..574513cf49b4 100644
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -493,7 +493,7 @@ static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq);
 static void rcu_init_one_nocb(struct rcu_node *rnp);
 static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
                             bool lazy, unsigned long flags);
-static bool rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
+static bool rcu_nocb_adopt_orphan_cbs(struct rcu_data *my_rdp,
                                       struct rcu_data *rdp,
                                       unsigned long flags);
 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp);
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 908b309d60d7..ff7d5ee49816 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -1961,30 +1961,19 @@ static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
  * Adopt orphaned callbacks on a no-CBs CPU, or return 0 if this is
  * not a no-CBs CPU.
  */
-static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
+static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_data *my_rdp,
                                                      struct rcu_data *rdp,
                                                      unsigned long flags)
 {
-        long ql = rsp->orphan_done.len;
-        long qll = rsp->orphan_done.len_lazy;
-
-        /* If this is not a no-CBs CPU, tell the caller to do it the old way. */
+        RCU_LOCKDEP_WARN(!irqs_disabled(), "rcu_nocb_adopt_orphan_cbs() invoked with irqs enabled!!!");
         if (!rcu_is_nocb_cpu(smp_processor_id()))
-                return false;
-
-        /* First, enqueue the donelist, if any.  This preserves CB ordering. */
-        if (rsp->orphan_done.head) {
-                __call_rcu_nocb_enqueue(rdp, rcu_cblist_head(&rsp->orphan_done),
-                                        rcu_cblist_tail(&rsp->orphan_done),
-                                        ql, qll, flags);
-        }
-        if (rsp->orphan_pend.head) {
-                __call_rcu_nocb_enqueue(rdp, rcu_cblist_head(&rsp->orphan_pend),
-                                        rcu_cblist_tail(&rsp->orphan_pend),
-                                        ql, qll, flags);
-        }
-        rcu_cblist_init(&rsp->orphan_done);
-        rcu_cblist_init(&rsp->orphan_pend);
+                return false; /* Not NOCBs CPU, caller must migrate CBs. */
+        __call_rcu_nocb_enqueue(my_rdp, rcu_segcblist_head(&rdp->cblist),
+                                rcu_segcblist_tail(&rdp->cblist),
+                                rcu_segcblist_n_cbs(&rdp->cblist),
+                                rcu_segcblist_n_lazy_cbs(&rdp->cblist), flags);
+        rcu_segcblist_init(&rdp->cblist);
+        rcu_segcblist_disable(&rdp->cblist);
         return true;
 }
 
@@ -2459,7 +2448,7 @@ static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
         return false;
 }
 
-static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
+static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_data *my_rdp,
                                                      struct rcu_data *rdp,
                                                      unsigned long flags)
 {