rcu: Make NOCB CPUs migrate CBs directly from outgoing CPU

RCU's CPU-hotplug callback-migration code first moves the outgoing
CPU's callbacks to ->orphan_done and ->orphan_pend, and only then
moves them to the NOCB callback list.  This commit avoids the
extra step (and simplifies the code) by moving the callbacks directly
from the outgoing CPU's callback list to the NOCB callback list.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

1 files changed, 8 insertions, 6 deletions

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index aeea697d6f9f..4ea28e820f4a 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -3899,11 +3899,6 @@ static void rcu_adopt_orphan_cbs(struct rcu_state *rsp, unsigned long flags)
 
         lockdep_assert_held(&rsp->orphan_lock);
 
-        /* No-CBs CPUs are handled specially. */
-        if (!IS_ENABLED(CONFIG_HOTPLUG_CPU) ||
-            rcu_nocb_adopt_orphan_cbs(rsp, rdp, flags))
-                return;
-
         /* Do the accounting first. */
         if (rsp->orphan_done.len_lazy != rsp->orphan_done.len)
                 rcu_idle_count_callbacks_posted();
@@ -3928,13 +3923,20 @@ static void rcu_adopt_orphan_cbs(struct rcu_state *rsp, unsigned long flags)
 static void rcu_migrate_callbacks(int cpu, struct rcu_state *rsp)
 {
         unsigned long flags;
+        struct rcu_data *my_rdp;
         struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
         struct rcu_node *rnp = rdp->mynode;  /* Outgoing CPU's rdp & rnp. */
 
         if (rcu_is_nocb_cpu(cpu) || rcu_segcblist_empty(&rdp->cblist))
                 return;  /* No callbacks to migrate. */
 
-        raw_spin_lock_irqsave(&rsp->orphan_lock, flags);
+        local_irq_save(flags);
+        my_rdp = this_cpu_ptr(rsp->rda);
+        if (rcu_nocb_adopt_orphan_cbs(my_rdp, rdp, flags)) {
+                local_irq_restore(flags);
+                return;
+        }
+        raw_spin_lock(&rsp->orphan_lock); /* irqs already disabled. */
         rcu_send_cbs_to_orphanage(cpu, rsp, rnp, rdp);
         rcu_adopt_orphan_cbs(rsp, flags);
         raw_spin_unlock_irqrestore(&rsp->orphan_lock, flags);