1 files changed, 77 insertions, 83 deletions

diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index ccdc04c47981..dd4aea806f8e 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -67,9 +67,6 @@ static struct lock_class_key rcu_node_class[NUM_RCU_LVLS];
         .gpnum = -300, \
         .completed = -300, \
         .onofflock = __RAW_SPIN_LOCK_UNLOCKED(&structname.onofflock), \
-        .orphan_cbs_list = NULL, \
-        .orphan_cbs_tail = &structname.orphan_cbs_list, \
-        .orphan_qlen = 0, \
         .fqslock = __RAW_SPIN_LOCK_UNLOCKED(&structname.fqslock), \
         .n_force_qs = 0, \
         .n_force_qs_ngp = 0, \
@@ -367,8 +364,8 @@ void rcu_irq_exit(void)
         WARN_ON_ONCE(rdtp->dynticks & 0x1);
 
         /* If the interrupt queued a callback, get out of dyntick mode. */
-        if (__get_cpu_var(rcu_sched_data).nxtlist ||
-            __get_cpu_var(rcu_bh_data).nxtlist)
+        if (__this_cpu_read(rcu_sched_data.nxtlist) ||
+            __this_cpu_read(rcu_bh_data.nxtlist))
                 set_need_resched();
 }
 
@@ -620,9 +617,17 @@ static void __init check_cpu_stall_init(void)
 static void __note_new_gpnum(struct rcu_state *rsp, struct rcu_node *rnp, struct rcu_data *rdp)
 {
         if (rdp->gpnum != rnp->gpnum) {
-                rdp->qs_pending = 1;
-                rdp->passed_quiesc = 0;
+                /*
+                 * If the current grace period is waiting for this CPU,
+                 * set up to detect a quiescent state, otherwise don't
+                 * go looking for one.
+                 */
                 rdp->gpnum = rnp->gpnum;
+                if (rnp->qsmask & rdp->grpmask) {
+                        rdp->qs_pending = 1;
+                        rdp->passed_quiesc = 0;
+                } else
+                        rdp->qs_pending = 0;
         }
 }
 
@@ -681,6 +686,24 @@ __rcu_process_gp_end(struct rcu_state *rsp, struct rcu_node *rnp, struct rcu_dat
 
                 /* Remember that we saw this grace-period completion. */
                 rdp->completed = rnp->completed;
+
+                /*
+                 * If we were in an extended quiescent state, we may have
+                 * missed some grace periods that others CPUs handled on
+                 * our behalf. Catch up with this state to avoid noting
+                 * spurious new grace periods.  If another grace period
+                 * has started, then rnp->gpnum will have advanced, so
+                 * we will detect this later on.
+                 */
+                if (ULONG_CMP_LT(rdp->gpnum, rdp->completed))
+                        rdp->gpnum = rdp->completed;
+
+                /*
+                 * If RCU does not need a quiescent state from this CPU,
+                 * then make sure that this CPU doesn't go looking for one.
+                 */
+                if ((rnp->qsmask & rdp->grpmask) == 0)
+                        rdp->qs_pending = 0;
         }
 }
 
@@ -984,53 +1007,31 @@ rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp)
 #ifdef CONFIG_HOTPLUG_CPU
 
 /*
- * Move a dying CPU's RCU callbacks to the ->orphan_cbs_list for the
- * specified flavor of RCU.  The callbacks will be adopted by the next
- * _rcu_barrier() invocation or by the CPU_DEAD notifier, whichever
- * comes first.  Because this is invoked from the CPU_DYING notifier,
- * irqs are already disabled.
+ * Move a dying CPU's RCU callbacks to online CPU's callback list.
+ * Synchronization is not required because this function executes
+ * in stop_machine() context.
  */
-static void rcu_send_cbs_to_orphanage(struct rcu_state *rsp)
+static void rcu_send_cbs_to_online(struct rcu_state *rsp)
 {
         int i;
+        /* current DYING CPU is cleared in the cpu_online_mask */
+        int receive_cpu = cpumask_any(cpu_online_mask);
         struct rcu_data *rdp = this_cpu_ptr(rsp->rda);
+        struct rcu_data *receive_rdp = per_cpu_ptr(rsp->rda, receive_cpu);
 
         if (rdp->nxtlist == NULL)
                 return;  /* irqs disabled, so comparison is stable. */
-        raw_spin_lock(&rsp->onofflock);  /* irqs already disabled. */
-        *rsp->orphan_cbs_tail = rdp->nxtlist;
-        rsp->orphan_cbs_tail = rdp->nxttail[RCU_NEXT_TAIL];
+
+        *receive_rdp->nxttail[RCU_NEXT_TAIL] = rdp->nxtlist;
+        receive_rdp->nxttail[RCU_NEXT_TAIL] = rdp->nxttail[RCU_NEXT_TAIL];
+        receive_rdp->qlen += rdp->qlen;
+        receive_rdp->n_cbs_adopted += rdp->qlen;
+        rdp->n_cbs_orphaned += rdp->qlen;
+
         rdp->nxtlist = NULL;
         for (i = 0; i < RCU_NEXT_SIZE; i++)
                 rdp->nxttail[i] = &rdp->nxtlist;
-        rsp->orphan_qlen += rdp->qlen;
-        rdp->n_cbs_orphaned += rdp->qlen;
         rdp->qlen = 0;
-        raw_spin_unlock(&rsp->onofflock);  /* irqs remain disabled. */
-}
-
-/*
- * Adopt previously orphaned RCU callbacks.
- */
-static void rcu_adopt_orphan_cbs(struct rcu_state *rsp)
-{
-        unsigned long flags;
-        struct rcu_data *rdp;
-
-        raw_spin_lock_irqsave(&rsp->onofflock, flags);
-        rdp = this_cpu_ptr(rsp->rda);
-        if (rsp->orphan_cbs_list == NULL) {
-                raw_spin_unlock_irqrestore(&rsp->onofflock, flags);
-                return;
-        }
-        *rdp->nxttail[RCU_NEXT_TAIL] = rsp->orphan_cbs_list;
-        rdp->nxttail[RCU_NEXT_TAIL] = rsp->orphan_cbs_tail;
-        rdp->qlen += rsp->orphan_qlen;
-        rdp->n_cbs_adopted += rsp->orphan_qlen;
-        rsp->orphan_cbs_list = NULL;
-        rsp->orphan_cbs_tail = &rsp->orphan_cbs_list;
-        rsp->orphan_qlen = 0;
-        raw_spin_unlock_irqrestore(&rsp->onofflock, flags);
 }
 
 /*
@@ -1081,8 +1082,6 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)
                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
         if (need_report & RCU_OFL_TASKS_EXP_GP)
                 rcu_report_exp_rnp(rsp, rnp);
-
-        rcu_adopt_orphan_cbs(rsp);
 }
 
 /*
@@ -1100,11 +1099,7 @@ static void rcu_offline_cpu(int cpu)
 
 #else /* #ifdef CONFIG_HOTPLUG_CPU */
 
-static void rcu_send_cbs_to_orphanage(struct rcu_state *rsp)
-{
-}
-
-static void rcu_adopt_orphan_cbs(struct rcu_state *rsp)
+static void rcu_send_cbs_to_online(struct rcu_state *rsp)
 {
 }
 
@@ -1440,22 +1435,11 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
          */
         local_irq_save(flags);
         rdp = this_cpu_ptr(rsp->rda);
-        rcu_process_gp_end(rsp, rdp);
-        check_for_new_grace_period(rsp, rdp);
 
         /* Add the callback to our list. */
         *rdp->nxttail[RCU_NEXT_TAIL] = head;
         rdp->nxttail[RCU_NEXT_TAIL] = &head->next;
 
-        /* Start a new grace period if one not already started. */
-        if (!rcu_gp_in_progress(rsp)) {
-                unsigned long nestflag;
-                struct rcu_node *rnp_root = rcu_get_root(rsp);
-
-                raw_spin_lock_irqsave(&rnp_root->lock, nestflag);
-                rcu_start_gp(rsp, nestflag);  /* releases rnp_root->lock. */
-        }
-
         /*
          * Force the grace period if too many callbacks or too long waiting.
          * Enforce hysteresis, and don't invoke force_quiescent_state()
@@ -1464,12 +1448,27 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
          * is the only one waiting for a grace period to complete.
          */
         if (unlikely(++rdp->qlen > rdp->qlen_last_fqs_check + qhimark)) {
-                rdp->blimit = LONG_MAX;
-                if (rsp->n_force_qs == rdp->n_force_qs_snap &&
-                    *rdp->nxttail[RCU_DONE_TAIL] != head)
-                        force_quiescent_state(rsp, 0);
-                rdp->n_force_qs_snap = rsp->n_force_qs;
-                rdp->qlen_last_fqs_check = rdp->qlen;
+
+                /* Are we ignoring a completed grace period? */
+                rcu_process_gp_end(rsp, rdp);
+                check_for_new_grace_period(rsp, rdp);
+
+                /* Start a new grace period if one not already started. */
+                if (!rcu_gp_in_progress(rsp)) {
+                        unsigned long nestflag;
+                        struct rcu_node *rnp_root = rcu_get_root(rsp);
+
+                        raw_spin_lock_irqsave(&rnp_root->lock, nestflag);
+                        rcu_start_gp(rsp, nestflag);  /* rlses rnp_root->lock */
+                } else {
+                        /* Give the grace period a kick. */
+                        rdp->blimit = LONG_MAX;
+                        if (rsp->n_force_qs == rdp->n_force_qs_snap &&
+                            *rdp->nxttail[RCU_DONE_TAIL] != head)
+                                force_quiescent_state(rsp, 0);
+                        rdp->n_force_qs_snap = rsp->n_force_qs;
+                        rdp->qlen_last_fqs_check = rdp->qlen;
+                }
         } else if (ULONG_CMP_LT(ACCESS_ONCE(rsp->jiffies_force_qs), jiffies))
                 force_quiescent_state(rsp, 1);
         local_irq_restore(flags);
@@ -1699,13 +1698,12 @@ static void _rcu_barrier(struct rcu_state *rsp,
          * decrement rcu_barrier_cpu_count -- otherwise the first CPU
          * might complete its grace period before all of the other CPUs
          * did their increment, causing this function to return too
-         * early.
+         * early.  Note that on_each_cpu() disables irqs, which prevents
+         * any CPUs from coming online or going offline until each online
+         * CPU has queued its RCU-barrier callback.
          */
         atomic_set(&rcu_barrier_cpu_count, 1);
-        preempt_disable(); /* stop CPU_DYING from filling orphan_cbs_list */
-        rcu_adopt_orphan_cbs(rsp);
         on_each_cpu(rcu_barrier_func, (void *)call_rcu_func, 1);
-        preempt_enable(); /* CPU_DYING can again fill orphan_cbs_list */
         if (atomic_dec_and_test(&rcu_barrier_cpu_count))
                 complete(&rcu_barrier_completion);
         wait_for_completion(&rcu_barrier_completion);
@@ -1831,18 +1829,13 @@ static int __cpuinit rcu_cpu_notify(struct notifier_block *self,
         case CPU_DYING:
         case CPU_DYING_FROZEN:
                 /*
-                 * preempt_disable() in _rcu_barrier() prevents stop_machine(),
-                 * so when "on_each_cpu(rcu_barrier_func, (void *)type, 1);"
-                 * returns, all online cpus have queued rcu_barrier_func().
-                 * The dying CPU clears its cpu_online_mask bit and
-                 * moves all of its RCU callbacks to ->orphan_cbs_list
-                 * in the context of stop_machine(), so subsequent calls
-                 * to _rcu_barrier() will adopt these callbacks and only
-                 * then queue rcu_barrier_func() on all remaining CPUs.
+                 * The whole machine is "stopped" except this CPU, so we can
+                 * touch any data without introducing corruption. We send the
+                 * dying CPU's callbacks to an arbitrarily chosen online CPU.
                  */
-                rcu_send_cbs_to_orphanage(&rcu_bh_state);
-                rcu_send_cbs_to_orphanage(&rcu_sched_state);
-                rcu_preempt_send_cbs_to_orphanage();
+                rcu_send_cbs_to_online(&rcu_bh_state);
+                rcu_send_cbs_to_online(&rcu_sched_state);
+                rcu_preempt_send_cbs_to_online();
                 break;
         case CPU_DEAD:
         case CPU_DEAD_FROZEN:
@@ -1880,8 +1873,9 @@ static void __init rcu_init_levelspread(struct rcu_state *rsp)
 {
         int i;
 
-        for (i = NUM_RCU_LVLS - 1; i >= 0; i--)
+        for (i = NUM_RCU_LVLS - 1; i > 0; i--)
                 rsp->levelspread[i] = CONFIG_RCU_FANOUT;
+        rsp->levelspread[0] = RCU_FANOUT_LEAF;
 }
 #else /* #ifdef CONFIG_RCU_FANOUT_EXACT */
 static void __init rcu_init_levelspread(struct rcu_state *rsp)