1 files changed, 89 insertions, 13 deletions

diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 08a765232432..6e2ef4b2b920 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -204,6 +204,7 @@ static void rcu_preempt_note_context_switch(int cpu)
                 rdp = per_cpu_ptr(rcu_preempt_state.rda, cpu);
                 rnp = rdp->mynode;
                 raw_spin_lock_irqsave(&rnp->lock, flags);
+                smp_mb__after_unlock_lock();
                 t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BLOCKED;
                 t->rcu_blocked_node = rnp;
 
@@ -312,6 +313,7 @@ static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp, unsigned long flags)
         mask = rnp->grpmask;
         raw_spin_unlock(&rnp->lock);    /* irqs remain disabled. */
         raw_spin_lock(&rnp_p->lock);    /* irqs already disabled. */
+        smp_mb__after_unlock_lock();
         rcu_report_qs_rnp(mask, &rcu_preempt_state, rnp_p, flags);
 }
 
@@ -361,10 +363,14 @@ void rcu_read_unlock_special(struct task_struct *t)
         special = t->rcu_read_unlock_special;
         if (special & RCU_READ_UNLOCK_NEED_QS) {
                 rcu_preempt_qs(smp_processor_id());
+                if (!t->rcu_read_unlock_special) {
+                        local_irq_restore(flags);
+                        return;
+                }
         }
 
-        /* Hardware IRQ handlers cannot block. */
-        if (in_irq() || in_serving_softirq()) {
+        /* Hardware IRQ handlers cannot block, complain if they get here. */
+        if (WARN_ON_ONCE(in_irq() || in_serving_softirq())) {
                 local_irq_restore(flags);
                 return;
         }
@@ -381,6 +387,7 @@ void rcu_read_unlock_special(struct task_struct *t)
                 for (;;) {
                         rnp = t->rcu_blocked_node;
                         raw_spin_lock(&rnp->lock);  /* irqs already disabled. */
+                        smp_mb__after_unlock_lock();
                         if (rnp == t->rcu_blocked_node)
                                 break;
                         raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
@@ -605,6 +612,7 @@ static int rcu_preempt_offline_tasks(struct rcu_state *rsp,
         while (!list_empty(lp)) {
                 t = list_entry(lp->next, typeof(*t), rcu_node_entry);
                 raw_spin_lock(&rnp_root->lock); /* irqs already disabled */
+                smp_mb__after_unlock_lock();
                 list_del(&t->rcu_node_entry);
                 t->rcu_blocked_node = rnp_root;
                 list_add(&t->rcu_node_entry, lp_root);
@@ -629,6 +637,7 @@ static int rcu_preempt_offline_tasks(struct rcu_state *rsp,
          * in this case.
          */
         raw_spin_lock(&rnp_root->lock); /* irqs already disabled */
+        smp_mb__after_unlock_lock();
         if (rnp_root->boost_tasks != NULL &&
             rnp_root->boost_tasks != rnp_root->gp_tasks &&
             rnp_root->boost_tasks != rnp_root->exp_tasks)
@@ -772,6 +781,7 @@ static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
         unsigned long mask;
 
         raw_spin_lock_irqsave(&rnp->lock, flags);
+        smp_mb__after_unlock_lock();
         for (;;) {
                 if (!sync_rcu_preempt_exp_done(rnp)) {
                         raw_spin_unlock_irqrestore(&rnp->lock, flags);
@@ -779,14 +789,17 @@ static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
                 }
                 if (rnp->parent == NULL) {
                         raw_spin_unlock_irqrestore(&rnp->lock, flags);
-                        if (wake)
+                        if (wake) {
+                                smp_mb(); /* EGP done before wake_up(). */
                                 wake_up(&sync_rcu_preempt_exp_wq);
+                        }
                         break;
                 }
                 mask = rnp->grpmask;
                 raw_spin_unlock(&rnp->lock); /* irqs remain disabled */
                 rnp = rnp->parent;
                 raw_spin_lock(&rnp->lock); /* irqs already disabled */
+                smp_mb__after_unlock_lock();
                 rnp->expmask &= ~mask;
         }
 }
@@ -806,6 +819,7 @@ sync_rcu_preempt_exp_init(struct rcu_state *rsp, struct rcu_node *rnp)
         int must_wait = 0;
 
         raw_spin_lock_irqsave(&rnp->lock, flags);
+        smp_mb__after_unlock_lock();
         if (list_empty(&rnp->blkd_tasks)) {
                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
         } else {
@@ -886,6 +900,7 @@ void synchronize_rcu_expedited(void)
         /* Initialize ->expmask for all non-leaf rcu_node structures. */
         rcu_for_each_nonleaf_node_breadth_first(rsp, rnp) {
                 raw_spin_lock_irqsave(&rnp->lock, flags);
+                smp_mb__after_unlock_lock();
                 rnp->expmask = rnp->qsmaskinit;
                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
         }
@@ -1191,6 +1206,7 @@ static int rcu_boost(struct rcu_node *rnp)
                 return 0;  /* Nothing left to boost. */
 
         raw_spin_lock_irqsave(&rnp->lock, flags);
+        smp_mb__after_unlock_lock();
 
         /*
          * Recheck under the lock: all tasks in need of boosting
@@ -1377,6 +1393,7 @@ static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
         if (IS_ERR(t))
                 return PTR_ERR(t);
         raw_spin_lock_irqsave(&rnp->lock, flags);
+        smp_mb__after_unlock_lock();
         rnp->boost_kthread_task = t;
         raw_spin_unlock_irqrestore(&rnp->lock, flags);
         sp.sched_priority = RCU_BOOST_PRIO;
@@ -1769,6 +1786,7 @@ static void rcu_prepare_for_idle(int cpu)
                         continue;
                 rnp = rdp->mynode;
                 raw_spin_lock(&rnp->lock); /* irqs already disabled. */
+                smp_mb__after_unlock_lock();
                 rcu_accelerate_cbs(rsp, rnp, rdp);
                 raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
         }
@@ -1852,6 +1870,7 @@ static int rcu_oom_notify(struct notifier_block *self,
 
         /* Wait for callbacks from earlier instance to complete. */
         wait_event(oom_callback_wq, atomic_read(&oom_callback_count) == 0);
+        smp_mb(); /* Ensure callback reuse happens after callback invocation. */
 
         /*
          * Prevent premature wakeup: ensure that all increments happen
@@ -2101,7 +2120,8 @@ bool rcu_is_nocb_cpu(int cpu)
 static void __call_rcu_nocb_enqueue(struct rcu_data *rdp,
                                     struct rcu_head *rhp,
                                     struct rcu_head **rhtp,
-                                    int rhcount, int rhcount_lazy)
+                                    int rhcount, int rhcount_lazy,
+                                    unsigned long flags)
 {
         int len;
         struct rcu_head **old_rhpp;
@@ -2122,9 +2142,16 @@ static void __call_rcu_nocb_enqueue(struct rcu_data *rdp,
         }
         len = atomic_long_read(&rdp->nocb_q_count);
         if (old_rhpp == &rdp->nocb_head) {
-                wake_up(&rdp->nocb_wq); /* ... only if queue was empty ... */
+                if (!irqs_disabled_flags(flags)) {
+                        wake_up(&rdp->nocb_wq); /* ... if queue was empty ... */
+                        trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
+                                            TPS("WakeEmpty"));
+                } else {
+                        rdp->nocb_defer_wakeup = true;
+                        trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
+                                            TPS("WakeEmptyIsDeferred"));
+                }
                 rdp->qlen_last_fqs_check = 0;
-                trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("WakeEmpty"));
         } else if (len > rdp->qlen_last_fqs_check + qhimark) {
                 wake_up_process(t); /* ... or if many callbacks queued. */
                 rdp->qlen_last_fqs_check = LONG_MAX / 2;
@@ -2145,12 +2172,12 @@ static void __call_rcu_nocb_enqueue(struct rcu_data *rdp,
  * "rcuo" kthread can find it.
  */
 static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
-                            bool lazy)
+                            bool lazy, unsigned long flags)
 {
 
         if (!rcu_is_nocb_cpu(rdp->cpu))
                 return 0;
-        __call_rcu_nocb_enqueue(rdp, rhp, &rhp->next, 1, lazy);
+        __call_rcu_nocb_enqueue(rdp, rhp, &rhp->next, 1, lazy, flags);
         if (__is_kfree_rcu_offset((unsigned long)rhp->func))
                 trace_rcu_kfree_callback(rdp->rsp->name, rhp,
                                          (unsigned long)rhp->func,
@@ -2168,7 +2195,8 @@ static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
  * not a no-CBs CPU.
  */
 static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
-                                                     struct rcu_data *rdp)
+                                                     struct rcu_data *rdp,
+                                                     unsigned long flags)
 {
         long ql = rsp->qlen;
         long qll = rsp->qlen_lazy;
@@ -2182,14 +2210,14 @@ static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
         /* First, enqueue the donelist, if any.  This preserves CB ordering. */
         if (rsp->orphan_donelist != NULL) {
                 __call_rcu_nocb_enqueue(rdp, rsp->orphan_donelist,
-                                        rsp->orphan_donetail, ql, qll);
+                                        rsp->orphan_donetail, ql, qll, flags);
                 ql = qll = 0;
                 rsp->orphan_donelist = NULL;
                 rsp->orphan_donetail = &rsp->orphan_donelist;
         }
         if (rsp->orphan_nxtlist != NULL) {
                 __call_rcu_nocb_enqueue(rdp, rsp->orphan_nxtlist,
-                                        rsp->orphan_nxttail, ql, qll);
+                                        rsp->orphan_nxttail, ql, qll, flags);
                 ql = qll = 0;
                 rsp->orphan_nxtlist = NULL;
                 rsp->orphan_nxttail = &rsp->orphan_nxtlist;
@@ -2209,6 +2237,7 @@ static void rcu_nocb_wait_gp(struct rcu_data *rdp)
         struct rcu_node *rnp = rdp->mynode;
 
         raw_spin_lock_irqsave(&rnp->lock, flags);
+        smp_mb__after_unlock_lock();
         c = rcu_start_future_gp(rnp, rdp);
         raw_spin_unlock_irqrestore(&rnp->lock, flags);
 
@@ -2250,6 +2279,7 @@ static int rcu_nocb_kthread(void *arg)
                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
                                             TPS("Sleep"));
                         wait_event_interruptible(rdp->nocb_wq, rdp->nocb_head);
+                        /* Memory barrier provide by xchg() below. */
                 } else if (firsttime) {
                         firsttime = 0;
                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
@@ -2310,6 +2340,22 @@ static int rcu_nocb_kthread(void *arg)
         return 0;
 }
 
+/* Is a deferred wakeup of rcu_nocb_kthread() required? */
+static bool rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)
+{
+        return ACCESS_ONCE(rdp->nocb_defer_wakeup);
+}
+
+/* Do a deferred wakeup of rcu_nocb_kthread(). */
+static void do_nocb_deferred_wakeup(struct rcu_data *rdp)
+{
+        if (!rcu_nocb_need_deferred_wakeup(rdp))
+                return;
+        ACCESS_ONCE(rdp->nocb_defer_wakeup) = false;
+        wake_up(&rdp->nocb_wq);
+        trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("DeferredWakeEmpty"));
+}
+
 /* Initialize per-rcu_data variables for no-CBs CPUs. */
 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
 {
@@ -2365,13 +2411,14 @@ static void rcu_init_one_nocb(struct rcu_node *rnp)
 }
 
 static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
-                            bool lazy)
+                            bool lazy, unsigned long flags)
 {
         return 0;
 }
 
 static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
-                                                     struct rcu_data *rdp)
+                                                     struct rcu_data *rdp,
+                                                     unsigned long flags)
 {
         return 0;
 }
@@ -2380,6 +2427,15 @@ static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
 {
 }
 
+static bool rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)
+{
+        return false;
+}
+
+static void do_nocb_deferred_wakeup(struct rcu_data *rdp)
+{
+}
+
 static void __init rcu_spawn_nocb_kthreads(struct rcu_state *rsp)
 {
 }
@@ -2829,3 +2885,23 @@ static void rcu_sysidle_init_percpu_data(struct rcu_dynticks *rdtp)
 }
 
 #endif /* #else #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
+
+/*
+ * Is this CPU a NO_HZ_FULL CPU that should ignore RCU so that the
+ * grace-period kthread will do force_quiescent_state() processing?
+ * The idea is to avoid waking up RCU core processing on such a
+ * CPU unless the grace period has extended for too long.
+ *
+ * This code relies on the fact that all NO_HZ_FULL CPUs are also
+ * CONFIG_RCU_NOCB_CPUs.
+ */
+static bool rcu_nohz_full_cpu(struct rcu_state *rsp)
+{
+#ifdef CONFIG_NO_HZ_FULL
+        if (tick_nohz_full_cpu(smp_processor_id()) &&
+            (!rcu_gp_in_progress(rsp) ||
+             ULONG_CMP_LT(jiffies, ACCESS_ONCE(rsp->gp_start) + HZ)))
+                return 1;
+#endif /* #ifdef CONFIG_NO_HZ_FULL */
+        return 0;
+}