aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/rcu/tree_plugin.h
diff options
context:
space:
mode:
authorPaul E. McKenney <paulmck@linux.vnet.ibm.com>2014-08-12 13:47:48 -0400
committerPaul E. McKenney <paulmck@linux.vnet.ibm.com>2014-09-16 13:08:03 -0400
commit39953dfd40077c7480b1d5deb4d617e086b1c865 (patch)
tree5839c9da287c82a7c7103199a3b793ecb5391f2f /kernel/rcu/tree_plugin.h
parent663e131090dd10bac9dc0b4f5b624dd3211b20f6 (diff)
rcu: Avoid misordering in __call_rcu_nocb_enqueue()
The NOCB leader wakeup ordering depends on the store to the header happening before the check for the leader already being awake. However, because atomic_long_add() does not return a value, it does not provide ordering guarantees, the incorrect comment in wake_nocb_leader() notwithstanding. This commit therefore adds a smp_mb__after_atomic() after the final atomic_long_add() to provide the needed ordering guarantee. Reported-by: Amit Shah <amit.shah@redhat.com> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Tested-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Diffstat (limited to 'kernel/rcu/tree_plugin.h')
-rw-r--r--kernel/rcu/tree_plugin.h3
1 files changed, 2 insertions, 1 deletions
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index d5aec549558d..4ad63d861599 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -2042,7 +2042,7 @@ static void wake_nocb_leader(struct rcu_data *rdp, bool force)
2042 if (!ACCESS_ONCE(rdp_leader->nocb_kthread)) 2042 if (!ACCESS_ONCE(rdp_leader->nocb_kthread))
2043 return; 2043 return;
2044 if (ACCESS_ONCE(rdp_leader->nocb_leader_sleep) || force) { 2044 if (ACCESS_ONCE(rdp_leader->nocb_leader_sleep) || force) {
2045 /* Prior xchg orders against prior callback enqueue. */ 2045 /* Prior smp_mb__after_atomic() orders against prior enqueue. */
2046 ACCESS_ONCE(rdp_leader->nocb_leader_sleep) = false; 2046 ACCESS_ONCE(rdp_leader->nocb_leader_sleep) = false;
2047 wake_up(&rdp_leader->nocb_wq); 2047 wake_up(&rdp_leader->nocb_wq);
2048 } 2048 }
@@ -2071,6 +2071,7 @@ static void __call_rcu_nocb_enqueue(struct rcu_data *rdp,
2071 ACCESS_ONCE(*old_rhpp) = rhp; 2071 ACCESS_ONCE(*old_rhpp) = rhp;
2072 atomic_long_add(rhcount, &rdp->nocb_q_count); 2072 atomic_long_add(rhcount, &rdp->nocb_q_count);
2073 atomic_long_add(rhcount_lazy, &rdp->nocb_q_count_lazy); 2073 atomic_long_add(rhcount_lazy, &rdp->nocb_q_count_lazy);
2074 smp_mb__after_atomic(); /* Store *old_rhpp before _wake test. */
2074 2075
2075 /* If we are not being polled and there is a kthread, awaken it ... */ 2076 /* If we are not being polled and there is a kthread, awaken it ... */
2076 t = ACCESS_ONCE(rdp->nocb_kthread); 2077 t = ACCESS_ONCE(rdp->nocb_kthread);