rcu: Abstract the dynticks momentary-idle operation

This commit is the first step towards full abstraction of all accesses to
the ->dynticks counter, implementing the previously open-coded atomic add
of two in a new rcu_dynticks_momentary_idle() function.  This abstraction
will ease changes to the ->dynticks counter operation.

Note that this commit gets rid of the smp_mb__before_atomic() and the
smp_mb__after_atomic() calls that were previously present.  The reason
that this is OK from a memory-ordering perspective is that the atomic
operation is now atomic_add_return(), which, as a value-returning atomic,
guarantees full ordering.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>

1 files changed, 14 insertions, 5 deletions

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index cb4e2056ccf3..14e283c351f6 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -281,6 +281,19 @@ static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
 #endif /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
 };
 
+/*
+ * Do a double-increment of the ->dynticks counter to emulate a
+ * momentary idle-CPU quiescent state.
+ */
+static void rcu_dynticks_momentary_idle(void)
+{
+        struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+        int special = atomic_add_return(2, &rdtp->dynticks);
+
+        /* It is illegal to call this from idle state. */
+        WARN_ON_ONCE(!(special & 0x1));
+}
+
 DEFINE_PER_CPU_SHARED_ALIGNED(unsigned long, rcu_qs_ctr);
 EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr);
 
@@ -300,7 +313,6 @@ EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr);
 static void rcu_momentary_dyntick_idle(void)
 {
         struct rcu_data *rdp;
-        struct rcu_dynticks *rdtp;
         int resched_mask;
         struct rcu_state *rsp;
 
@@ -327,10 +339,7 @@ static void rcu_momentary_dyntick_idle(void)
                  * quiescent state, with no need for this CPU to do anything
                  * further.
                  */
-                rdtp = this_cpu_ptr(&rcu_dynticks);
-                smp_mb__before_atomic(); /* Earlier stuff before QS. */
-                atomic_add(2, &rdtp->dynticks);  /* QS. */
-                smp_mb__after_atomic(); /* Later stuff after QS. */
+                rcu_dynticks_momentary_idle();
                 break;
         }
 }