1 files changed, 14 insertions, 5 deletions

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index cb4e2056ccf3..14e283c351f6 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -281,6 +281,19 @@ static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
 #endif /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
 };
 
+/*
+ * Do a double-increment of the ->dynticks counter to emulate a
+ * momentary idle-CPU quiescent state.
+ */
+static void rcu_dynticks_momentary_idle(void)
+{
+        struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+        int special = atomic_add_return(2, &rdtp->dynticks);
+
+        /* It is illegal to call this from idle state. */
+        WARN_ON_ONCE(!(special & 0x1));
+}
+
 DEFINE_PER_CPU_SHARED_ALIGNED(unsigned long, rcu_qs_ctr);
 EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr);
 
@@ -300,7 +313,6 @@ EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr);
 static void rcu_momentary_dyntick_idle(void)
 {
         struct rcu_data *rdp;
-        struct rcu_dynticks *rdtp;
         int resched_mask;
         struct rcu_state *rsp;
 
@@ -327,10 +339,7 @@ static void rcu_momentary_dyntick_idle(void)
                  * quiescent state, with no need for this CPU to do anything
                  * further.
                  */
-                rdtp = this_cpu_ptr(&rcu_dynticks);
-                smp_mb__before_atomic(); /* Earlier stuff before QS. */
-                atomic_add(2, &rdtp->dynticks);  /* QS. */
-                smp_mb__after_atomic(); /* Later stuff after QS. */
+                rcu_dynticks_momentary_idle();
                 break;
         }
 }