rcu: Switch ->dynticks to rcu_data structure, remove rcu_dynticks

This commit move ->dynticks from the rcu_dynticks structure to the
rcu_data structure, replacing the field of the same name.  It also updates
the code to access ->dynticks from the rcu_data structure and to use the
rcu_data structure rather than following to now-gone ->dynticks field
to the now-gone rcu_dynticks structure.  While in the area, this commit
also fixes up comments.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

4 files changed, 56 insertions, 80 deletions

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index bfa264a6f3fc..32f500fb24d3 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -73,9 +73,20 @@
 
 /* Data structures. */
 
+/*
+ * Steal a bit from the bottom of ->dynticks for idle entry/exit
+ * control.  Initially this is for TLB flushing.
+ */
+#define RCU_DYNTICK_CTRL_MASK 0x1
+#define RCU_DYNTICK_CTRL_CTR  (RCU_DYNTICK_CTRL_MASK + 1)
+#ifndef rcu_eqs_special_exit
+#define rcu_eqs_special_exit() do { } while (0)
+#endif
+
 static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = {
         .dynticks_nesting = 1,
         .dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE,
+        .dynticks = ATOMIC_INIT(RCU_DYNTICK_CTRL_CTR),
 };
 struct rcu_state rcu_state = {
         .level = { &rcu_state.node[0] },
@@ -203,26 +214,12 @@ void rcu_softirq_qs(void)
 }
 
 /*
- * Steal a bit from the bottom of ->dynticks for idle entry/exit
- * control.  Initially this is for TLB flushing.
- */
-#define RCU_DYNTICK_CTRL_MASK 0x1
-#define RCU_DYNTICK_CTRL_CTR  (RCU_DYNTICK_CTRL_MASK + 1)
-#ifndef rcu_eqs_special_exit
-#define rcu_eqs_special_exit() do { } while (0)
-#endif
-
-static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
-        .dynticks = ATOMIC_INIT(RCU_DYNTICK_CTRL_CTR),
-};
-
-/*
  * Record entry into an extended quiescent state.  This is only to be
  * called when not already in an extended quiescent state.
  */
 static void rcu_dynticks_eqs_enter(void)
 {
-        struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+        struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
         int seq;
 
         /*
@@ -230,7 +227,7 @@ static void rcu_dynticks_eqs_enter(void)
          * critical sections, and we also must force ordering with the
          * next idle sojourn.
          */
-        seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdtp->dynticks);
+        seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
         /* Better be in an extended quiescent state! */
         WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
                      (seq & RCU_DYNTICK_CTRL_CTR));
@@ -245,7 +242,7 @@ static void rcu_dynticks_eqs_enter(void)
  */
 static void rcu_dynticks_eqs_exit(void)
 {
-        struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+        struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
         int seq;
 
         /*
@@ -253,11 +250,11 @@ static void rcu_dynticks_eqs_exit(void)
          * and we also must force ordering with the next RCU read-side
          * critical section.
          */
-        seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdtp->dynticks);
+        seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
         WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
                      !(seq & RCU_DYNTICK_CTRL_CTR));
         if (seq & RCU_DYNTICK_CTRL_MASK) {
-                atomic_andnot(RCU_DYNTICK_CTRL_MASK, &rdtp->dynticks);
+                atomic_andnot(RCU_DYNTICK_CTRL_MASK, &rdp->dynticks);
                 smp_mb__after_atomic(); /* _exit after clearing mask. */
                 /* Prefer duplicate flushes to losing a flush. */
                 rcu_eqs_special_exit();
@@ -276,11 +273,11 @@ static void rcu_dynticks_eqs_exit(void)
  */
 static void rcu_dynticks_eqs_online(void)
 {
-        struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+        struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
 
-        if (atomic_read(&rdtp->dynticks) & RCU_DYNTICK_CTRL_CTR)
+        if (atomic_read(&rdp->dynticks) & RCU_DYNTICK_CTRL_CTR)
                 return;
-        atomic_add(RCU_DYNTICK_CTRL_CTR, &rdtp->dynticks);
+        atomic_add(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
 }
 
 /*
@@ -290,18 +287,18 @@ static void rcu_dynticks_eqs_online(void)
  */
 bool rcu_dynticks_curr_cpu_in_eqs(void)
 {
-        struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+        struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
 
-        return !(atomic_read(&rdtp->dynticks) & RCU_DYNTICK_CTRL_CTR);
+        return !(atomic_read(&rdp->dynticks) & RCU_DYNTICK_CTRL_CTR);
 }
 
 /*
  * Snapshot the ->dynticks counter with full ordering so as to allow
  * stable comparison of this counter with past and future snapshots.
  */
-int rcu_dynticks_snap(struct rcu_dynticks *rdtp)
+int rcu_dynticks_snap(struct rcu_data *rdp)
 {
-        int snap = atomic_add_return(0, &rdtp->dynticks);
+        int snap = atomic_add_return(0, &rdp->dynticks);
 
         return snap & ~RCU_DYNTICK_CTRL_MASK;
 }
@@ -316,13 +313,13 @@ static bool rcu_dynticks_in_eqs(int snap)
 }
 
 /*
- * Return true if the CPU corresponding to the specified rcu_dynticks
+ * Return true if the CPU corresponding to the specified rcu_data
  * structure has spent some time in an extended quiescent state since
  * rcu_dynticks_snap() returned the specified snapshot.
  */
-static bool rcu_dynticks_in_eqs_since(struct rcu_dynticks *rdtp, int snap)
+static bool rcu_dynticks_in_eqs_since(struct rcu_data *rdp, int snap)
 {
-        return snap != rcu_dynticks_snap(rdtp);
+        return snap != rcu_dynticks_snap(rdp);
 }
 
 /*
@@ -336,14 +333,14 @@ bool rcu_eqs_special_set(int cpu)
 {
         int old;
         int new;
-        struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
+        struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
 
         do {
-                old = atomic_read(&rdtp->dynticks);
+                old = atomic_read(&rdp->dynticks);
                 if (old & RCU_DYNTICK_CTRL_CTR)
                         return false;
                 new = old | RCU_DYNTICK_CTRL_MASK;
-        } while (atomic_cmpxchg(&rdtp->dynticks, old, new) != old);
+        } while (atomic_cmpxchg(&rdp->dynticks, old, new) != old);
         return true;
 }
 
@@ -360,11 +357,11 @@ bool rcu_eqs_special_set(int cpu)
  */
 static void __maybe_unused rcu_momentary_dyntick_idle(void)
 {
-        struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
         int special;
 
         raw_cpu_write(rcu_data.rcu_need_heavy_qs, false);
-        special = atomic_add_return(2 * RCU_DYNTICK_CTRL_CTR, &rdtp->dynticks);
+        special = atomic_add_return(2 * RCU_DYNTICK_CTRL_CTR,
+                                    &this_cpu_ptr(&rcu_data)->dynticks);
         /* It is illegal to call this from idle state. */