rcu: Make rcutorture's batches-completed API use ->gp_seq

The rcutorture test invokes rcu_batches_started(),
rcu_batches_completed(), rcu_batches_started_bh(),
rcu_batches_completed_bh(), rcu_batches_started_sched(), and
rcu_batches_completed_sched() to do grace-period consistency checks,
and rcuperf uses the _completed variants for statistics.
These functions use ->gpnum and ->completed.  This commit therefore
replaces them with rcu_get_gp_seq(), rcu_bh_get_gp_seq(), and
rcu_sched_get_gp_seq(), adjusting rcutorture and rcuperf to make
use of them.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

4 files changed, 45 insertions, 100 deletions

diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h
index db0870acfdff..f0907f9f6cd0 100644
--- a/kernel/rcu/rcu.h
+++ b/kernel/rcu/rcu.h
@@ -463,12 +463,9 @@ void srcutorture_get_gp_data(enum rcutorture_type test_type,
 #endif
 
 #ifdef CONFIG_TINY_RCU
-static inline unsigned long rcu_batches_started(void) { return 0; }
-static inline unsigned long rcu_batches_started_bh(void) { return 0; }
-static inline unsigned long rcu_batches_started_sched(void) { return 0; }
-static inline unsigned long rcu_batches_completed(void) { return 0; }
-static inline unsigned long rcu_batches_completed_bh(void) { return 0; }
-static inline unsigned long rcu_batches_completed_sched(void) { return 0; }
+static inline unsigned long rcu_get_gp_seq(void) { return 0; }
+static inline unsigned long rcu_bh_get_gp_seq(void) { return 0; }
+static inline unsigned long rcu_sched_get_gp_seq(void) { return 0; }
 static inline unsigned long rcu_exp_batches_completed(void) { return 0; }
 static inline unsigned long rcu_exp_batches_completed_sched(void) { return 0; }
 static inline unsigned long
@@ -480,12 +477,9 @@ static inline void show_rcu_gp_kthreads(void) { }
 #else /* #ifdef CONFIG_TINY_RCU */
 extern unsigned long rcutorture_testseq;
 extern unsigned long rcutorture_vernum;
-unsigned long rcu_batches_started(void);
-unsigned long rcu_batches_started_bh(void);
-unsigned long rcu_batches_started_sched(void);
-unsigned long rcu_batches_completed(void);
-unsigned long rcu_batches_completed_bh(void);
-unsigned long rcu_batches_completed_sched(void);
+unsigned long rcu_get_gp_seq(void);
+unsigned long rcu_bh_get_gp_seq(void);
+unsigned long rcu_sched_get_gp_seq(void);
 unsigned long rcu_exp_batches_completed(void);
 unsigned long rcu_exp_batches_completed_sched(void);
 unsigned long srcu_batches_completed(struct srcu_struct *sp);
diff --git a/kernel/rcu/rcuperf.c b/kernel/rcu/rcuperf.c
index df29119b2013..2b5a613afcf3 100644
--- a/kernel/rcu/rcuperf.c
+++ b/kernel/rcu/rcuperf.c
@@ -138,8 +138,7 @@ struct rcu_perf_ops {
         void (*cleanup)(void);
         int (*readlock)(void);
         void (*readunlock)(int idx);
-        unsigned long (*started)(void);
-        unsigned long (*completed)(void);
+        unsigned long (*get_gp_seq)(void);
         unsigned long (*exp_completed)(void);
         void (*async)(struct rcu_head *head, rcu_callback_t func);
         void (*gp_barrier)(void);
@@ -179,8 +178,7 @@ static struct rcu_perf_ops rcu_ops = {
         .init           = rcu_sync_perf_init,
         .readlock       = rcu_perf_read_lock,
         .readunlock     = rcu_perf_read_unlock,
-        .started        = rcu_batches_started,
-        .completed      = rcu_batches_completed,
+        .get_gp_seq     = rcu_get_gp_seq,
         .exp_completed  = rcu_exp_batches_completed,
         .async          = call_rcu,
         .gp_barrier     = rcu_barrier,
@@ -209,8 +207,7 @@ static struct rcu_perf_ops rcu_bh_ops = {
         .init           = rcu_sync_perf_init,
         .readlock       = rcu_bh_perf_read_lock,
         .readunlock     = rcu_bh_perf_read_unlock,
-        .started        = rcu_batches_started_bh,
-        .completed      = rcu_batches_completed_bh,
+        .get_gp_seq     = rcu_bh_get_gp_seq,
         .exp_completed  = rcu_exp_batches_completed_sched,
         .async          = call_rcu_bh,
         .gp_barrier     = rcu_barrier_bh,
@@ -266,8 +263,7 @@ static struct rcu_perf_ops srcu_ops = {
         .init           = rcu_sync_perf_init,
         .readlock       = srcu_perf_read_lock,
         .readunlock     = srcu_perf_read_unlock,
-        .started        = NULL,
-        .completed      = srcu_perf_completed,
+        .get_gp_seq     = srcu_perf_completed,
         .exp_completed  = srcu_perf_completed,
         .async          = srcu_call_rcu,
         .gp_barrier     = srcu_rcu_barrier,
@@ -295,8 +291,7 @@ static struct rcu_perf_ops srcud_ops = {
         .cleanup        = srcu_sync_perf_cleanup,
         .readlock       = srcu_perf_read_lock,
         .readunlock     = srcu_perf_read_unlock,
-        .started        = NULL,
-        .completed      = srcu_perf_completed,
+        .get_gp_seq     = srcu_perf_completed,
         .exp_completed  = srcu_perf_completed,
         .async          = srcu_call_rcu,
         .gp_barrier     = srcu_rcu_barrier,
@@ -325,8 +320,7 @@ static struct rcu_perf_ops sched_ops = {
         .init           = rcu_sync_perf_init,
         .readlock       = sched_perf_read_lock,
         .readunlock     = sched_perf_read_unlock,
-        .started        = rcu_batches_started_sched,
-        .completed      = rcu_batches_completed_sched,
+        .get_gp_seq     = rcu_sched_get_gp_seq,
         .exp_completed  = rcu_exp_batches_completed_sched,
         .async          = call_rcu_sched,
         .gp_barrier     = rcu_barrier_sched,
@@ -353,8 +347,7 @@ static struct rcu_perf_ops tasks_ops = {
         .init           = rcu_sync_perf_init,
         .readlock       = tasks_perf_read_lock,
         .readunlock     = tasks_perf_read_unlock,
-        .started        = rcu_no_completed,
-        .completed      = rcu_no_completed,
+        .get_gp_seq     = rcu_no_completed,
         .async          = call_rcu_tasks,
         .gp_barrier     = rcu_barrier_tasks,
         .sync           = synchronize_rcu_tasks,
@@ -447,8 +440,7 @@ rcu_perf_writer(void *arg)
                         b_rcu_perf_writer_started =
                                 cur_ops->exp_completed() / 2;
                 } else {
-                        b_rcu_perf_writer_started =
-                                cur_ops->completed();
+                        b_rcu_perf_writer_started = cur_ops->get_gp_seq();
                 }
         }
 
@@ -505,7 +497,7 @@ retry:
                                                 cur_ops->exp_completed() / 2;
                                 } else {
                                         b_rcu_perf_writer_finished =
-                                                cur_ops->completed();
+                                                cur_ops->get_gp_seq();
                                 }
                                 if (shutdown) {
                                         smp_mb(); /* Assign before wake. */
diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c
index 5604bfac8df4..1f66597c7783 100644
--- a/kernel/rcu/rcutorture.c
+++ b/kernel/rcu/rcutorture.c
@@ -264,8 +264,7 @@ struct rcu_torture_ops {
         int (*readlock)(void);
         void (*read_delay)(struct torture_random_state *rrsp);
         void (*readunlock)(int idx);
-        unsigned long (*started)(void);
-        unsigned long (*completed)(void);
+        unsigned long (*get_gp_seq)(void);
         void (*deferred_free)(struct rcu_torture *p);
         void (*sync)(void);
         void (*exp_sync)(void);
@@ -305,10 +304,10 @@ static void rcu_read_delay(struct torture_random_state *rrsp)
          * force_quiescent_state. */
 
         if (!(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) {
-                started = cur_ops->completed();
+                started = cur_ops->get_gp_seq();
                 ts = rcu_trace_clock_local();
                 mdelay(longdelay_ms);
-                completed = cur_ops->completed();
+                completed = cur_ops->get_gp_seq();
                 do_trace_rcu_torture_read(cur_ops->name, NULL, ts,
                                           started, completed);
         }
@@ -400,8 +399,7 @@ static struct rcu_torture_ops rcu_ops = {
         .readlock       = rcu_torture_read_lock,
         .read_delay     = rcu_read_delay,
         .readunlock     = rcu_torture_read_unlock,
-        .started        = rcu_batches_started,
-        .completed      = rcu_batches_completed,
+        .get_gp_seq     = rcu_get_gp_seq,
         .deferred_free  = rcu_torture_deferred_free,
         .sync           = synchronize_rcu,
         .exp_sync       = synchronize_rcu_expedited,
@@ -442,8 +440,7 @@ static struct rcu_torture_ops rcu_bh_ops = {
         .readlock       = rcu_bh_torture_read_lock,
         .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
         .readunlock     = rcu_bh_torture_read_unlock,
-        .started        = rcu_batches_started_bh,
-        .completed      = rcu_batches_completed_bh,
+        .get_gp_seq     = rcu_bh_get_gp_seq,
         .deferred_free  = rcu_bh_torture_deferred_free,
         .sync           = synchronize_rcu_bh,
         .exp_sync       = synchronize_rcu_bh_expedited,
@@ -486,8 +483,7 @@ static struct rcu_torture_ops rcu_busted_ops = {
         .readlock       = rcu_torture_read_lock,
         .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
         .readunlock     = rcu_torture_read_unlock,
-        .started        = rcu_no_completed,
-        .completed      = rcu_no_completed,
+        .get_gp_seq     = rcu_no_completed,
         .deferred_free  = rcu_busted_torture_deferred_free,
         .sync           = synchronize_rcu_busted,
         .exp_sync       = synchronize_rcu_busted,
@@ -575,8 +571,7 @@ static struct rcu_torture_ops srcu_ops = {
         .readlock       = srcu_torture_read_lock,
         .read_delay     = srcu_read_delay,
         .readunlock     = srcu_torture_read_unlock,
-        .started        = NULL,
-        .completed      = srcu_torture_completed,
+        .get_gp_seq     = srcu_torture_completed,
         .deferred_free  = srcu_torture_deferred_free,
         .sync           = srcu_torture_synchronize,
         .exp_sync       = srcu_torture_synchronize_expedited,
@@ -613,8 +608,7 @@ static struct rcu_torture_ops srcud_ops = {
         .readlock       = srcu_torture_read_lock,
         .read_delay     = srcu_read_delay,
         .readunlock     = srcu_torture_read_unlock,
-        .started        = NULL,
-        .completed      = srcu_torture_completed,
+        .get_gp_seq     = srcu_torture_completed,
         .deferred_free  = srcu_torture_deferred_free,
         .sync           = srcu_torture_synchronize,
         .exp_sync       = srcu_torture_synchronize_expedited,
@@ -651,8 +645,7 @@ static struct rcu_torture_ops sched_ops = {
         .readlock       = sched_torture_read_lock,
         .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
         .readunlock     = sched_torture_read_unlock,
-        .started        = rcu_batches_started_sched,
-        .completed      = rcu_batches_completed_sched,
+        .get_gp_seq     = rcu_sched_get_gp_seq,
         .deferred_free  = rcu_sched_torture_deferred_free,
         .sync           = synchronize_sched,
         .exp_sync       = synchronize_sched_expedited,
@@ -690,8 +683,7 @@ static struct rcu_torture_ops tasks_ops = {
         .readlock       = tasks_torture_read_lock,
         .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
         .readunlock     = tasks_torture_read_unlock,
-        .started        = rcu_no_completed,
-        .completed      = rcu_no_completed,
+        .get_gp_seq     = rcu_no_completed,
         .deferred_free  = rcu_tasks_torture_deferred_free,
         .sync           = synchronize_rcu_tasks,
         .exp_sync       = synchronize_rcu_tasks,
@@ -1104,10 +1096,7 @@ static void rcu_torture_timer(struct timer_list *unused)
         unsigned long long ts;
 
         idx = cur_ops->readlock();
-        if (cur_ops->started)
-                started = cur_ops->started();
-        else
-                started = cur_ops->completed();
+        started = cur_ops->get_gp_seq();
         ts = rcu_trace_clock_local();
         p = rcu_dereference_check(rcu_torture_current,
                                   rcu_read_lock_bh_held() ||
@@ -1131,7 +1120,7 @@ static void rcu_torture_timer(struct timer_list *unused)
                 /* Should not happen, but... */
                 pipe_count = RCU_TORTURE_PIPE_LEN;
         }
-        completed = cur_ops->completed();
+        completed = cur_ops->get_gp_seq();
         if (pipe_count > 1) {
                 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, ts,
                                           started, completed);
@@ -1139,8 +1128,8 @@ static void rcu_torture_timer(struct timer_list *unused)
         }
         __this_cpu_inc(rcu_torture_count[pipe_count]);
         completed = completed - started;
-        if (cur_ops->started)
-                completed++;
+        if (completed > ULONG_MAX >> 1)
+                completed = 0; /* Not all gp_seq have full range. */
         if (completed > RCU_TORTURE_PIPE_LEN) {
                 /* Should not happen, but... */
                 completed = RCU_TORTURE_PIPE_LEN;
@@ -1187,10 +1176,7 @@ rcu_torture_reader(void *arg)
                                 mod_timer(&t, jiffies + 1);
                 }
                 idx = cur_ops->readlock();
-                if (cur_ops->started)
-                        started = cur_ops->started();
-                else
-                        started = cur_ops->completed();
+                started = cur_ops->get_gp_seq();
                 ts = rcu_trace_clock_local();
                 p = rcu_dereference_check(rcu_torture_current,
                                           rcu_read_lock_bh_held() ||
@@ -1212,7 +1198,7 @@ rcu_torture_reader(void *arg)
                         /* Should not happen, but... */
                         pipe_count = RCU_TORTURE_PIPE_LEN;
                 }
-                completed = cur_ops->completed();
+                completed = cur_ops->get_gp_seq();
                 if (pipe_count > 1) {
                         do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu,
                                                   ts, started, completed);
@@ -1220,8 +1206,8 @@ rcu_torture_reader(void *arg)
                 }
                 __this_cpu_inc(rcu_torture_count[pipe_count]);
                 completed = completed - started;
-                if (cur_ops->started)
-                        completed++;
+                if (completed > ULONG_MAX >> 1)
+                        completed = 0; /* Not all gp_seq have full range. */
                 if (completed > RCU_TORTURE_PIPE_LEN) {
                         /* Should not happen, but... */
                         completed = RCU_TORTURE_PIPE_LEN;
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 3c3af7e2758f..547112bec26a 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -530,58 +530,31 @@ static void force_quiescent_state(struct rcu_state *rsp);
 static int rcu_pending(void);
 
 /*
- * Return the number of RCU batches started thus far for debug & stats.
+ * Return the number of RCU GPs completed thus far for debug & stats.
  */
-unsigned long rcu_batches_started(void)
+unsigned long rcu_get_gp_seq(void)
 {
-        return rcu_state_p->gpnum;
+        return rcu_seq_ctr(READ_ONCE(rcu_state_p->gp_seq));
 }
-EXPORT_SYMBOL_GPL(rcu_batches_started);
+EXPORT_SYMBOL_GPL(rcu_get_gp_seq);
 
 /*
- * Return the number of RCU-sched batches started thus far for debug & stats.
+ * Return the number of RCU-sched GPs completed thus far for debug & stats.
  */
-unsigned long rcu_batches_started_sched(void)
+unsigned long rcu_sched_get_gp_seq(void)
 {
-        return rcu_sched_state.gpnum;
+        return rcu_seq_ctr(READ_ONCE(rcu_sched_state.gp_seq));
 }
-EXPORT_SYMBOL_GPL(rcu_batches_started_sched);
+EXPORT_SYMBOL_GPL(rcu_sched_get_gp_seq);
 
 /*
- * Return the number of RCU BH batches started thus far for debug & stats.
+ * Return the number of RCU-bh GPs completed thus far for debug & stats.
  */
-unsigned long rcu_batches_started_bh(void)
+unsigned long rcu_bh_get_gp_seq(void)
 {
-        return rcu_bh_state.gpnum;
+        return rcu_seq_ctr(READ_ONCE(rcu_bh_state.gp_seq));
 }
-EXPORT_SYMBOL_GPL(rcu_batches_started_bh);
-
-/*
- * Return the number of RCU batches completed thus far for debug & stats.
- */
-unsigned long rcu_batches_completed(void)
-{
-        return rcu_state_p->completed;
-}
-EXPORT_SYMBOL_GPL(rcu_batches_completed);
-
-/*
- * Return the number of RCU-sched batches completed thus far for debug & stats.
- */
-unsigned long rcu_batches_completed_sched(void)
-{
-        return rcu_sched_state.completed;
-}
-EXPORT_SYMBOL_GPL(rcu_batches_completed_sched);
-
-/*
- * Return the number of RCU BH batches completed thus far for debug & stats.
- */
-unsigned long rcu_batches_completed_bh(void)
-{
-        return rcu_bh_state.completed;
-}
-EXPORT_SYMBOL_GPL(rcu_batches_completed_bh);
+EXPORT_SYMBOL_GPL(rcu_bh_get_gp_seq);
 
 /*
  * Return the number of RCU expedited batches completed thus far for