rcu: Introduce rcutorture testing for rcu_barrier()

Although rcutorture does invoke rcu_barrier() and friends, it cannot
really be called a torture test given that it invokes them only once
at the end of the test.  This commit therefore introduces heavy-duty
rcutorture testing for rcu_barrier(), which may be carried out
concurrently with normal rcutorture testing.

Signed-off-by: Paul E. McKenney <paul.mckenney@linaro.org>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

2 files changed, 200 insertions, 9 deletions

diff --git a/Documentation/RCU/torture.txt b/Documentation/RCU/torture.txt
index 375d3fb71437..4ddf3913fd8c 100644
--- a/Documentation/RCU/torture.txt
+++ b/Documentation/RCU/torture.txt
@@ -47,6 +47,16 @@ irqreader	Says to invoke RCU readers from irq level.  This is currently
                 permit this.  (Or, more accurately, variants of RCU that do
                 -not- permit this know to ignore this variable.)
 
+n_barrier_cbs   If this is nonzero, RCU barrier testing will be conducted,
+                in which case n_barrier_cbs specifies the number of
+                RCU callbacks (and corresponding kthreads) to use for
+                this testing.  The value cannot be negative.  If you
+                specify this to be non-zero when torture_type indicates a
+                synchronous RCU implementation (one for which a member of
+                the synchronize_rcu() rather than the call_rcu() family is
+                used -- see the documentation for torture_type below), an
+                error will be reported and no testing will be carried out.
+
 nfakewriters    This is the number of RCU fake writer threads to run.  Fake
                 writer threads repeatedly use the synchronous "wait for
                 current readers" function of the interface selected by
@@ -188,7 +198,7 @@ OUTPUT
 The statistics output is as follows:
 
         rcu-torture:--- Start of test: nreaders=16 nfakewriters=4 stat_interval=30 verbose=0 test_no_idle_hz=1 shuffle_interval=3 stutter=5 irqreader=1 fqs_duration=0 fqs_holdoff=0 fqs_stutter=3 test_boost=1/0 test_boost_interval=7 test_boost_duration=4
-        rcu-torture: rtc:           (null) ver: 155441 tfle: 0 rta: 155441 rtaf: 8884 rtf: 155440 rtmbe: 0 rtbke: 0 rtbre: 0 rtbf: 0 rtb: 0 nt: 3055767
+        rcu-torture: rtc:           (null) ver: 155441 tfle: 0 rta: 155441 rtaf: 8884 rtf: 155440 rtmbe: 0 rtbe: 0 rtbke: 0 rtbre: 0 rtbf: 0 rtb: 0 nt: 3055767
         rcu-torture: Reader Pipe:  727860534 34213 0 0 0 0 0 0 0 0 0
         rcu-torture: Reader Batch:  727877838 17003 0 0 0 0 0 0 0 0 0
         rcu-torture: Free-Block Circulation:  155440 155440 155440 155440 155440 155440 155440 155440 155440 155440 0
@@ -230,6 +240,9 @@ o	"rtmbe": A non-zero value indicates that rcutorture believes that
         rcu_assign_pointer() and rcu_dereference() are not working
         correctly.  This value should be zero.
 
+o       "rtbe": A non-zero value indicates that one of the rcu_barrier()
+        family of functions is not working correctly.
+
 o       "rtbke": rcutorture was unable to create the real-time kthreads
         used to force RCU priority inversion.  This value should be zero.
 
diff --git a/kernel/rcutorture.c b/kernel/rcutorture.c
index 1463a0636443..8cd262b41499 100644
--- a/kernel/rcutorture.c
+++ b/kernel/rcutorture.c
@@ -64,6 +64,7 @@ static int irqreader = 1;	/* RCU readers from irq (timers). */
 static int fqs_duration;        /* Duration of bursts (us), 0 to disable. */
 static int fqs_holdoff;         /* Hold time within burst (us). */
 static int fqs_stutter = 3;     /* Wait time between bursts (s). */
+static int n_barrier_cbs;       /* Number of callbacks to test RCU barriers. */
 static int onoff_interval;      /* Wait time between CPU hotplugs, 0=disable. */
 static int onoff_holdoff;       /* Seconds after boot before CPU hotplugs. */
 static int shutdown_secs;       /* Shutdown time (s).  <=0 for no shutdown. */
@@ -96,6 +97,8 @@ module_param(fqs_holdoff, int, 0444);
 MODULE_PARM_DESC(fqs_holdoff, "Holdoff time within fqs bursts (us)");
 module_param(fqs_stutter, int, 0444);
 MODULE_PARM_DESC(fqs_stutter, "Wait time between fqs bursts (s)");
+module_param(n_barrier_cbs, int, 0444);
+MODULE_PARM_DESC(n_barrier_cbs, "# of callbacks/kthreads for barrier testing");
 module_param(onoff_interval, int, 0444);
 MODULE_PARM_DESC(onoff_interval, "Time between CPU hotplugs (s), 0=disable");
 module_param(onoff_holdoff, int, 0444);
@@ -139,6 +142,8 @@ static struct task_struct *shutdown_task;
 static struct task_struct *onoff_task;
 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
 static struct task_struct *stall_task;
+static struct task_struct **barrier_cbs_tasks;
+static struct task_struct *barrier_task;
 
 #define RCU_TORTURE_PIPE_LEN 10
 
@@ -164,6 +169,7 @@ static atomic_t n_rcu_torture_alloc_fail;
 static atomic_t n_rcu_torture_free;
 static atomic_t n_rcu_torture_mberror;
 static atomic_t n_rcu_torture_error;
+static long n_rcu_torture_barrier_error;
 static long n_rcu_torture_boost_ktrerror;
 static long n_rcu_torture_boost_rterror;
 static long n_rcu_torture_boost_failure;
@@ -173,6 +179,8 @@ static long n_offline_attempts;
 static long n_offline_successes;
 static long n_online_attempts;
 static long n_online_successes;
+static long n_barrier_attempts;
+static long n_barrier_successes;
 static struct list_head rcu_torture_removed;
 static cpumask_var_t shuffle_tmp_mask;
 
@@ -197,6 +205,10 @@ static unsigned long shutdown_time;	/* jiffies to system shutdown. */
 static unsigned long boost_starttime;   /* jiffies of next boost test start. */
 DEFINE_MUTEX(boost_mutex);              /* protect setting boost_starttime */
                                         /*  and boost task create/destroy. */
+static atomic_t barrier_cbs_count;      /* Barrier callbacks registered. */
+static atomic_t barrier_cbs_invoked;    /* Barrier callbacks invoked. */
+static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */
+static DECLARE_WAIT_QUEUE_HEAD(barrier_wq);
 
 /* Mediate rmmod and system shutdown.  Concurrent rmmod & shutdown illegal! */
 
@@ -327,6 +339,7 @@ struct rcu_torture_ops {
         int (*completed)(void);
         void (*deferred_free)(struct rcu_torture *p);
         void (*sync)(void);
+        void (*call)(struct rcu_head *head, void (*func)(struct rcu_head *rcu));
         void (*cb_barrier)(void);
         void (*fqs)(void);
         int (*stats)(char *page);
@@ -417,6 +430,7 @@ static struct rcu_torture_ops rcu_ops = {
         .completed      = rcu_torture_completed,
         .deferred_free  = rcu_torture_deferred_free,
         .sync           = synchronize_rcu,
+        .call           = call_rcu,
         .cb_barrier     = rcu_barrier,
         .fqs            = rcu_force_quiescent_state,
         .stats          = NULL,
@@ -460,6 +474,7 @@ static struct rcu_torture_ops rcu_sync_ops = {
         .completed      = rcu_torture_completed,
         .deferred_free  = rcu_sync_torture_deferred_free,
         .sync           = synchronize_rcu,
+        .call           = NULL,
         .cb_barrier     = NULL,
         .fqs            = rcu_force_quiescent_state,
         .stats          = NULL,
@@ -477,6 +492,7 @@ static struct rcu_torture_ops rcu_expedited_ops = {
         .completed      = rcu_no_completed,
         .deferred_free  = rcu_sync_torture_deferred_free,
         .sync           = synchronize_rcu_expedited,
+        .call           = NULL,
         .cb_barrier     = NULL,
         .fqs            = rcu_force_quiescent_state,
         .stats          = NULL,
@@ -519,6 +535,7 @@ static struct rcu_torture_ops rcu_bh_ops = {
         .completed      = rcu_bh_torture_completed,
         .deferred_free  = rcu_bh_torture_deferred_free,
         .sync           = synchronize_rcu_bh,
+        .call           = call_rcu_bh,
         .cb_barrier     = rcu_barrier_bh,
         .fqs            = rcu_bh_force_quiescent_state,
         .stats          = NULL,
@@ -535,6 +552,7 @@ static struct rcu_torture_ops rcu_bh_sync_ops = {
         .completed      = rcu_bh_torture_completed,
         .deferred_free  = rcu_sync_torture_deferred_free,
         .sync           = synchronize_rcu_bh,
+        .call           = NULL,
         .cb_barrier     = NULL,
         .fqs            = rcu_bh_force_quiescent_state,
         .stats          = NULL,
@@ -551,6 +569,7 @@ static struct rcu_torture_ops rcu_bh_expedited_ops = {
         .completed      = rcu_bh_torture_completed,
         .deferred_free  = rcu_sync_torture_deferred_free,
         .sync           = synchronize_rcu_bh_expedited,
+        .call           = NULL,
         .cb_barrier     = NULL,
         .fqs            = rcu_bh_force_quiescent_state,
         .stats          = NULL,
@@ -637,6 +656,7 @@ static struct rcu_torture_ops srcu_ops = {
         .completed      = srcu_torture_completed,
         .deferred_free  = rcu_sync_torture_deferred_free,
         .sync           = srcu_torture_synchronize,
+        .call           = NULL,
         .cb_barrier     = NULL,
         .stats          = srcu_torture_stats,
         .name           = "srcu"
@@ -661,6 +681,7 @@ static struct rcu_torture_ops srcu_raw_ops = {
         .completed      = srcu_torture_completed,
         .deferred_free  = rcu_sync_torture_deferred_free,
         .sync           = srcu_torture_synchronize,
+        .call           = NULL,
         .cb_barrier     = NULL,
         .stats          = srcu_torture_stats,
         .name           = "srcu_raw"
@@ -680,6 +701,7 @@ static struct rcu_torture_ops srcu_expedited_ops = {
         .completed      = srcu_torture_completed,
         .deferred_free  = rcu_sync_torture_deferred_free,
         .sync           = srcu_torture_synchronize_expedited,
+        .call           = NULL,
         .cb_barrier     = NULL,
         .stats          = srcu_torture_stats,
         .name           = "srcu_expedited"
@@ -1129,7 +1151,8 @@ rcu_torture_printk(char *page)
                        "rtc: %p ver: %lu tfle: %d rta: %d rtaf: %d rtf: %d "
                        "rtmbe: %d rtbke: %ld rtbre: %ld "
                        "rtbf: %ld rtb: %ld nt: %ld "
-                       "onoff: %ld/%ld:%ld/%ld",
+                       "onoff: %ld/%ld:%ld/%ld "
+                       "barrier: %ld/%ld:%ld",
                        rcu_torture_current,
                        rcu_torture_current_version,
                        list_empty(&rcu_torture_freelist),
@@ -1145,14 +1168,17 @@ rcu_torture_printk(char *page)
                        n_online_successes,
                        n_online_attempts,
                        n_offline_successes,
-                       n_offline_attempts);
+                       n_offline_attempts,
+                       n_barrier_successes,
+                       n_barrier_attempts,
+                       n_rcu_torture_barrier_error);
+        cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
         if (atomic_read(&n_rcu_torture_mberror) != 0 ||
+            n_rcu_torture_barrier_error != 0 ||
             n_rcu_torture_boost_ktrerror != 0 ||
             n_rcu_torture_boost_rterror != 0 ||
-            n_rcu_torture_boost_failure != 0)
-                cnt += sprintf(&page[cnt], " !!!");
-        cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
-        if (i > 1) {
+            n_rcu_torture_boost_failure != 0 ||
+            i > 1) {
                 cnt += sprintf(&page[cnt], "!!! ");
                 atomic_inc(&n_rcu_torture_error);
                 WARN_ON_ONCE(1);
@@ -1560,6 +1586,151 @@ static void rcu_torture_stall_cleanup(void)
         stall_task = NULL;
 }
 
+/* Callback function for RCU barrier testing. */
+void rcu_torture_barrier_cbf(struct rcu_head *rcu)
+{
+        atomic_inc(&barrier_cbs_invoked);
+}
+
+/* kthread function to register callbacks used to test RCU barriers. */
+static int rcu_torture_barrier_cbs(void *arg)
+{
+        long myid = (long)arg;
+        struct rcu_head rcu;
+
+        init_rcu_head_on_stack(&rcu);
+        VERBOSE_PRINTK_STRING("rcu_torture_barrier_cbs task started");
+        set_user_nice(current, 19);
+        do {
+                wait_event(barrier_cbs_wq[myid],
+                           atomic_read(&barrier_cbs_count) == n_barrier_cbs ||
+                           kthread_should_stop() ||
+                           fullstop != FULLSTOP_DONTSTOP);
+                if (kthread_should_stop() || fullstop != FULLSTOP_DONTSTOP)
+                        break;
+                cur_ops->call(&rcu, rcu_torture_barrier_cbf);
+                if (atomic_dec_and_test(&barrier_cbs_count))
+                        wake_up(&barrier_wq);
+        } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
+        VERBOSE_PRINTK_STRING("rcu_torture_barrier_cbs task stopping");
+        rcutorture_shutdown_absorb("rcu_torture_barrier_cbs");
+        while (!kthread_should_stop())
+                schedule_timeout_interruptible(1);
+        cur_ops->cb_barrier();
+        destroy_rcu_head_on_stack(&rcu);
+        return 0;
+}
+
+/* kthread function to drive and coordinate RCU barrier testing. */
+static int rcu_torture_barrier(void *arg)
+{
+        int i;
+
+        VERBOSE_PRINTK_STRING("rcu_torture_barrier task starting");
+        do {
+                atomic_set(&barrier_cbs_invoked, 0);
+                atomic_set(&barrier_cbs_count, n_barrier_cbs);
+                /* wake_up() path contains the required barriers. */
+                for (i = 0; i < n_barrier_cbs; i++)
+                        wake_up(&barrier_cbs_wq[i]);
+                wait_event(barrier_wq,
+                           atomic_read(&barrier_cbs_count) == 0 ||
+                           kthread_should_stop() ||
+                           fullstop != FULLSTOP_DONTSTOP);
+                if (kthread_should_stop() || fullstop != FULLSTOP_DONTSTOP)
+                        break;
+                n_barrier_attempts++;
+                cur_ops->cb_barrier();
+                if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) {
+                        n_rcu_torture_barrier_error++;
+                        WARN_ON_ONCE(1);
+                }
+                n_barrier_successes++;
+                schedule_timeout_interruptible(HZ / 10);
+        } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
+        VERBOSE_PRINTK_STRING("rcu_torture_barrier task stopping");
+        rcutorture_shutdown_absorb("rcu_torture_barrier_cbs");
+        while (!kthread_should_stop())
+                schedule_timeout_interruptible(1);
+        return 0;
+}
+
+/* Initialize RCU barrier testing. */
+static int rcu_torture_barrier_init(void)
+{
+        int i;
+        int ret;
+
+        if (n_barrier_cbs == 0)
+                return 0;
+        if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) {
+                printk(KERN_ALERT "%s" TORTURE_FLAG
+                       " Call or barrier ops missing for %s,\n",
+                       torture_type, cur_ops->name);
+                printk(KERN_ALERT "%s" TORTURE_FLAG
+                       " RCU barrier testing omitted from run.\n",
+                       torture_type);
+                return 0;
+        }
+        atomic_set(&barrier_cbs_count, 0);
+        atomic_set(&barrier_cbs_invoked, 0);
+        barrier_cbs_tasks =
+                kzalloc(n_barrier_cbs * sizeof(barrier_cbs_tasks[0]),
+                        GFP_KERNEL);
+        barrier_cbs_wq =
+                kzalloc(n_barrier_cbs * sizeof(barrier_cbs_wq[0]),
+                        GFP_KERNEL);
+        if (barrier_cbs_tasks == NULL || barrier_cbs_wq == 0)
+                return -ENOMEM;
+        for (i = 0; i < n_barrier_cbs; i++) {
+                init_waitqueue_head(&barrier_cbs_wq[i]);
+                barrier_cbs_tasks[i] = kthread_run(rcu_torture_barrier_cbs,
+                                                   (void *)i,
+                                                   "rcu_torture_barrier_cbs");
+                if (IS_ERR(barrier_cbs_tasks[i])) {
+                        ret = PTR_ERR(barrier_cbs_tasks[i]);
+                        VERBOSE_PRINTK_ERRSTRING("Failed to create rcu_torture_barrier_cbs");
+                        barrier_cbs_tasks[i] = NULL;
+                        return ret;
+                }
+        }
+        barrier_task = kthread_run(rcu_torture_barrier, NULL,
+                                   "rcu_torture_barrier");
+        if (IS_ERR(barrier_task)) {
+                ret = PTR_ERR(barrier_task);
+                VERBOSE_PRINTK_ERRSTRING("Failed to create rcu_torture_barrier");
+                barrier_task = NULL;
+        }
+        return 0;
+}
+
+/* Clean up after RCU barrier testing. */
+static void rcu_torture_barrier_cleanup(void)
+{
+        int i;
+
+        if (barrier_task != NULL) {
+                VERBOSE_PRINTK_STRING("Stopping rcu_torture_barrier task");
+                kthread_stop(barrier_task);
+                barrier_task = NULL;
+        }
+        if (barrier_cbs_tasks != NULL) {
+                for (i = 0; i < n_barrier_cbs; i++) {
+                        if (barrier_cbs_tasks[i] != NULL) {
+                                VERBOSE_PRINTK_STRING("Stopping rcu_torture_barrier_cbs task");
+                                kthread_stop(barrier_cbs_tasks[i]);
+                                barrier_cbs_tasks[i] = NULL;
+                        }
+                }
+                kfree(barrier_cbs_tasks);
+                barrier_cbs_tasks = NULL;
+        }
+        if (barrier_cbs_wq != NULL) {
+                kfree(barrier_cbs_wq);
+                barrier_cbs_wq = NULL;
+        }
+}
+
 static int rcutorture_cpu_notify(struct notifier_block *self,
                                  unsigned long action, void *hcpu)
 {
@@ -1602,6 +1773,7 @@ rcu_torture_cleanup(void)
         fullstop = FULLSTOP_RMMOD;
         mutex_unlock(&fullstop_mutex);
         unregister_reboot_notifier(&rcutorture_shutdown_nb);
+        rcu_torture_barrier_cleanup();
         rcu_torture_stall_cleanup();
         if (stutter_task) {
                 VERBOSE_PRINTK_STRING("Stopping rcu_torture_stutter task");
@@ -1681,7 +1853,7 @@ rcu_torture_cleanup(void)
 
         if (cur_ops->cleanup)
                 cur_ops->cleanup();
-        if (atomic_read(&n_rcu_torture_error))
+        if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error)
                 rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE");
         else if (n_online_successes != n_online_attempts ||
                  n_offline_successes != n_offline_attempts)
@@ -1697,6 +1869,7 @@ rcu_torture_init(void)
         int i;
         int cpu;
         int firsterr = 0;
+        int retval;
         static struct rcu_torture_ops *torture_ops[] =
                 { &rcu_ops, &rcu_sync_ops, &rcu_expedited_ops,
                   &rcu_bh_ops, &rcu_bh_sync_ops, &rcu_bh_expedited_ops,
@@ -1754,6 +1927,7 @@ rcu_torture_init(void)
         atomic_set(&n_rcu_torture_free, 0);
         atomic_set(&n_rcu_torture_mberror, 0);
         atomic_set(&n_rcu_torture_error, 0);
+        n_rcu_torture_barrier_error = 0;
         n_rcu_torture_boost_ktrerror = 0;
         n_rcu_torture_boost_rterror = 0;
         n_rcu_torture_boost_failure = 0;
@@ -1877,7 +2051,6 @@ rcu_torture_init(void)
                 test_boost_duration = 2;
         if ((test_boost == 1 && cur_ops->can_boost) ||
             test_boost == 2) {
-                int retval;
 
                 boost_starttime = jiffies + test_boost_interval * HZ;
                 register_cpu_notifier(&rcutorture_cpu_nb);
@@ -1913,6 +2086,11 @@ rcu_torture_init(void)
                 firsterr = i;
                 goto unwind;
         }
+        retval = rcu_torture_barrier_init();
+        if (retval != 0) {
+                firsterr = retval;
+                goto unwind;
+        }
         rcutorture_record_test_transition();
         mutex_unlock(&fullstop_mutex);
         return 0;