1 files changed, 54 insertions, 12 deletions

diff --git a/kernel/rcutorture.c b/kernel/rcutorture.c
index 31dea01c85fd..e1f3a8c96724 100644
--- a/kernel/rcutorture.c
+++ b/kernel/rcutorture.c
@@ -46,6 +46,7 @@
 #include <linux/stat.h>
 #include <linux/srcu.h>
 #include <linux/slab.h>
+#include <linux/trace_clock.h>
 #include <asm/byteorder.h>
 
 MODULE_LICENSE("GPL");
@@ -207,6 +208,20 @@ MODULE_PARM_DESC(rcutorture_runnable, "Start rcutorture at boot");
 #define rcu_can_boost() 0
 #endif /* #else #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */
 
+#ifdef CONFIG_RCU_TRACE
+static u64 notrace rcu_trace_clock_local(void)
+{
+        u64 ts = trace_clock_local();
+        unsigned long __maybe_unused ts_rem = do_div(ts, NSEC_PER_USEC);
+        return ts;
+}
+#else /* #ifdef CONFIG_RCU_TRACE */
+static u64 notrace rcu_trace_clock_local(void)
+{
+        return 0ULL;
+}
+#endif /* #else #ifdef CONFIG_RCU_TRACE */
+
 static unsigned long shutdown_time;     /* jiffies to system shutdown. */
 static unsigned long boost_starttime;   /* jiffies of next boost test start. */
 DEFINE_MUTEX(boost_mutex);              /* protect setting boost_starttime */
@@ -845,7 +860,7 @@ static int rcu_torture_boost(void *arg)
                 /* Wait for the next test interval. */
                 oldstarttime = boost_starttime;
                 while (ULONG_CMP_LT(jiffies, oldstarttime)) {
-                        schedule_timeout_uninterruptible(1);
+                        schedule_timeout_interruptible(oldstarttime - jiffies);
                         rcu_stutter_wait("rcu_torture_boost");
                         if (kthread_should_stop() ||
                             fullstop != FULLSTOP_DONTSTOP)
@@ -1028,7 +1043,6 @@ void rcutorture_trace_dump(void)
                 return;
         if (atomic_xchg(&beenhere, 1) != 0)
                 return;
-        do_trace_rcu_torture_read(cur_ops->name, (struct rcu_head *)~0UL);
         ftrace_dump(DUMP_ALL);
 }
 
@@ -1042,13 +1056,16 @@ static void rcu_torture_timer(unsigned long unused)
 {
         int idx;
         int completed;
+        int completed_end;
         static DEFINE_RCU_RANDOM(rand);
         static DEFINE_SPINLOCK(rand_lock);
         struct rcu_torture *p;
         int pipe_count;
+        unsigned long long ts;
 
         idx = cur_ops->readlock();
         completed = cur_ops->completed();
+        ts = rcu_trace_clock_local();
         p = rcu_dereference_check(rcu_torture_current,
                                   rcu_read_lock_bh_held() ||
                                   rcu_read_lock_sched_held() ||
@@ -1058,7 +1075,6 @@ static void rcu_torture_timer(unsigned long unused)
                 cur_ops->readunlock(idx);
                 return;
         }
-        do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu);
         if (p->rtort_mbtest == 0)
                 atomic_inc(&n_rcu_torture_mberror);
         spin_lock(&rand_lock);
@@ -1071,10 +1087,14 @@ static void rcu_torture_timer(unsigned long unused)
                 /* Should not happen, but... */
                 pipe_count = RCU_TORTURE_PIPE_LEN;
         }
-        if (pipe_count > 1)
+        completed_end = cur_ops->completed();
+        if (pipe_count > 1) {
+                do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, ts,
+                                          completed, completed_end);
                 rcutorture_trace_dump();
+        }
         __this_cpu_inc(rcu_torture_count[pipe_count]);
-        completed = cur_ops->completed() - completed;
+        completed = completed_end - completed;
         if (completed > RCU_TORTURE_PIPE_LEN) {
                 /* Should not happen, but... */
                 completed = RCU_TORTURE_PIPE_LEN;
@@ -1094,11 +1114,13 @@ static int
 rcu_torture_reader(void *arg)
 {
         int completed;
+        int completed_end;
         int idx;
         DEFINE_RCU_RANDOM(rand);
         struct rcu_torture *p;
         int pipe_count;
         struct timer_list t;
+        unsigned long long ts;
 
         VERBOSE_PRINTK_STRING("rcu_torture_reader task started");
         set_user_nice(current, 19);
@@ -1112,6 +1134,7 @@ rcu_torture_reader(void *arg)
                 }
                 idx = cur_ops->readlock();
                 completed = cur_ops->completed();
+                ts = rcu_trace_clock_local();
                 p = rcu_dereference_check(rcu_torture_current,
                                           rcu_read_lock_bh_held() ||
                                           rcu_read_lock_sched_held() ||
@@ -1122,7 +1145,6 @@ rcu_torture_reader(void *arg)
                         schedule_timeout_interruptible(HZ);
                         continue;
                 }
-                do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu);
                 if (p->rtort_mbtest == 0)
                         atomic_inc(&n_rcu_torture_mberror);
                 cur_ops->read_delay(&rand);
@@ -1132,10 +1154,14 @@ rcu_torture_reader(void *arg)
                         /* Should not happen, but... */
                         pipe_count = RCU_TORTURE_PIPE_LEN;
                 }
-                if (pipe_count > 1)
+                completed_end = cur_ops->completed();
+                if (pipe_count > 1) {
+                        do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu,
+                                                  ts, completed, completed_end);
                         rcutorture_trace_dump();
+                }
                 __this_cpu_inc(rcu_torture_count[pipe_count]);
-                completed = cur_ops->completed() - completed;
+                completed = completed_end - completed;
                 if (completed > RCU_TORTURE_PIPE_LEN) {
                         /* Should not happen, but... */
                         completed = RCU_TORTURE_PIPE_LEN;
@@ -1301,19 +1327,35 @@ static void rcu_torture_shuffle_tasks(void)
                                 set_cpus_allowed_ptr(reader_tasks[i],
                                                      shuffle_tmp_mask);
         }
-
         if (fakewriter_tasks) {
                 for (i = 0; i < nfakewriters; i++)
                         if (fakewriter_tasks[i])
                                 set_cpus_allowed_ptr(fakewriter_tasks[i],
                                                      shuffle_tmp_mask);
         }
-
         if (writer_task)
                 set_cpus_allowed_ptr(writer_task, shuffle_tmp_mask);
-
         if (stats_task)
                 set_cpus_allowed_ptr(stats_task, shuffle_tmp_mask);
+        if (stutter_task)
+                set_cpus_allowed_ptr(stutter_task, shuffle_tmp_mask);
+        if (fqs_task)
+                set_cpus_allowed_ptr(fqs_task, shuffle_tmp_mask);
+        if (shutdown_task)
+                set_cpus_allowed_ptr(shutdown_task, shuffle_tmp_mask);
+#ifdef CONFIG_HOTPLUG_CPU
+        if (onoff_task)
+                set_cpus_allowed_ptr(onoff_task, shuffle_tmp_mask);
+#endif /* #ifdef CONFIG_HOTPLUG_CPU */
+        if (stall_task)
+                set_cpus_allowed_ptr(stall_task, shuffle_tmp_mask);
+        if (barrier_cbs_tasks)
+                for (i = 0; i < n_barrier_cbs; i++)
+                        if (barrier_cbs_tasks[i])
+                                set_cpus_allowed_ptr(barrier_cbs_tasks[i],
+                                                     shuffle_tmp_mask);
+        if (barrier_task)
+                set_cpus_allowed_ptr(barrier_task, shuffle_tmp_mask);
 
         if (rcu_idle_cpu == -1)
                 rcu_idle_cpu = num_online_cpus() - 1;
@@ -1749,7 +1791,7 @@ static int rcu_torture_barrier_init(void)
         barrier_cbs_wq =
                 kzalloc(n_barrier_cbs * sizeof(barrier_cbs_wq[0]),
                         GFP_KERNEL);
-        if (barrier_cbs_tasks == NULL || barrier_cbs_wq == 0)
+        if (barrier_cbs_tasks == NULL || !barrier_cbs_wq)
                 return -ENOMEM;
         for (i = 0; i < n_barrier_cbs; i++) {
                 init_waitqueue_head(&barrier_cbs_wq[i]);