aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
authorPaul E. McKenney <paulmck@us.ibm.com>2005-10-30 18:03:12 -0500
committerLinus Torvalds <torvalds@g5.osdl.org>2005-10-30 20:37:27 -0500
commita241ec65aeac3d69a08a7b153cccbdb7ea35063f (patch)
treee96585e8b1e699f31bad1fa61f34d2ec7c3a187c /kernel
parentb3099b48da23686d8378133b0264ee00385ee5fa (diff)
[PATCH] RCU torture-testing kernel module
This patch is a rewrite of the one submitted on October 1st, using modules (http://marc.theaimsgroup.com/?l=linux-kernel&m=112819093522998&w=2). This rewrite adds a tristate CONFIG_RCU_TORTURE_TEST, which enables an intense torture test of the RCU infratructure. This is needed due to the continued changes to the RCU infrastructure to accommodate dynamic ticks, CPU hotplug, realtime, and so on. Most of the code is in a separate file that is compiled only if the CONFIG variable is set. Documentation on how to run the test and interpret the output is also included. This code has been tested on i386 and ppc64, and an earlier version of the code has received extensive testing on a number of architectures as part of the PREEMPT_RT patchset. Signed-off-by: "Paul E. McKenney" <paulmck@us.ibm.com> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Diffstat (limited to 'kernel')
-rw-r--r--kernel/Makefile1
-rw-r--r--kernel/rcupdate.c10
-rw-r--r--kernel/rcutorture.c492
3 files changed, 503 insertions, 0 deletions
diff --git a/kernel/Makefile b/kernel/Makefile
index 980b5e454441..4f5a1453093a 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -31,6 +31,7 @@ obj-$(CONFIG_DETECT_SOFTLOCKUP) += softlockup.o
31obj-$(CONFIG_GENERIC_HARDIRQS) += irq/ 31obj-$(CONFIG_GENERIC_HARDIRQS) += irq/
32obj-$(CONFIG_CRASH_DUMP) += crash_dump.o 32obj-$(CONFIG_CRASH_DUMP) += crash_dump.o
33obj-$(CONFIG_SECCOMP) += seccomp.o 33obj-$(CONFIG_SECCOMP) += seccomp.o
34obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o
34 35
35ifneq ($(CONFIG_SCHED_NO_NO_OMIT_FRAME_POINTER),y) 36ifneq ($(CONFIG_SCHED_NO_NO_OMIT_FRAME_POINTER),y)
36# According to Alan Modra <alan@linuxcare.com.au>, the -fno-omit-frame-pointer is 37# According to Alan Modra <alan@linuxcare.com.au>, the -fno-omit-frame-pointer is
diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c
index 2559d4b8f23f..c4d159a21e04 100644
--- a/kernel/rcupdate.c
+++ b/kernel/rcupdate.c
@@ -154,6 +154,15 @@ void fastcall call_rcu_bh(struct rcu_head *head,
154} 154}
155 155
156/* 156/*
157 * Return the number of RCU batches processed thus far. Useful
158 * for debug and statistics.
159 */
160long rcu_batches_completed(void)
161{
162 return rcu_ctrlblk.completed;
163}
164
165/*
157 * Invoke the completed RCU callbacks. They are expected to be in 166 * Invoke the completed RCU callbacks. They are expected to be in
158 * a per-cpu list. 167 * a per-cpu list.
159 */ 168 */
@@ -501,6 +510,7 @@ void synchronize_kernel(void)
501} 510}
502 511
503module_param(maxbatch, int, 0); 512module_param(maxbatch, int, 0);
513EXPORT_SYMBOL_GPL(rcu_batches_completed);
504EXPORT_SYMBOL(call_rcu); /* WARNING: GPL-only in April 2006. */ 514EXPORT_SYMBOL(call_rcu); /* WARNING: GPL-only in April 2006. */
505EXPORT_SYMBOL(call_rcu_bh); /* WARNING: GPL-only in April 2006. */ 515EXPORT_SYMBOL(call_rcu_bh); /* WARNING: GPL-only in April 2006. */
506EXPORT_SYMBOL_GPL(synchronize_rcu); 516EXPORT_SYMBOL_GPL(synchronize_rcu);
diff --git a/kernel/rcutorture.c b/kernel/rcutorture.c
new file mode 100644
index 000000000000..9b58f1eff3ca
--- /dev/null
+++ b/kernel/rcutorture.c
@@ -0,0 +1,492 @@
1/*
2 * Read-Copy Update /proc-based torture test facility
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
18 * Copyright (C) IBM Corporation, 2005
19 *
20 * Authors: Paul E. McKenney <paulmck@us.ibm.com>
21 *
22 * See also: Documentation/RCU/torture.txt
23 */
24#include <linux/types.h>
25#include <linux/kernel.h>
26#include <linux/init.h>
27#include <linux/module.h>
28#include <linux/kthread.h>
29#include <linux/err.h>
30#include <linux/spinlock.h>
31#include <linux/smp.h>
32#include <linux/rcupdate.h>
33#include <linux/interrupt.h>
34#include <linux/sched.h>
35#include <asm/atomic.h>
36#include <linux/bitops.h>
37#include <linux/module.h>
38#include <linux/completion.h>
39#include <linux/moduleparam.h>
40#include <linux/percpu.h>
41#include <linux/notifier.h>
42#include <linux/rcuref.h>
43#include <linux/cpu.h>
44#include <linux/random.h>
45#include <linux/delay.h>
46#include <linux/byteorder/swabb.h>
47#include <linux/stat.h>
48
49MODULE_LICENSE("GPL");
50
51static int nreaders = -1; /* # reader threads, defaults to 4*ncpus */
52static int stat_interval = 0; /* Interval between stats, in seconds. */
53 /* Defaults to "only at end of test". */
54static int verbose = 0; /* Print more debug info. */
55
56MODULE_PARM(nreaders, "i");
57MODULE_PARM_DESC(nreaders, "Number of RCU reader threads");
58MODULE_PARM(stat_interval, "i");
59MODULE_PARM_DESC(stat_interval, "Number of seconds between stats printk()s");
60MODULE_PARM(verbose, "i");
61MODULE_PARM_DESC(verbose, "Enable verbose debugging printk()s");
62#define TORTURE_FLAG "rcutorture: "
63#define PRINTK_STRING(s) \
64 do { printk(KERN_ALERT TORTURE_FLAG s "\n"); } while (0)
65#define VERBOSE_PRINTK_STRING(s) \
66 do { if (verbose) printk(KERN_ALERT TORTURE_FLAG s "\n"); } while (0)
67#define VERBOSE_PRINTK_ERRSTRING(s) \
68 do { if (verbose) printk(KERN_ALERT TORTURE_FLAG "!!! " s "\n"); } while (0)
69
70static char printk_buf[4096];
71
72static int nrealreaders;
73static struct task_struct *writer_task;
74static struct task_struct **reader_tasks;
75static struct task_struct *stats_task;
76
77#define RCU_TORTURE_PIPE_LEN 10
78
79struct rcu_torture {
80 struct rcu_head rtort_rcu;
81 int rtort_pipe_count;
82 struct list_head rtort_free;
83};
84
85static int fullstop = 0; /* stop generating callbacks at test end. */
86static LIST_HEAD(rcu_torture_freelist);
87static struct rcu_torture *rcu_torture_current = NULL;
88static long rcu_torture_current_version = 0;
89static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN];
90static DEFINE_SPINLOCK(rcu_torture_lock);
91static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count) =
92 { 0 };
93static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch) =
94 { 0 };
95static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1];
96atomic_t n_rcu_torture_alloc;
97atomic_t n_rcu_torture_alloc_fail;
98atomic_t n_rcu_torture_free;
99
100/*
101 * Allocate an element from the rcu_tortures pool.
102 */
103struct rcu_torture *
104rcu_torture_alloc(void)
105{
106 struct list_head *p;
107
108 spin_lock(&rcu_torture_lock);
109 if (list_empty(&rcu_torture_freelist)) {
110 atomic_inc(&n_rcu_torture_alloc_fail);
111 spin_unlock(&rcu_torture_lock);
112 return NULL;
113 }
114 atomic_inc(&n_rcu_torture_alloc);
115 p = rcu_torture_freelist.next;
116 list_del_init(p);
117 spin_unlock(&rcu_torture_lock);
118 return container_of(p, struct rcu_torture, rtort_free);
119}
120
121/*
122 * Free an element to the rcu_tortures pool.
123 */
124static void
125rcu_torture_free(struct rcu_torture *p)
126{
127 atomic_inc(&n_rcu_torture_free);
128 spin_lock(&rcu_torture_lock);
129 list_add_tail(&p->rtort_free, &rcu_torture_freelist);
130 spin_unlock(&rcu_torture_lock);
131}
132
133static void
134rcu_torture_cb(struct rcu_head *p)
135{
136 int i;
137 struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu);
138
139 if (fullstop) {
140 /* Test is ending, just drop callbacks on the floor. */
141 /* The next initialization will pick up the pieces. */
142 return;
143 }
144 i = rp->rtort_pipe_count;
145 if (i > RCU_TORTURE_PIPE_LEN)
146 i = RCU_TORTURE_PIPE_LEN;
147 atomic_inc(&rcu_torture_wcount[i]);
148 if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN)
149 rcu_torture_free(rp);
150 else
151 call_rcu(p, rcu_torture_cb);
152}
153
154struct rcu_random_state {
155 unsigned long rrs_state;
156 unsigned long rrs_count;
157};
158
159#define RCU_RANDOM_MULT 39916801 /* prime */
160#define RCU_RANDOM_ADD 479001701 /* prime */
161#define RCU_RANDOM_REFRESH 10000
162
163#define DEFINE_RCU_RANDOM(name) struct rcu_random_state name = { 0, 0 }
164
165/*
166 * Crude but fast random-number generator. Uses a linear congruential
167 * generator, with occasional help from get_random_bytes().
168 */
169static long
170rcu_random(struct rcu_random_state *rrsp)
171{
172 long refresh;
173
174 if (--rrsp->rrs_count < 0) {
175 get_random_bytes(&refresh, sizeof(refresh));
176 rrsp->rrs_state += refresh;
177 rrsp->rrs_count = RCU_RANDOM_REFRESH;
178 }
179 rrsp->rrs_state = rrsp->rrs_state * RCU_RANDOM_MULT + RCU_RANDOM_ADD;
180 return swahw32(rrsp->rrs_state);
181}
182
183/*
184 * RCU torture writer kthread. Repeatedly substitutes a new structure
185 * for that pointed to by rcu_torture_current, freeing the old structure
186 * after a series of grace periods (the "pipeline").
187 */
188static int
189rcu_torture_writer(void *arg)
190{
191 int i;
192 long oldbatch = rcu_batches_completed();
193 struct rcu_torture *rp;
194 struct rcu_torture *old_rp;
195 static DEFINE_RCU_RANDOM(rand);
196
197 VERBOSE_PRINTK_STRING("rcu_torture_writer task started");
198 do {
199 schedule_timeout_uninterruptible(1);
200 if (rcu_batches_completed() == oldbatch)
201 continue;
202 if ((rp = rcu_torture_alloc()) == NULL)
203 continue;
204 rp->rtort_pipe_count = 0;
205 udelay(rcu_random(&rand) & 0x3ff);
206 old_rp = rcu_torture_current;
207 rcu_assign_pointer(rcu_torture_current, rp);
208 smp_wmb();
209 if (old_rp != NULL) {
210 i = old_rp->rtort_pipe_count;
211 if (i > RCU_TORTURE_PIPE_LEN)
212 i = RCU_TORTURE_PIPE_LEN;
213 atomic_inc(&rcu_torture_wcount[i]);
214 old_rp->rtort_pipe_count++;
215 call_rcu(&old_rp->rtort_rcu, rcu_torture_cb);
216 }
217 rcu_torture_current_version++;
218 oldbatch = rcu_batches_completed();
219 } while (!kthread_should_stop() && !fullstop);
220 VERBOSE_PRINTK_STRING("rcu_torture_writer task stopping");
221 while (!kthread_should_stop())
222 schedule_timeout_uninterruptible(1);
223 return 0;
224}
225
226/*
227 * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current,
228 * incrementing the corresponding element of the pipeline array. The
229 * counter in the element should never be greater than 1, otherwise, the
230 * RCU implementation is broken.
231 */
232static int
233rcu_torture_reader(void *arg)
234{
235 int completed;
236 DEFINE_RCU_RANDOM(rand);
237 struct rcu_torture *p;
238 int pipe_count;
239
240 VERBOSE_PRINTK_STRING("rcu_torture_reader task started");
241 do {
242 rcu_read_lock();
243 completed = rcu_batches_completed();
244 p = rcu_dereference(rcu_torture_current);
245 if (p == NULL) {
246 /* Wait for rcu_torture_writer to get underway */
247 rcu_read_unlock();
248 schedule_timeout_interruptible(HZ);
249 continue;
250 }
251 udelay(rcu_random(&rand) & 0x7f);
252 preempt_disable();
253 pipe_count = p->rtort_pipe_count;
254 if (pipe_count > RCU_TORTURE_PIPE_LEN) {
255 /* Should not happen, but... */
256 pipe_count = RCU_TORTURE_PIPE_LEN;
257 }
258 ++__get_cpu_var(rcu_torture_count)[pipe_count];
259 completed = rcu_batches_completed() - completed;
260 if (completed > RCU_TORTURE_PIPE_LEN) {
261 /* Should not happen, but... */
262 completed = RCU_TORTURE_PIPE_LEN;
263 }
264 ++__get_cpu_var(rcu_torture_batch)[completed];
265 preempt_enable();
266 rcu_read_unlock();
267 schedule();
268 } while (!kthread_should_stop() && !fullstop);
269 VERBOSE_PRINTK_STRING("rcu_torture_reader task stopping");
270 while (!kthread_should_stop())
271 schedule_timeout_uninterruptible(1);
272 return 0;
273}
274
275/*
276 * Create an RCU-torture statistics message in the specified buffer.
277 */
278static int
279rcu_torture_printk(char *page)
280{
281 int cnt = 0;
282 int cpu;
283 int i;
284 long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
285 long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
286
287 for_each_cpu(cpu) {
288 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
289 pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i];
290 batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i];
291 }
292 }
293 for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) {
294 if (pipesummary[i] != 0)
295 break;
296 }
297 cnt += sprintf(&page[cnt], "rcutorture: ");
298 cnt += sprintf(&page[cnt],
299 "rtc: %p ver: %ld tfle: %d rta: %d rtaf: %d rtf: %d",
300 rcu_torture_current,
301 rcu_torture_current_version,
302 list_empty(&rcu_torture_freelist),
303 atomic_read(&n_rcu_torture_alloc),
304 atomic_read(&n_rcu_torture_alloc_fail),
305 atomic_read(&n_rcu_torture_free));
306 cnt += sprintf(&page[cnt], "\nrcutorture: ");
307 if (i > 1)
308 cnt += sprintf(&page[cnt], "!!! ");
309 cnt += sprintf(&page[cnt], "Reader Pipe: ");
310 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
311 cnt += sprintf(&page[cnt], " %ld", pipesummary[i]);
312 cnt += sprintf(&page[cnt], "\nrcutorture: ");
313 cnt += sprintf(&page[cnt], "Reader Batch: ");
314 for (i = 0; i < RCU_TORTURE_PIPE_LEN; i++)
315 cnt += sprintf(&page[cnt], " %ld", batchsummary[i]);
316 cnt += sprintf(&page[cnt], "\nrcutorture: ");
317 cnt += sprintf(&page[cnt], "Free-Block Circulation: ");
318 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
319 cnt += sprintf(&page[cnt], " %d",
320 atomic_read(&rcu_torture_wcount[i]));
321 }
322 cnt += sprintf(&page[cnt], "\n");
323 return cnt;
324}
325
326/*
327 * Print torture statistics. Caller must ensure that there is only
328 * one call to this function at a given time!!! This is normally
329 * accomplished by relying on the module system to only have one copy
330 * of the module loaded, and then by giving the rcu_torture_stats
331 * kthread full control (or the init/cleanup functions when rcu_torture_stats
332 * thread is not running).
333 */
334static void
335rcu_torture_stats_print(void)
336{
337 int cnt;
338
339 cnt = rcu_torture_printk(printk_buf);
340 printk(KERN_ALERT "%s", printk_buf);
341}
342
343/*
344 * Periodically prints torture statistics, if periodic statistics printing
345 * was specified via the stat_interval module parameter.
346 *
347 * No need to worry about fullstop here, since this one doesn't reference
348 * volatile state or register callbacks.
349 */
350static int
351rcu_torture_stats(void *arg)
352{
353 VERBOSE_PRINTK_STRING("rcu_torture_stats task started");
354 do {
355 schedule_timeout_interruptible(stat_interval * HZ);
356 rcu_torture_stats_print();
357 } while (!kthread_should_stop());
358 VERBOSE_PRINTK_STRING("rcu_torture_stats task stopping");
359 return 0;
360}
361
362static void
363rcu_torture_cleanup(void)
364{
365 int i;
366
367 fullstop = 1;
368 if (writer_task != NULL) {
369 VERBOSE_PRINTK_STRING("Stopping rcu_torture_writer task");
370 kthread_stop(writer_task);
371 }
372 writer_task = NULL;
373
374 if (reader_tasks != NULL) {
375 for (i = 0; i < nrealreaders; i++) {
376 if (reader_tasks[i] != NULL) {
377 VERBOSE_PRINTK_STRING(
378 "Stopping rcu_torture_reader task");
379 kthread_stop(reader_tasks[i]);
380 }
381 reader_tasks[i] = NULL;
382 }
383 kfree(reader_tasks);
384 reader_tasks = NULL;
385 }
386 rcu_torture_current = NULL;
387
388 if (stats_task != NULL) {
389 VERBOSE_PRINTK_STRING("Stopping rcu_torture_stats task");
390 kthread_stop(stats_task);
391 }
392 stats_task = NULL;
393
394 /* Wait for all RCU callbacks to fire. */
395
396 for (i = 0; i < RCU_TORTURE_PIPE_LEN; i++)
397 synchronize_rcu();
398 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */
399 PRINTK_STRING("--- End of test");
400}
401
402static int
403rcu_torture_init(void)
404{
405 int i;
406 int cpu;
407 int firsterr = 0;
408
409 /* Process args and tell the world that the torturer is on the job. */
410
411 if (nreaders >= 0)
412 nrealreaders = nreaders;
413 else
414 nrealreaders = 2 * num_online_cpus();
415 printk(KERN_ALERT TORTURE_FLAG
416 "--- Start of test: nreaders=%d stat_interval=%d verbose=%d\n",
417 nrealreaders, stat_interval, verbose);
418 fullstop = 0;
419
420 /* Set up the freelist. */
421
422 INIT_LIST_HEAD(&rcu_torture_freelist);
423 for (i = 0; i < sizeof(rcu_tortures) / sizeof(rcu_tortures[0]); i++) {
424 list_add_tail(&rcu_tortures[i].rtort_free,
425 &rcu_torture_freelist);
426 }
427
428 /* Initialize the statistics so that each run gets its own numbers. */
429
430 rcu_torture_current = NULL;
431 rcu_torture_current_version = 0;
432 atomic_set(&n_rcu_torture_alloc, 0);
433 atomic_set(&n_rcu_torture_alloc_fail, 0);
434 atomic_set(&n_rcu_torture_free, 0);
435 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
436 atomic_set(&rcu_torture_wcount[i], 0);
437 for_each_cpu(cpu) {
438 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
439 per_cpu(rcu_torture_count, cpu)[i] = 0;
440 per_cpu(rcu_torture_batch, cpu)[i] = 0;
441 }
442 }
443
444 /* Start up the kthreads. */
445
446 VERBOSE_PRINTK_STRING("Creating rcu_torture_writer task");
447 writer_task = kthread_run(rcu_torture_writer, NULL,
448 "rcu_torture_writer");
449 if (IS_ERR(writer_task)) {
450 firsterr = PTR_ERR(writer_task);
451 VERBOSE_PRINTK_ERRSTRING("Failed to create writer");
452 writer_task = NULL;
453 goto unwind;
454 }
455 reader_tasks = kmalloc(nrealreaders * sizeof(reader_tasks[0]),
456 GFP_KERNEL);
457 if (reader_tasks == NULL) {
458 VERBOSE_PRINTK_ERRSTRING("out of memory");
459 firsterr = -ENOMEM;
460 goto unwind;
461 }
462 for (i = 0; i < nrealreaders; i++) {
463 VERBOSE_PRINTK_STRING("Creating rcu_torture_reader task");
464 reader_tasks[i] = kthread_run(rcu_torture_reader, NULL,
465 "rcu_torture_reader");
466 if (IS_ERR(reader_tasks[i])) {
467 firsterr = PTR_ERR(reader_tasks[i]);
468 VERBOSE_PRINTK_ERRSTRING("Failed to create reader");
469 reader_tasks[i] = NULL;
470 goto unwind;
471 }
472 }
473 if (stat_interval > 0) {
474 VERBOSE_PRINTK_STRING("Creating rcu_torture_stats task");
475 stats_task = kthread_run(rcu_torture_stats, NULL,
476 "rcu_torture_stats");
477 if (IS_ERR(stats_task)) {
478 firsterr = PTR_ERR(stats_task);
479 VERBOSE_PRINTK_ERRSTRING("Failed to create stats");
480 stats_task = NULL;
481 goto unwind;
482 }
483 }
484 return 0;
485
486unwind:
487 rcu_torture_cleanup();
488 return firsterr;
489}
490
491module_init(rcu_torture_init);
492module_exit(rcu_torture_cleanup);